diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 26b648c66084e..36d284ee60378 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -36,21 +36,11 @@ updates: schedule: interval: "daily" -- package-ecosystem: "pip" - directory: "/tools/github" - schedule: - interval: "daily" - - package-ecosystem: "pip" directory: "/tools/config_validation" schedule: interval: "daily" -- package-ecosystem: "pip" - directory: "/tools/docker" - schedule: - interval: "daily" - - package-ecosystem: "pip" directory: "/tools/dependency" schedule: @@ -62,7 +52,7 @@ updates: interval: "daily" - package-ecosystem: "pip" - directory: "/tools/gpg" + directory: "/tools/distribution" schedule: interval: "daily" diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 659bfefac0ee6..6b9c483a6ea72 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -44,29 +44,19 @@ def _python_deps(): requirements = "@envoy//tools/docs:requirements.txt", extra_pip_args = ["--require-hashes"], ) - pip_install( - name = "docker_pip3", - requirements = "@envoy//tools/docker:requirements.txt", - extra_pip_args = ["--require-hashes"], - ) pip_install( name = "deps_pip3", requirements = "@envoy//tools/dependency:requirements.txt", extra_pip_args = ["--require-hashes"], ) pip_install( - name = "git_pip3", - requirements = "@envoy//tools/git:requirements.txt", - extra_pip_args = ["--require-hashes"], - ) - pip_install( - name = "github_pip3", - requirements = "@envoy//tools/github:requirements.txt", + name = "distribution_pip3", + requirements = "@envoy//tools/distribution:requirements.txt", extra_pip_args = ["--require-hashes"], ) pip_install( - name = "gpg_pip3", - requirements = "@envoy//tools/gpg:requirements.txt", + name = "git_pip3", + requirements = "@envoy//tools/git:requirements.txt", extra_pip_args = ["--require-hashes"], ) pip_install( diff --git a/tools/base/BUILD b/tools/base/BUILD index c7c1de9dcbe33..8de9977da9dcd 100644 --- a/tools/base/BUILD +++ b/tools/base/BUILD @@ -6,12 +6,14 @@ licenses(["notice"]) # Apache 2 envoy_package() -envoy_py_library("tools.base.abstract") +exports_files([ + "base_command.py", +]) envoy_py_library( "tools.base.aio", deps = [ - ":functional", + requirement("aio.functional"), ], ) @@ -22,8 +24,6 @@ envoy_py_library( ], ) -envoy_py_library("tools.base.functional") - envoy_py_library( "tools.base.runner", deps = [ diff --git a/tools/base/abstract.py b/tools/base/abstract.py deleted file mode 100644 index 88d6f8ccb370a..0000000000000 --- a/tools/base/abstract.py +++ /dev/null @@ -1,165 +0,0 @@ -from abc import ABCMeta -from typing import Any, Dict, List, Tuple, Type, Union - - -class Implementer(type): - """Metaclass for implementers of an Abstract interface - - Any `Abstraction` classes that are listed in `__implements__` for the - class are added as bases (ie added to the class's inheritance). - - Any docs for methods are copied from the interface method to the - implementation if the implementation method has no docs of its own. - - For example: - - ``` - - from tools.base.abstract import Abstraction, Implementer - - - class AFoo(metaclass=Abstraction): - - @abstractmethod - def do_something(self): - \"""Do something\""" - raise NotImplementedError - - class Foo(metaclass=Implementer) - __implements__ = (AFoo, ) - - def do_something(self): - return "DONE" - ``` - Given the above, you should see that instantiating `Foo`: - - ``` - >>> isinstance(Foo(), AFoo) - True - >>> Foo().do_something.__doc__ - 'Do something' - ``` - """ - - @classmethod - def abstract_info(cls, abstract: "Abstraction") -> Tuple[str, Union[str, None], List[str]]: - """Information for a specific abstract implementation class - - For given abstract class, returns: - - - qualified class name - - class docstring - - abstract methods - """ - if not isinstance(abstract, Abstraction): - raise TypeError( - f"Implementers can only implement subclasses of `abstract.Abstraction`, unrecognized: '{abstract}'" - ) - methods: List[str] = [] - for method in getattr(abstract, "__abstractmethods__", []): - methods.append(method) - return f"{abstract.__module__}.{abstract.__name__}", abstract.__doc__, methods - - @classmethod - def add_docs(cls, clsdict: Dict, klass: "Implementer") -> None: - """Add docs to the implementation class - - If the implementation class has no docstring, then a docstring is generated with - the format: - - ``` - Implements: foo.bar.ABaz - An implementer of the ABaz protocol... - - Implements: foo.bar.AOtherBaz - An implementer of the AOtherBaz protocol... - ``` - - For each of the methods that are marked abstract in any of the abstract - classes, if the method in the implementation class has no docstring the docstring - is resolved from the abstract methods, using standard mro. - """ - abstract_docs, abstract_methods = cls.implementation_info(clsdict) - if not klass.__doc__: - klass.__doc__ = "\n".join(f"Implements: {k}\n{v}\n" for k, v in abstract_docs.items()) - for abstract_method, abstract_klass in abstract_methods.items(): - method = getattr(klass, abstract_method, None) - if not method: - # this will not instantiate, so bail now - return - # Only set the doc for the method if its not already set. - # `@classmethod` `__doc__`s are immutable, so skip them. - if not method.__doc__ and not hasattr(method, "__self__"): - method.__doc__ = getattr(abstract_klass, abstract_method).__doc__ - - @classmethod - def get_bases(cls, bases: Tuple[Type, ...], clsdict: Dict) -> Tuple[Type, ...]: - """Returns a tuple of base classes, with `__implements__` classes included""" - return bases + tuple(x for x in clsdict["__implements__"] if x not in bases) - - @classmethod - def implementation_info(cls, clsdict: Dict) -> Tuple[Dict[str, str], Dict[str, Type]]: - """Returns 2 dictionaries - - - abstract_docs: abstract docs for all abstract classes - - abstract_methods: resolved abstract methods -> abstract class - """ - abstract_docs: Dict[str, str] = {} - abstract_methods: Dict[str, Type] = {} - for abstract in reversed(clsdict["__implements__"]): - name, docs, methods = cls.abstract_info(abstract) - for method in methods: - abstract_methods[method] = abstract - if docs: - abstract_docs[name] = docs - return abstract_docs, abstract_methods - - def __new__(cls, clsname: str, bases: Tuple[Type, ...], clsdict: Dict) -> "Implementer": - """Create a new Implementer class""" - if "__implements__" not in clsdict: - return super().__new__(cls, clsname, bases, clsdict) - klass = super().__new__(cls, clsname, cls.get_bases(bases, clsdict), clsdict) - cls.add_docs(clsdict, klass) - return klass - - -class Abstraction(Implementer, ABCMeta): - pass - - -def implementer(implements): - """Decorator for implementers - - Assuming you have an abstract class `AFoo` which has a `metaclass` of - type `Abstraction`, it can be used as follows: - - ``` - from tools.base import abstract - - @abstract.implementer(AFoo) - class Foo: - - def bar(self): - return "BAZ" - - ``` - - """ - - if not isinstance(implements, (tuple, list, set)): - implements = (implements,) - - def wrapper(klass: Any) -> Implementer: - # This is a v annoying workaround for mypy, see: - # https://github.com/python/mypy/issues/9183 - _klass: Any = klass - - class Implementation(_klass, metaclass=Implementer): - __implements__ = implements - __doc__ = _klass.__doc__ - - Implementation.__qualname__ = klass.__name__ - Implementation.__name__ = klass.__name__ - return Implementation - - return wrapper diff --git a/tools/base/aio.py b/tools/base/aio.py index 5ef5ea2a04f08..a07787c31133f 100644 --- a/tools/base/aio.py +++ b/tools/base/aio.py @@ -9,7 +9,7 @@ Any, AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Iterable, Iterator, List, Optional, Union) -from tools.base.functional import async_property +from aio.functional import async_property class ConcurrentError(Exception): diff --git a/tools/base/functional.py b/tools/base/functional.py deleted file mode 100644 index 004132f3a8e50..0000000000000 --- a/tools/base/functional.py +++ /dev/null @@ -1,87 +0,0 @@ -# -# Functional utilities -# - -import inspect -from typing import Any, Callable, Optional - - -class NoCache(Exception): # noqa: N818 - pass - - -class async_property: # noqa: N801 - name = None - cache_name = "__async_prop_cache__" - _instance = None - - # If the decorator is called with `kwargs` then `fun` is `None` - # and instead `__call__` is triggered with `fun` - def __init__(self, fun: Optional[Callable] = None, cache: bool = False): - self.cache = cache - self._fun = fun - self.name = getattr(fun, "__name__", None) - self.__doc__ = getattr(fun, '__doc__') - - def __call__(self, fun: Callable) -> 'async_property': - self._fun = fun - self.name = self.name or fun.__name__ - self.__doc__ = getattr(fun, '__doc__') - return self - - def __get__(self, instance: Any, cls=None) -> Any: - if instance is None: - return self - self._instance = instance - if inspect.isasyncgenfunction(self._fun): - return self.async_iter_result() - return self.async_result() - - def fun(self, *args, **kwargs): - if self._fun: - return self._fun(*args, **kwargs) - - @property - def prop_cache(self) -> dict: - return getattr(self._instance, self.cache_name, {}) - - # An async wrapper function to return the result - # This is returned when the prop is called if the wrapped - # method is an async generator - async def async_iter_result(self): - # retrieve the value from cache if available - try: - result = self.get_cached_prop() - except (NoCache, KeyError): - result = None - - if result is None: - result = self.set_prop_cache(self.fun(self._instance)) - - async for item in result: - yield item - - # An async wrapper function to return the result - # This is returned when the prop is called - async def async_result(self) -> Any: - # retrieve the value from cache if available - try: - return self.get_cached_prop() - except (NoCache, KeyError): - pass - - # derive the result, set the cache if required, and return the result - return self.set_prop_cache(await self.fun(self._instance)) - - def get_cached_prop(self) -> Any: - if not self.cache: - raise NoCache - return self.prop_cache[self.name] - - def set_prop_cache(self, result: Any) -> Any: - if not self.cache: - return result - cache = self.prop_cache - cache[self.name] = result - setattr(self._instance, self.cache_name, cache) - return result diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index ef0b192cca2d5..658174e0bc9ff 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -4,6 +4,12 @@ # # pip-compile --allow-unsafe --generate-hashes tools/base/requirements.txt # +abstracts==0.0.12 \ + --hash=sha256:acc01ff56c8a05fb88150dff62e295f9071fc33388c42f1dfc2787a8d1c755ff + # via aio.functional +aio.functional==0.0.9 \ + --hash=sha256:824a997a394ad891bc9f403426babc13c9d0d1f4d1708c38e77d6aecae1cab1d + # via -r tools/base/requirements.txt colorama==0.4.4 \ --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 @@ -13,8 +19,8 @@ coloredlogs==15.0.1 \ --hash=sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0 # via -r tools/base/requirements.txt frozendict==2.0.6 \ - --hash=sha256:5d3f75832c35d4df041f0e19c268964cbef29c1eb34cd3517cf883f1c2d089b9 \ - --hash=sha256:3f00de72805cf4c9e81b334f3f04809278b967d2fed84552313a0fcce511beb1 + --hash=sha256:3f00de72805cf4c9e81b334f3f04809278b967d2fed84552313a0fcce511beb1 \ + --hash=sha256:5d3f75832c35d4df041f0e19c268964cbef29c1eb34cd3517cf883f1c2d089b9 # via -r tools/base/requirements.txt humanfriendly==9.2 \ --hash=sha256:332da98c24cc150efcc91b5508b19115209272bfdf4b0764a56795932f854271 \ diff --git a/tools/base/tests/test_abstract.py b/tools/base/tests/test_abstract.py deleted file mode 100644 index c996233606a04..0000000000000 --- a/tools/base/tests/test_abstract.py +++ /dev/null @@ -1,253 +0,0 @@ -from abc import abstractmethod -from random import random -from unittest.mock import MagicMock - -import pytest - -from tools.base import abstract - - -def test_implementer_constructor(): - with pytest.raises(TypeError): - abstract.Implementer() - - assert issubclass(abstract.Implementer, type) - - -@pytest.mark.parametrize("isinst", [True, False]) -def test_implementer_abstract_info(patches, isinst): - abstract_methods = [f"METHOD{i}" for i in range(0, 5)] - abstract_class = MagicMock() - abstract_class.__abstractmethods__ = abstract_methods - abstract_class.__name__ = MagicMock() - - patched = patches( - "isinstance", - prefix="tools.base.abstract") - - with patched as (m_inst, ): - m_inst.return_value = isinst - if not isinst: - with pytest.raises(TypeError) as e: - abstract.Implementer.abstract_info(abstract_class) - assert ( - e.value.args[0] - == f"Implementers can only implement subclasses of `abstract.Abstraction`, unrecognized: '{abstract_class}'") - else: - assert ( - abstract.Implementer.abstract_info(abstract_class) - == (f"{abstract_class.__module__}.{abstract_class.__name__}", - abstract_class.__doc__, - abstract_methods)) - - -@pytest.mark.parametrize("methods", [[], [f"METHOD{i}" for i in range(0, 2)], [f"METHOD{i}" for i in range(0, 5)]]) -@pytest.mark.parametrize("has_docs", [[], [f"METHOD{i}" for i in range(0, 2)], [f"METHOD{i}" for i in range(0, 5)]]) -@pytest.mark.parametrize("is_classmethod", [[], [f"METHOD{i}" for i in range(0, 2)], [f"METHOD{i}" for i in range(0, 5)]]) -@pytest.mark.parametrize("doc", [True, False]) -def test_implementer_add_docs(patches, methods, has_docs, doc, is_classmethod): - abstract_docs = {f"DOC{i}": int(random() * 10) for i in range(0, 5)} - abstract_methods = {f"METHOD{i}": MagicMock() for i in range(0, 5)} - - clsdict = MagicMock() - klass = MagicMock() - - if not doc: - klass.__doc__ = "" - - for method, abstract_klass in abstract_methods.items(): - getattr(abstract_klass, method).__doc__ = "KLASS DOCS" - if method not in methods: - delattr(klass, method) - continue - if method not in has_docs: - getattr(klass, method).__doc__ = "" - if method in is_classmethod: - getattr(klass, method).__self__ = "CLASSMETHOD_CLASS" - - patched = patches( - "Implementer.implementation_info", - prefix="tools.base.abstract") - - with patched as (m_info, ): - m_info.return_value = abstract_docs, abstract_methods - assert not abstract.Implementer.add_docs(clsdict, klass) - - assert ( - klass.__doc__ - == (MagicMock.__doc__ - if doc - else "\n".join(f"Implements: {k}\n{v}\n" for k, v in abstract_docs.items()))) - - failed = False - for abstract_method, abstract_klass in abstract_methods.items(): - if not abstract_method in methods: - continue - expected = MagicMock.__doc__ - if abstract_method in is_classmethod and abstract_method not in has_docs: - expected = "" - elif abstract_method not in has_docs: - expected = "KLASS DOCS" - assert ( - getattr(klass, abstract_method).__doc__ - == expected) - - -@pytest.mark.parametrize("bases", [(), ("A", "B", "C"), ("A", "C")]) -@pytest.mark.parametrize("implements", [(), ("A", "B", "C"), ("A", "C")]) -def test_implementer_get_bases(bases, implements): - clsdict = dict(__implements__=implements) - assert ( - abstract.Implementer.get_bases(bases, clsdict) - == bases + tuple(x for x in clsdict["__implements__"] if x not in bases)) - - -@pytest.mark.parametrize("has_docs", ["odd", "even"]) -def test_implementer_implementation_info(patches, has_docs): - patched = patches( - "Implementer.abstract_info", - prefix="tools.base.abstract") - - def iter_implements(): - for x in range(0, 5): - yield f"ABSTRACT{x}" - - implements = list(iter_implements()) - clsdict = dict(__implements__=implements) - - def abstract_info(abstract): - x = int(abstract[-1]) - methods = [f"METHOD{i}" for i in range(0, x + 1)] - docs = ( - x % 2 - if has_docs == "even" - else not x % 2) - return abstract, docs, methods - - with patched as (m_info, ): - m_info.side_effect = abstract_info - docs, methods = abstract.Implementer.implementation_info(clsdict) - - def oddeven(i): - return ( - i % 2 - if has_docs == "even" - else not i % 2) - - assert docs == {f"ABSTRACT{i}": 1 for i in range(0, 5) if oddeven(i)} - assert ( - methods - == {f'METHOD{i}': f'ABSTRACT{i}' for i in range(0, 5)}) - - -@pytest.mark.parametrize("has_implements", [True, False]) -def test_implementer_dunder_new(patches, has_implements): - patched = patches( - "super", - "Implementer.add_docs", - prefix="tools.base.abstract") - bases = MagicMock() - clsdict = dict(FOO="BAR") - - if has_implements: - clsdict["__implements__"] = "IMPLEMENTS" - - class SubImplementer(abstract.Implementer): - pass - - class Super: - - def __new__(cls, *args): - if not args: - return cls - return "NEW" - - with patched as (m_super, m_docs): - m_super.side_effect = Super - assert ( - abstract.Implementer.__new__(abstract.Implementer, "NAME", bases, clsdict) - == "NEW") - - if has_implements: - assert ( - list(m_docs.call_args) - == [(clsdict, "NEW"), {}]) - else: - assert not m_docs.called - - -class AFoo(metaclass=abstract.Abstraction): - - @classmethod - @abstractmethod - def do_something_classy(cls): - pass - - @abstractmethod - def do_something(self): - """Do something""" - raise NotImplementedError - - -class ABar(metaclass=abstract.Abstraction): - - @abstractmethod - def do_something_else(self): - """Do something else""" - raise NotImplementedError - - -class Baz: - - def do_nothing(self): - pass - - -@pytest.mark.parametrize("implements", [(), AFoo, (AFoo, ), (AFoo, ABar), (AFoo, ABar, Baz), Baz]) -def test_implementer_decorator(patches, implements): - iterable_implements = ( - implements - if isinstance(implements, tuple) - else (implements, )) - should_fail = any( - not isinstance(impl, abstract.Abstraction) - for impl - in iterable_implements) - - if should_fail: - with pytest.raises(TypeError): - _implementer(implements) - return - - implementer = _implementer(implements) - for impl in iterable_implements: - assert issubclass(implementer, impl) - assert ( - implementer.__name__ - == implementer.__qualname__ - == "ImplementationOfAnImplementer") - assert ( - implementer.__doc__ - == 'A test implementation of an implementer') - - -def _implementer(implements): - - @abstract.implementer(implements) - class ImplementationOfAnImplementer: - """A test implementation of an implementer""" - - @classmethod - def do_something_classy(cls): - pass - - def do_something(self): - pass - - def do_something_else(self): - pass - - def do_nothing(self): - pass - - return ImplementationOfAnImplementer diff --git a/tools/base/tests/test_functional.py b/tools/base/tests/test_functional.py deleted file mode 100644 index c8631714897b1..0000000000000 --- a/tools/base/tests/test_functional.py +++ /dev/null @@ -1,119 +0,0 @@ -import types -from unittest.mock import AsyncMock - -import pytest - -from tools.base import functional - - -@pytest.mark.asyncio -@pytest.mark.parametrize("cache", [None, True, False]) -@pytest.mark.parametrize("raises", [True, False]) -@pytest.mark.parametrize("result", [None, False, "X", 23]) -async def test_functional_async_property(cache, raises, result): - m_async = AsyncMock(return_value=result) - - class SomeError(Exception): - pass - - if cache is None: - decorator = functional.async_property - iter_decorator = functional.async_property - else: - decorator = functional.async_property(cache=cache) - iter_decorator = functional.async_property(cache=cache) - - items = [f"ITEM{i}" for i in range(0, 5)] - - class Klass: - - @decorator - async def prop(self): - """This prop deserves some docs""" - if raises: - await m_async() - raise SomeError("AN ERROR OCCURRED") - else: - return await m_async() - - @iter_decorator - async def iter_prop(self): - """This prop also deserves some docs""" - if raises: - await m_async() - raise SomeError("AN ITERATING ERROR OCCURRED") - result = await m_async() - for item in items: - yield item, result - - klass = Klass() - - # The class.prop should be an instance of async_prop - # and should have the name and docs of the wrapped method. - assert isinstance( - type(klass).prop, - functional.async_property) - assert ( - type(klass).prop.__doc__ - == "This prop deserves some docs") - assert ( - type(klass).prop.name - == "prop") - - if raises: - with pytest.raises(SomeError) as e: - await klass.prop - - with pytest.raises(SomeError) as e2: - async for result in klass.iter_prop: - pass - - assert ( - e.value.args[0] - == 'AN ERROR OCCURRED') - assert ( - e2.value.args[0] - == 'AN ITERATING ERROR OCCURRED') - assert ( - list(m_async.call_args_list) - == [[(), {}]] * 2) - return - - # results can be repeatedly awaited - assert await klass.prop == result - assert await klass.prop == result - - # results can also be repeatedly iterated - results1 = [] - async for returned_result in klass.iter_prop: - results1.append(returned_result) - assert results1 == [(item, result) for item in items] - - results2 = [] - async for returned_result in klass.iter_prop: - results2.append(returned_result) - - if not cache: - assert results2 == results1 - assert ( - list(list(c) for c in m_async.call_args_list) - == [[(), {}]] * 4) - assert not hasattr(klass, functional.async_property.cache_name) - return - - # with cache we can keep awaiting the result but the fun - # is still only called once - assert await klass.prop == result - assert await klass.prop == result - assert ( - list(list(c) for c in m_async.call_args_list) - == [[(), {}]] * 2) - - iter_prop = getattr(klass, functional.async_property.cache_name)["iter_prop"] - assert isinstance(iter_prop, types.AsyncGeneratorType) - assert ( - getattr(klass, functional.async_property.cache_name) - == dict(prop=m_async.return_value, iter_prop=iter_prop)) - - # cached iterators dont give any more results once they are done - assert results2 == [] diff --git a/tools/distribution/BUILD b/tools/distribution/BUILD index c63fecc51eb26..11142e4324f0e 100644 --- a/tools/distribution/BUILD +++ b/tools/distribution/BUILD @@ -1,5 +1,6 @@ load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_py_binary", "envoy_py_library") +load("//tools/base:envoy_python.bzl", "envoy_py_script") +load("@distribution_pip3//:requirements.bzl", "requirement") licenses(["notice"]) # Apache 2 @@ -9,31 +10,26 @@ exports_files([ "distrotest.sh", ]) -envoy_py_library( - name = "tools.distribution.distrotest", - data = [ - "distrotest.yaml", - ], +envoy_py_script( + name = "tools.distribution.release", + entry_point = "envoy.distribution.release", deps = [ - "//tools/base:checker", - "//tools/base:utils", - "//tools/docker:utils", + requirement("envoy.distribution.release"), ], ) -envoy_py_binary( +envoy_py_script( name = "tools.distribution.sign", + entry_point = "envoy.gpg.sign", deps = [ - "//tools/base:runner", - "//tools/base:utils", - "//tools/gpg:identity", + requirement("envoy.gpg.sign"), ], ) -envoy_py_binary( +envoy_py_script( name = "tools.distribution.verify", + entry_point = "envoy.distribution.verify", deps = [ - ":distrotest", - "//tools/base:checker", + requirement("envoy.distribution.verify"), ], ) diff --git a/tools/distribution/distrotest.py b/tools/distribution/distrotest.py deleted file mode 100644 index 265c99ba89817..0000000000000 --- a/tools/distribution/distrotest.py +++ /dev/null @@ -1,722 +0,0 @@ -import logging -import pathlib -import re -import shutil -from functools import cached_property -from itertools import chain -from typing import Callable, Iterable, List, Optional, Tuple, Type - -import verboselogs # type:ignore - -import aiodocker - -from tools.base import checker, utils -from tools.docker import utils as docker_utils - -DISTROTEST_CONFIG_PATH = "tools/distribution/distrotest.yaml" - -DOCKER_IMAGE_PREFIX = "envoybuild_" -DOCKER_CONTAINER_PREFIX = "envoytest_" -DOCKERFILE_TEMPLATE = """ -FROM {build_image} -{env} - -ADD {install_dir} {install_mount_path} -ADD {testfile_name} {test_mount_path} -ADD {keyfile_name} {key_mount_path} -RUN {build_command} - -CMD ["tail", "-f", "/dev/null"] -""" - - -class BuildError(Exception): - pass - - -class ConfigurationError(Exception): - pass - - -class ContainerError(Exception): - pass - - -class DistroTestConfig(object): - """Configuration object for distro tests - - This holds the configuration (and Docker connect) across a batch of tests. - - It also allows `DistroTest` to adapt `checker.Checker`. - - Init parameters: - - `path` is the path to the directory that will be used as the Docker context. - `tarball` is the path to a tarball containing packages. - `keyfile` is the path to the public key used to sign the packages - `testfile` is the bash script to run inside the test containers. - `maintainer` is the expected maintainer/packager the packages were signed with. - `version` is the expected version of the packages. - `config_path` allows you to override the default `distrotest.yaml` - """ - - packages_name = "packages" - - def __init__( - self, - docker: aiodocker.Docker, - path: pathlib.Path, - tarball: pathlib.Path, - keyfile: pathlib.Path, - testfile: pathlib.Path, - maintainer: str, - version: str, - config_path: Optional[pathlib.Path] = None): - self.docker = docker - self._keyfile = keyfile - self.path = path - self.tarball = tarball - self._testfile = testfile - self.maintainer = maintainer - self.version = version - self._config_path = config_path - - def __getitem__(self, k): - return self.config[k] - - @cached_property - def config(self) -> dict: - """Configuration for test types - eg deb/rpm - - This contains build information for different types of test image, - and some defaults for specific test configuration. - """ - result = utils.from_yaml(self.config_path) - if not isinstance(result, dict): - raise ConfigurationError(f"Unable to parse configuration: {self.config_path}") - return result - - @cached_property - def config_path(self) -> pathlib.Path: - """Path to a test configuration file""" - return pathlib.Path(self._config_path or DISTROTEST_CONFIG_PATH) - - @cached_property - def ctx_dockerfile(self) -> pathlib.Path: - """Path to the Dockerfile in the Docker context""" - return self.path.joinpath("Dockerfile") - - @cached_property - def ctx_keyfile(self) -> pathlib.Path: - """Path to the keyfile in the Docker context""" - return self.path.joinpath(self._keyfile.name) - - @cached_property - def rel_ctx_packages(self) -> pathlib.Path: - """Path to the directory (in the Docker context) containing packages to - test - """ - return self.path.joinpath(self.packages_name) - - @cached_property - def ctx_testfile(self) -> pathlib.Path: - """Path to the testfile in the Docker context""" - return self.path.joinpath(self._testfile.name) - - @cached_property - def images(self) -> dict: - """Mapping of images -> ext/types - - eg `debian` -> type=`deb` ext=`changes` - `registry.access.redhat.com/ubi8/ubi` -> type=`rpm` ext=`rpm` - - for each image: - - the `type` is used to find the directory of packages. - - the `ext` is used to find packages within the directory. - - """ - return dict( - chain.from_iterable(((image, dict(type=k, ext=v["ext"])) - for image in v["images"]) - for k, v in self.items())) - - @cached_property - def install_img_path(self) -> pathlib.PurePosixPath: - """Path to the install directory within the image/container""" - return pathlib.PurePosixPath("/tmp/install") - - @cached_property - def keyfile(self) -> pathlib.Path: - """Path to the keyfile in the Docker context - - Copies the keyfile to the path on first access. - """ - # Add the keyfile and return the path - shutil.copyfile(self._keyfile, self.ctx_keyfile) - return self.ctx_keyfile - - @cached_property - def keyfile_img_path(self) -> pathlib.PurePosixPath: - """Path to the public key of the key used to sign the packages, inside - the Docker image/container - """ - return pathlib.PurePosixPath("/tmp/gpg/signing.key") - - @cached_property - def packages_dir(self) -> pathlib.Path: - """The directory containing packages. - - Packages are extracted on first access - """ - utils.extract(self.rel_ctx_packages, self.tarball) - return self.rel_ctx_packages - - @cached_property - def testfile(self) -> pathlib.Path: - """Path to the testfile in the Docker context - - Copies the testfile to the path on first access. - """ - # Add the testfile - distrotest.sh - and return the path - shutil.copyfile(self._testfile, self.ctx_testfile) - return self.ctx_testfile - - @cached_property - def testfile_img_path(self) -> pathlib.PurePosixPath: - """Path to the testfile within the image/container""" - return pathlib.PurePosixPath("/tmp").joinpath(self.testfile.name) - - def get_config(self, image: str) -> dict: - """Return the type/ext config for a particular image - - If the full image name - ie `image:tag` is provided, the `tag` is removed. - """ - return self.images[self.get_image_name(image)] - - def get_image_name(self, image: str) -> str: - """Get the image part of a full Docker image tag - eg `debian:buster-slim` -> `debian`. - """ - return image.split(":")[0] - - def get_package_type(self, image: str) -> str: - """Get the package type for a particular image - eg `debian:buster-slim` will resolve to `deb` - - If it cannot resolve a type from the configuration in `distrotest.yaml` - it raises a `ConfigurationError` - """ - image = self.get_image_name(image) - for k, v in self.items(): - if image in v["images"]: - return k - raise ConfigurationError(f"Unrecognized image: {image}") - - def get_packages(self, type: str, ext: str) -> List[pathlib.Path]: - """List of packages of a given type/ext found for testing""" - return list(self.packages_dir.joinpath(type).glob(f"*.{ext}")) - - def items(self): - return self.config.items() - - -class DistroTestImage(object): - """A Docker image for running tests - - The image is installed with some basic utilities for testing. - - The image can be built if required. - - The built image also contains: - - - `self.dockerfile` - the `Dockerfile` build instructions - - `self.keyfile` - the path to a populated file containing the package - maintainer's public key. - - `self.testfile` - the path to a populated file containing the test script. - - These are loaded into the Docker context when building. - - Init paramaters: - - `build_image`: the image to build - eg debian/buster-slim - `name`: name to give the built image - eg `debian_buster` - `stream`: optional callable to stream Docker output to - """ - - def __init__( - self, - test_config: DistroTestConfig, - build_image: str, - name: str, - stream: Optional[Callable] = None): - self.test_config = test_config - self.build_image = build_image - self.name = name - self._stream = stream - - @property - def build_command(self) -> str: - """Command to build the Docker image""" - return self.config["build"]["command"].strip().replace("\n", " && ") - - @cached_property - def config(self) -> dict: - """Config specific to this type of Docker image""" - return self.test_config[self.package_type] - - @property - def ctx_dockerfile(self) -> pathlib.Path: - return self.test_config.ctx_dockerfile - - @property - def ctx_install_dir(self) -> pathlib.Path: - """Directory containing packages - - *relative to the Docker context root* - """ - return pathlib.Path(self.packages_name).joinpath(self.package_type) - - @property - def docker(self) -> aiodocker.Docker: - return self.test_config.docker - - @cached_property - def dockerfile(self) -> str: - """The contents of the build Dockerfile""" - return self.dockerfile_template.format( - build_image=self.build_image, - env=self.env, - build_command=self.build_command, - install_dir=self.ctx_install_dir, - install_mount_path=self.install_img_path, - testfile_name=self.testfile.name, - test_mount_path=self.testfile_img_path, - keyfile_name=self.keyfile.name, - key_mount_path=self.keyfile_img_path) - - @property - def dockerfile_template(self) -> str: - """Dockerfile template""" - return DOCKERFILE_TEMPLATE - - @property - def env(self) -> str: - """The `ENV` string to use in the `Dockerfile`""" - _env = self.config["build"].get("env", "") - return f"ENV {_env}" if _env else "" - - @property - def install_img_path(self) -> pathlib.PurePosixPath: - return self.test_config.install_img_path - - @property - def keyfile_img_path(self) -> pathlib.PurePosixPath: - return self.test_config.keyfile_img_path - - @property - def keyfile(self) -> pathlib.Path: - return self.test_config.keyfile - - @cached_property - def package_type(self) -> str: - return self.test_config.get_package_type(self.build_image) - - @property - def packages_name(self) -> str: - return self.test_config.packages_name - - @property - def path(self) -> pathlib.Path: - return self.test_config.path - - @property - def prefix(self) -> str: - """Prefix for the Docker image name that we be built""" - return DOCKER_IMAGE_PREFIX - - @cached_property - def tag(self) -> str: - """Tag for the Docker test image build""" - return f"{self.prefix}{self.name}:latest" - - @property - def testfile(self) -> pathlib.Path: - return self.test_config.testfile - - @property - def testfile_img_path(self) -> pathlib.PurePosixPath: - return self.test_config.testfile_img_path - - def add_dockerfile(self) -> None: - """Add the Dockerfile for the test Docker image""" - self.stream(self.dockerfile) - self.ctx_dockerfile.write_text(self.dockerfile) - - async def build(self) -> None: - """Build the Docker image for the test""" - self.add_dockerfile() - try: - await docker_utils.build_image( - self.docker, self.path, self.tag, stream=self.stream, forcerm=True) - except docker_utils.BuildError as e: - raise BuildError(e.args[0]) - - async def exists(self) -> bool: - """Check if the Docker image exists already for the distribution""" - return self.tag in await self.images() - - def get_environment(self, package_filename: str, package_name: str, name: str) -> dict: - """Creates a dictionary of environment variables that are injected when - the test is `exec`ed - - Defaults are added from the global test configuration - (ie `distrotest.yaml`), the package `ext` can be overridden by the - passed in `yaml` test config file. - - Each var is formatted with the existing env dict, so you can - interpolate any previously defined vars. - """ - env = dict( - ENVOY_MAINTAINER=self.test_config.maintainer, - ENVOY_VERSION=self.test_config.version, - ENVOY_INSTALL_BINARY=self.installable_img_path( - self.get_install_binary(package_filename)), - ENVOY_INSTALLABLE=self.installable_img_path(package_filename), - PACKAGE=package_name, - DISTRO=name) - for k, v in self.config["test"].items(): - env[k.upper()] = v.format(**env) - return env - - def get_install_binary(self, package: str) -> str: - """Get the name of the installation binary from the installable file. - - For debian this will be the `.deb` file associated with the installable `.changes` file. - - For redhat its just the `.rpm` - """ - return ( - re.sub( - self.config["binary_name"]["match"], self.config["binary_name"]["replace"], package) - if "binary_name" in self.config else package) - - async def images(self) -> Iterable[str]: - """The currently built Docker image tag names""" - return chain.from_iterable([image["RepoTags"] for image in await self.docker.images.list()]) - - def installable_img_path(self, package_filename: str) -> pathlib.PurePosixPath: - """Path to a package inside the container""" - return self.install_img_path.joinpath(package_filename) - - def stream(self, msg: str) -> None: - if self._stream: - self._stream(msg) - - -class DistroTest(object): - """A distribution <> package test - - The test image is only built if it does not exist already. - - The test starts the distro test container with the test image, and then - `execs` the test script inside the container to run the tests. - - Init parameters: - - `name`: the distro test name - eg `redhat_8.3` - `image`: the test image - eg `registry.access.redhat.com/ubi8/ubi:8.3` - `installable`: is the path to the actual package to test. - `rebuild`: flag to rebuild the image if it exists - """ - - def __init__( - self, - checker: checker.AsyncChecker, - test_config: DistroTestConfig, - name: str, - image: str, - installable: pathlib.Path, - rebuild: bool = False): - self.checker = checker - self.test_config = test_config - self.installable = installable - self.distro = name - self.build_image = image - self.rebuild = rebuild - self._failures: List[str] = [] - - @property - def config(self) -> dict: - """Docker container config""" - # Dont use `AutoRemove` as we want the logs from failed containers - return dict(Image=self.image.tag) - - @property - def docker(self) -> aiodocker.Docker: - """aiodocker.Docker connection""" - return self.test_config.docker - - @property - def environment(self) -> dict: - """Docker exec environment for the test""" - return self.image.get_environment(self.installable.name, self.package_name, self.distro) - - @property - def errors(self) -> dict: - """Dictionary of test errors stored on the provided Checker""" - return self.checker.errors - - @property - def exiting(self) -> bool: - """Flag to indicate that the program is exiting due to - `KeyboardInterrupt` - """ - return self.checker.exiting - - @property - def failed(self) -> bool: - """Flag to indicate whether there are test failures from running - the test inside the container - """ - return len(self.failures) > 0 - - @property - def failures(self) -> list: - """List of test failures from running the test inside the container - """ - return self._failures - - @cached_property - def image(self) -> DistroTestImage: - """A Docker image used for testing that can be built if required""" - return self.image_class( - self.test_config, self.build_image, self.distro, stream=self.stdout.info) - - @property - def image_class(self) -> Type[DistroTestImage]: - return DistroTestImage - - @property - def log(self) -> verboselogs.VerboseLogger: - """A logger to send progress information to""" - return self.checker.log - - @cached_property - def name(self) -> str: - """The name of the Docker container used to test""" - return f"{self.prefix}{self.distro}" - - @cached_property - def package_name(self) -> str: - """The name of the package derived from the filename - eg `envoy-1.19`""" - return self.installable.name.split("_")[0] - - @property - def prefix(self) -> str: - """Prefix for the container name""" - return DOCKER_CONTAINER_PREFIX - - @property - def stdout(self) -> logging.Logger: - """A logger for raw logging""" - return self.checker.stdout - - @property - def test_cmd(self) -> tuple: - """The test command to run inside the test container""" - return (str(self.test_config.testfile_img_path),) - - @property - def testfile(self) -> pathlib.Path: - """Path to the testfile""" - return self.test_config.testfile - - async def build(self) -> None: - """Build the Docker image for the test if required""" - if not self.rebuild and await self.image.exists(): - return - self.run_log("Building image", msg_type="notice") - await self.image.build() - self.run_log("Image built") - - async def cleanup(self) -> None: - """Attempt to kill the test container. - - As this is cleanup code, run when system is exiting, *ignore all errors*. - """ - try: - await self.stop(await self.docker.containers.get(self.name)) - finally: - return - - async def create(self) -> aiodocker.containers.DockerContainer: - """Create a Docker container for the test""" - return await self.docker.containers.create_or_replace(config=self.config, name=self.name) - - async def exec(self, container: aiodocker.containers.DockerContainer) -> None: - """Run Docker `exec` with the test""" - execute = await container.exec(self.test_cmd, environment=self.environment) - - # The reason for using `_out` here is to catch the situation where it - # outputs one log and then fails before any tests have run - # in that case we want to catch and log the error and not just send it to - # stdout - async with execute.start(detach=False) as stream: - msg = await stream.read_out() - _out = "" - while msg: - if _out: - self.handle_test_output(_out) - _out = msg.data.decode("utf-8").strip() - msg = await stream.read_out() - - # We only log an error if `exec` failed and there are no test failures - return_code = (await execute.inspect())["ExitCode"] - _log_error = _out and return_code and not self.failed - if _log_error: - self._failures.append("container-start") - self.error([f"[{self.distro}] Error executing test in container\n{_out}"]) - elif _out: - self.handle_test_output(_out) - - def error(self, errors: Optional[Iterable[str]]) -> int: - """Fail a test and log the errors""" - return self.checker.error(self.checker.active_check, errors) - - def handle_test_error(self, msg: str) -> None: - """Handle a test error - - Any "control" lines in the test that contain `ERROR` will cause the - test to fail and any additional lines are output to stderr. - """ - # testrun is eg `debian_buster/envoy-1.19` - # testname is eg `proxy-responds` - testrun, testname = msg.split("]")[0].strip("[").split(":") - - # Record the failure for summarizing - self._failures.append(testname) - - # Fail the test, log an error, and output any extra `msg` content as - # raw logs - self.error([f"[{testrun}:{testname}] Test failed"]) - _msg = msg.split("\n", 1) - if len(_msg) > 1: - self.stdout.error(_msg[1]) - - def handle_test_output(self, msg: str) -> None: - """Handle and log stream from test container - - If the message startswith eg `[debian_buster/envoy-19` then treat the - message as a control message, otherwise log directly to stdout. - - If a control message contains `ERROR` then its treated as an error, - and the test is marked as failed - - If a non-control message contains `\n` then the first line is split - and output, and the method recurses with the remainder. - """ - if not msg.startswith(f"[{self.distro}"): - if "\n" not in msg: - # raw log - self.stdout.info(msg) - return - - # Sometimes lines come joined together. This handles that, - # and prevents control messages being missed. - _msg = msg.split("\n", 1) - self.stdout.info(_msg[0]) - self.handle_test_output(_msg[1]) - return - - if "ERROR" not in msg: - # Log informational message - self.log.info(msg) - return - self.handle_test_error(msg) - - def log_failures(self) -> None: - """Log a failure summary of a test""" - if not self.failed: - return - self.run_log( - f"Package test had failures: {','.join(self.failures)}", - msg_type="error", - test=self.package_name) - - async def logs(self, container: aiodocker.containers.DockerContainer) -> str: - """Return the concatenated container logs, only called if the container fails to start""" - return ''.join(await container.log(stdout=True, stderr=True)) - - async def on_test_complete( - self, container: Optional[aiodocker.containers.DockerContainer], - failed: bool) -> Optional[Tuple[str]]: - """Stop the container and record the results""" - self.log_failures() - await self.stop(container) - if not (failed or self.failed): - self.checker.succeed( - self.checker.active_check, - [self.run_message(f"Package test passed", test=self.package_name)]) - - async def run(self) -> None: - """Run the test - build and start the container, and then exec the test inside""" - self.error(await self._run()) - - def run_log(self, message: str, msg_type: str = "info", test: Optional[str] = None) -> None: - """Log a message with test prefix""" - getattr(self.log, msg_type)(self.run_message(message, test=test)) - - def run_message(self, message: str, test: Optional[str] = None) -> str: - """A log message with relevant test prefix""" - return f"[{self.distro}/{test}] {message}" if test else f"[{self.distro}] {message}" - - async def start(self) -> aiodocker.containers.DockerContainer: - """Start and return the test container, error if it fails to start""" - container = await self.create() - await container.start() - info = await container.show() - if not info["State"]["Running"]: - raise ContainerError( - self.run_message( - f"Container unable to start\n{await self.logs(container)}", - test=self.package_name)) - self.run_log("Container started", test=self.package_name) - return container - - async def stop(self, container: Optional[aiodocker.containers.DockerContainer] = None) -> None: - """Stop the test container""" - if not container: - return - await container.kill() - await container.delete() - self.run_log("Container stopped", test=self.package_name) - - async def _run(self) -> Optional[Tuple[str, ...]]: - container = None - # As `finally` is always called, regardless of any errors being - # raised, we assume that something failed, unless build/start/exec - # complete without raising an error. - # actual test failures are recorded separately - failed = True - try: - # build, start and exec the container - await self.build() - container = await self.start() - await self.exec(container) - failed = False - except (BuildError, ConfigurationError, ContainerError) as e: - # Catch build/start/exec Docker errors and return - return e.args - except aiodocker.exceptions.DockerError as e: - # If there are any other Docker errors return the error message - return (e.args[1]["message"],) - finally: - # Stop the container and handle success/failure - try: - await self.on_test_complete(container, failed) - errors = None - except aiodocker.exceptions.DockerError as e: - # capture Docker errors from trying to stop the container - errors = (e.args[1]["message"],) - # Return errors from trying to stop the container if any - return errors diff --git a/tools/distribution/distrotest.yaml b/tools/distribution/distrotest.yaml deleted file mode 100644 index 826384276b835..0000000000000 --- a/tools/distribution/distrotest.yaml +++ /dev/null @@ -1,44 +0,0 @@ - -deb: - build: - env: DEBIAN_FRONTEND=noninteractive - command: | - chmod +x /tmp/distrotest.sh - apt-get update - apt-get install -y -qq -o=Dpkg::Use-Pty=0 --no-install-recommends curl devscripts gnupg2 procps sudo - mkdir /usr/share/debsign - gpg --no-default-keyring --keyring /usr/share/debsign/keyring.gpg --import /tmp/gpg/signing.key - binary_name: - # this transforms the `.changes` filename to the `.deb` filename - # eg `envoy.bullseye.changes` -> `envoy.deb` - match: (.*)\.[^.]+\.[^.]+$ - replace: \1.deb - ext: changes - images: - - debian - - ubuntu - test: - install_command: apt-get install -y -qq -o=Dpkg::Use-Pty=0 - uninstall_command: apt-get remove --purge -y -qq -o=Dpkg::Use-Pty=0 - maintainer_command: dpkg-deb -f {ENVOY_INSTALL_BINARY} maintainer - verify_command: dscverify --keyring /usr/share/debsign/keyring.gpg - binary_permissions: 755 root root - config_permissions: 555 root root - -rpm: - build: - command: | - chmod +x /tmp/distrotest.sh - echo 'localpkg_gpgcheck=1' >> /etc/yum.conf - rpm --import /tmp/gpg/signing.key - yum -y install procps sudo - ext: rpm - images: - - registry.access.redhat.com/ubi8/ubi - test: - install_command: "yum install -y -qq" - uninstall_command: "yum remove -y -qq" - maintainer_command: "rpm -q --queryformat '%{{PACKAGER}}' {ENVOY_INSTALLABLE}" - verify_command: "rpm -K" - binary_permissions: "555 envoy envoy" - config_permissions: "555 envoy envoy" diff --git a/tools/github/requirements.txt b/tools/distribution/requirements.txt similarity index 68% rename from tools/github/requirements.txt rename to tools/distribution/requirements.txt index 5b2f8e138755b..dcd1a7600ba2f 100644 --- a/tools/github/requirements.txt +++ b/tools/distribution/requirements.txt @@ -2,8 +2,39 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --generate-hashes tools/github/requirements.txt +# pip-compile --generate-hashes tools/distribution/requirements.txt # +abstracts==0.0.12 \ + --hash=sha256:acc01ff56c8a05fb88150dff62e295f9071fc33388c42f1dfc2787a8d1c755ff + # via + # aio.functional + # envoy.abstract.command + # envoy.github.abstract + # envoy.github.release +aio.functional==0.0.9 \ + --hash=sha256:824a997a394ad891bc9f403426babc13c9d0d1f4d1708c38e77d6aecae1cab1d + # via + # aio.tasks + # envoy.github.abstract + # envoy.github.release +aio.stream==0.0.2 \ + --hash=sha256:6f5baaff48f6319db134cd56c06ccf89db1f7c5f67a26382e081efc96f2f675d + # via envoy.github.release +aio.tasks==0.0.4 \ + --hash=sha256:9abd4b0881edb292c4f91a2f63b1dea7a9829a4bd4e8440225a1a412a90461fc + # via + # envoy.github.abstract + # envoy.github.release +aiodocker==0.21.0 \ + --hash=sha256:1f2e6db6377195962bb676d4822f6e3a0c525e1b5d60b8ebbab68230bff3d227 \ + --hash=sha256:6fe00135bb7dc40a407669d3157ecdfd856f3737d939df54f40a479d40cf7bdc + # via + # envoy.distribution.distrotest + # envoy.docker.utils +aiofiles==0.7.0 \ + --hash=sha256:a1c4fc9b2ff81568c83e21392a82f344ea9d23da906e4f6a52662764545e19d4 \ + --hash=sha256:c67a6823b5f23fcab0a2595a289cec7d8c863ffcb4322fb8cd6b90400aedfdbc + # via aio.stream aiohttp==3.7.4.post0 \ --hash=sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe \ --hash=sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe \ @@ -42,7 +73,11 @@ aiohttp==3.7.4.post0 \ --hash=sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc \ --hash=sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a \ --hash=sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95 - # via -r tools/github/requirements.txt + # via + # aio.stream + # aiodocker + # envoy.github.abstract + # envoy.github.release async-timeout==3.0.1 \ --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 @@ -102,29 +137,92 @@ chardet==4.0.0 \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 # via aiohttp +coloredlogs==15.0.1 \ + --hash=sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934 \ + --hash=sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0 + # via envoy.base.runner cryptography==3.4.8 \ - --hash=sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14 \ - --hash=sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7 \ --hash=sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e \ - --hash=sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085 \ --hash=sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b \ - --hash=sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb \ --hash=sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7 \ + --hash=sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085 \ --hash=sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc \ - --hash=sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5 \ - --hash=sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af \ --hash=sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a \ - --hash=sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06 \ --hash=sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498 \ - --hash=sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7 \ --hash=sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9 \ + --hash=sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c \ + --hash=sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7 \ + --hash=sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb \ + --hash=sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14 \ + --hash=sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af \ --hash=sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e \ - --hash=sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c + --hash=sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5 \ + --hash=sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06 \ + --hash=sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7 # via pyjwt +envoy.abstract.command==0.0.3 \ + --hash=sha256:4b7b15c91bea1f2eb7c2e8e35f95cd9437e1c8f151adc093bf7858fc85d48221 + # via + # envoy.base.runner + # envoy.distribution.release +envoy.base.checker==0.0.2 \ + --hash=sha256:2ac81efa20fd01fff644ff7dc7fadeac1c3e4dbb6210881ac7a7919ec0e048d8 + # via + # envoy.distribution.distrotest + # envoy.distribution.verify +envoy.base.runner==0.0.4 \ + --hash=sha256:4eeb2b661f1f0c402df4425852be554a8a83ef5d338bfae69ddcb9b90755379e + # via + # envoy.base.checker + # envoy.distribution.release + # envoy.github.abstract + # envoy.gpg.sign +envoy.base.utils==0.0.6 \ + --hash=sha256:58ed057137ebe80d78db90997efc59822115ee616e435a9afc3d7a19069bb53c + # via + # envoy.distribution.distrotest + # envoy.github.release + # envoy.gpg.sign +envoy.distribution.distrotest==0.0.3 \ + --hash=sha256:c094adbd959eb1336f93afc00aedb7ee4e68e8252e2365be816a6f9ede8a3de7 + # via envoy.distribution.verify +envoy.distribution.release==0.0.4 \ + --hash=sha256:41037e0488f0593ce5173739fe0cd1b45a4775f5a47738b85d9d04024ca241a2 + # via -r tools/distribution/requirements.txt +envoy.distribution.verify==0.0.2 \ + --hash=sha256:ae59134085de50203edf51c243dbf3301cbe5550db29f0ec6f9ea1c3b82fee1c + # via -r tools/distribution/requirements.txt +envoy.docker.utils==0.0.2 \ + --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76 + # via envoy.distribution.distrotest +envoy.github.abstract==0.0.16 \ + --hash=sha256:badf04104492fb6b37ba2163f2b225132ed04aba680beb218e7c7d918564f8ee + # via + # envoy.distribution.release + # envoy.github.release +envoy.github.release==0.0.8 \ + --hash=sha256:fbc4354030137eb565b8c4d679965e4ef60b01de0c09310441836e592ca0cd19 + # via envoy.distribution.release +envoy.gpg.identity==0.0.2 \ + --hash=sha256:7d32ff9133e00b9974b4dabd2512b4872b091b8c5069d0112240dcc1a56bc406 + # via envoy.gpg.sign +envoy.gpg.sign==0.0.3 \ + --hash=sha256:31667931f5d7ff05fd809b89748f277511486311c777652af4cb8889bd641049 + # via -r tools/distribution/requirements.txt +frozendict==2.0.6 \ + --hash=sha256:3f00de72805cf4c9e81b334f3f04809278b967d2fed84552313a0fcce511beb1 \ + --hash=sha256:5d3f75832c35d4df041f0e19c268964cbef29c1eb34cd3517cf883f1c2d089b9 + # via envoy.base.runner gidgethub==5.0.1 \ --hash=sha256:3efbd6998600254ec7a2869318bd3ffde38edc3a0d37be0c14bc46b45947b682 \ --hash=sha256:67245e93eb0918b37df038148af675df43b62e832c529d7f859f6b90d9f3e70d - # via -r tools/github/requirements.txt + # via + # envoy.github.abstract + # envoy.github.release +humanfriendly==9.2 \ + --hash=sha256:332da98c24cc150efcc91b5508b19115209272bfdf4b0764a56795932f854271 \ + --hash=sha256:f7dba53ac7935fd0b4a2fc9a29e316ddd9ea135fb3052d3d0279d10c18ff9c48 + # via coloredlogs idna==3.2 \ --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 @@ -173,7 +271,7 @@ multidict==5.1.0 \ packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 - # via -r tools/github/requirements.txt + # via envoy.github.release pycparser==2.20 \ --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 @@ -186,15 +284,63 @@ pyparsing==2.4.7 \ --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b # via packaging -typing-extensions==3.10.0.1 \ - --hash=sha256:8bbffbd37fbeb9747a0241fdfde5ae99d4531ad1d1a41ccaea62100e15a5814c \ - --hash=sha256:045dd532231acfa03628df5e0c66dba64e2cc8fc8b844538d4ad6d5dd6cb82dc \ - --hash=sha256:83af6730a045fda60f46510f7f1f094776d90321caa4d97d20ef38871bef4bd3 - # via aiohttp +python-gnupg==0.4.7 \ + --hash=sha256:2061f56b1942c29b92727bf9aecbd3cea3893acc9cccbdc7eb4604285efe4ac7 \ + --hash=sha256:3ff5b1bf5e397de6e1fe41a7c0f403dad4e242ac92b345f440eaecfb72a7ebae + # via envoy.gpg.identity +pyyaml==5.4.1 \ + --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ + --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ + --hash=sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393 \ + --hash=sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77 \ + --hash=sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922 \ + --hash=sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5 \ + --hash=sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8 \ + --hash=sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10 \ + --hash=sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc \ + --hash=sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018 \ + --hash=sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e \ + --hash=sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253 \ + --hash=sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347 \ + --hash=sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183 \ + --hash=sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541 \ + --hash=sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb \ + --hash=sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185 \ + --hash=sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc \ + --hash=sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db \ + --hash=sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa \ + --hash=sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46 \ + --hash=sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122 \ + --hash=sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b \ + --hash=sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63 \ + --hash=sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df \ + --hash=sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc \ + --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \ + --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \ + --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0 + # via envoy.base.utils +trycast==0.3.0 \ + --hash=sha256:1b7b4c0d4b0d674770a53f34a762e52a6cd6879eb251ab21625602699920080d \ + --hash=sha256:687185b812e8d1c45f2ba841e8de7bdcdee0695dcf3464f206800505d4c65f26 + # via envoy.base.utils +typing-extensions==3.10.0.2 \ + --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ + --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ + --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 + # via + # aiodocker + # aiohttp uritemplate==3.0.1 \ --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae # via gidgethub +verboselogs==1.7 \ + --hash=sha256:d63f23bf568295b95d3530c6864a0b580cec70e7ff974177dead1e4ffbc6ff49 \ + --hash=sha256:e33ddedcdfdafcb3a174701150430b11b46ceb64c2a9a26198c76a156568e427 + # via + # envoy.base.runner + # envoy.github.abstract + # envoy.github.release yarl==1.6.3 \ --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \ --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \ diff --git a/tools/distribution/sign.py b/tools/distribution/sign.py deleted file mode 100644 index 2fa35b882c78a..0000000000000 --- a/tools/distribution/sign.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env python3 - -# You will need to have the respective system tools required for -# package signing to use this tool. -# -# For example you will need debsign to sign debs, and rpmsign to -# sign rpms. -# -# usage -# -# with bazel: -# -# bazel run //tools/distribution:sign -- -h -# -# alternatively, if you have the necessary python deps available -# -# PYTHONPATH=. ./tools/distribution/sign.py -h -# -# python requires: coloredlogs, frozendict, python-gnupg, verboselogs -# - -import argparse -import pathlib -import shutil -import subprocess -import sys -import tarfile -from functools import cached_property -from itertools import chain -from typing import Iterator, Optional, Tuple, Type, Union - -import verboselogs # type:ignore - -from tools.base import runner, utils -from tools.gpg import identity - -# Replacable `__` maintainer/gpg config - python interpolation doesnt work easily -# with this string -RPMMACRO_TEMPLATE = """ -%_signature gpg -%_gpg_path __GPG_CONFIG__ -%_gpg_name __MAINTAINER__ -%_gpgbin __GPG_BIN__ -%__gpg_sign_cmd %{__gpg} gpg --force-v3-sigs --batch --verbose --no-armor --no-secmem-warning -u "%{_gpg_name}" -sbo %{__signature_filename} --digest-algo sha256 %{__plaintext_filename}' -""" - - -class SigningError(Exception): - pass - - -# Base directory signing util - - -class DirectorySigningUtil(object): - """Base class for signing utils - eg for deb or rpm packages""" - - command_name = "" - _package_type = "" - ext = "" - - def __init__( - self, - path: Union[pathlib.Path, str], - maintainer: identity.GPGIdentity, - log: verboselogs.VerboseLogger, - command: Optional[str] = ""): - self._path = path - self.maintainer = maintainer - self.log = log - self._command = command - - @cached_property - def command(self) -> str: - """Provided command name/path or path to available system version""" - command = self._command or shutil.which(self.command_name) - if command: - return command - raise SigningError(f"Signing software missing ({self.package_type}): {self.command_name}") - - @property - def command_args(self) -> tuple: - return () - - @property - def package_type(self) -> str: - return self._package_type or self.ext - - @property - def path(self) -> pathlib.Path: - return pathlib.Path(self._path) - - @property - def pkg_files(self) -> Tuple[pathlib.Path, ...]: - """Tuple of paths to package files to sign""" - # TODO?(phlax): check maintainer/packager field matches key id - return tuple( - pkg_file for pkg_file in self.path.glob("*") if pkg_file.name.endswith(f".{self.ext}")) - - def sign(self) -> None: - """Sign the packages""" - for pkg in self.pkg_files: - self.sign_pkg(pkg) - - def sign_command(self, pkg_file: pathlib.Path) -> tuple: - """Tuple of command parts to sign a specific package""" - return (self.command,) + self.command_args + (str(pkg_file),) - - def sign_pkg(self, pkg_file: pathlib.Path) -> None: - """Sign a specific package file""" - self.log.notice(f"Sign package ({self.package_type}): {pkg_file.name}") - response = subprocess.run( - self.sign_command(pkg_file), capture_output=True, encoding="utf-8") - - if response.returncode: - raise SigningError(response.stdout + response.stderr) - - self.log.success(f"Signed package ({self.package_type}): {pkg_file.name}") - - -# Runner - - -class PackageSigningRunner(runner.Runner): - """For a given `package_type` and `path` this will run the relevant signing - util for the packages they contain. - """ - - _signing_utils = () - - @classmethod - def register_util(cls, name: str, util: Type[DirectorySigningUtil]) -> None: - """Register util for signing a package type""" - cls._signing_utils = getattr(cls, "_signing_utils") + ((name, util),) - - @property - def extract(self) -> bool: - return self.args.extract - - @cached_property - def maintainer(self) -> identity.GPGIdentity: - """A representation of the maintainer with GPG capabilities""" - return self.maintainer_class(self.maintainer_name, self.maintainer_email, self.log) - - @property - def maintainer_class(self) -> Type[identity.GPGIdentity]: - return identity.GPGIdentity - - @property - def maintainer_email(self) -> str: - """Email of the maintainer if set""" - return self.args.maintainer_email - - @property - def maintainer_name(self) -> str: - """Name of the maintainer if set""" - return self.args.maintainer_name - - @property - def package_type(self) -> str: - """Package type - eg deb/rpm""" - return self.args.package_type - - @property - def path(self) -> pathlib.Path: - """Path to the packages directory""" - return pathlib.Path(self.args.path) - - @property - def tar(self) -> str: - return self.args.tar - - @cached_property - def signing_utils(self) -> dict: - """Configured signing utils - eg `DebSigningUtil`, `RPMSigningUtil`""" - return dict(getattr(self, "_signing_utils")) - - def add_arguments(self, parser: argparse.ArgumentParser) -> None: - super().add_arguments(parser) - parser.add_argument( - "path", default="", help="Path to the directory containing packages to sign") - parser.add_argument( - "--extract", - action="store_true", - help= - "If set, treat the path as a tarball containing directories according to package_type") - parser.add_argument("--tar", help="Path to save the signed packages as tar file") - parser.add_argument( - "--type", - default="", - choices=[c for c in self.signing_utils] + [""], - help="Package type to sign") - parser.add_argument( - "--maintainer-name", - default="", - help="Maintainer name to match when searching for a GPG key to match with") - parser.add_argument( - "--maintainer-email", - default="", - help="Maintainer email to match when searching for a GPG key to match with") - - def archive(self, path: Union[pathlib.Path, str]) -> None: - with tarfile.open(self.tar, "w") as tar: - tar.add(path, arcname=".") - - def get_signing_util(self, path: pathlib.Path) -> DirectorySigningUtil: - return self.signing_utils[path.name](path, self.maintainer, self.log) - - @runner.catches((identity.GPGError, SigningError)) - def run(self) -> None: - if self.extract: - self.sign_tarball() - else: - self.sign_directory() - self.log.success("Successfully signed packages") - - def sign(self, path: pathlib.Path) -> None: - self.log.notice(f"Signing {path.name}s ({self.maintainer}) {str(path)}") - self.get_signing_util(path).sign() - - def sign_all(self, path: pathlib.Path) -> None: - for directory in path.glob("*"): - if directory.name in self.signing_utils: - self.sign(directory) - - def sign_directory(self) -> None: - self.sign(self.path) - if self.tar: - self.archive(self.path) - - def sign_tarball(self) -> None: - if not self.tar: - raise SigningError("You must set a `--tar` file to save to when `--extract` is set") - with utils.untar(self.path) as tardir: - self.sign_all(tardir) - self.archive(tardir) - - -# RPM - - -class RPMMacro(object): - """`.rpmmacros` configuration for rpmsign""" - - _macro_filename = ".rpmmacros" - - def __init__(self, home: Union[pathlib.Path, str], overwrite: bool = False, **kwargs): - self._home = home - self.overwrite = bool(overwrite) - self.kwargs = kwargs - - @property - def home(self) -> pathlib.Path: - return pathlib.Path(self._home) - - @property - def path(self) -> pathlib.Path: - return self.home.joinpath(self._macro_filename) - - @property - def macro(self) -> str: - macro = self.template - for k, v in self.kwargs.items(): - macro = macro.replace(f"__{k.upper()}__", str(v)) - return macro - - @property - def template(self) -> str: - return RPMMACRO_TEMPLATE - - def write(self) -> None: - if not self.overwrite and self.path.exists(): - return - self.path.write_text(self.macro) - - -class RPMSigningUtil(DirectorySigningUtil): - """Sign all RPM packages in a given directory""" - - command_name = "rpmsign" - ext = "rpm" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.setup() - - @cached_property - def command(self) -> str: - if not (self.maintainer.gpg_bin and self.maintainer.gpg_bin.name == "gpg2"): - raise SigningError("GPG2 is required to sign RPM packages") - return super().command - - @cached_property - def command_args(self) -> tuple: - return ("--key-id", self.maintainer.fingerprint, "--addsign") - - @property - def rpmmacro(self) -> Type[RPMMacro]: - return RPMMacro - - def setup(self) -> None: - """Create the .rpmmacros file if it doesn't exist""" - self.rpmmacro( - self.maintainer.home, - maintainer=self.maintainer.name, - gpg_bin=self.maintainer.gpg_bin, - gpg_config=self.maintainer.gnupg_home).write() - - def sign_pkg(self, pkg_file: pathlib.Path) -> None: - pkg_file.chmod(0o755) - super().sign_pkg(pkg_file) - - -# Deb - - -class DebChangesFiles(object): - """Creates a set of `changes` files for specific distros from a src - `changes` file. - - eg, if src changes file is `envoy_1.100.changes` and `Distribution:` - field is `buster bullseye`, it creates: - - `envoy_1.100.changes` -> `envoy_1.100.buster.changes` - `envoy_1.100.changes` -> `envoy_1.100.bullseye.changes` - - while replacing any instances of the original distribution name in - the respective changes files, eg: - - `buster bullseye` -> `buster` - `buster bullseye` -> `bullseye` - - finally, it removes the src changes file. - """ - - def __init__(self, src): - self.src = src - - def __iter__(self) -> Iterator[pathlib.Path]: - """Iterate the required changes files, creating them, yielding the paths - of the newly created files, and deleting the original - """ - for path in self.files: - yield path - self.src.unlink() - - @cached_property - def distributions(self) -> str: - """Find and parse the `Distributions` header in the `changes` file""" - with open(self.src) as f: - line = f.readline() - while line: - if not line.startswith("Distribution:"): - line = f.readline() - continue - return line.split(":")[1].strip() - raise SigningError(f"Did not find Distribution field in changes file {self.src}") - - @property - def files(self) -> Iterator[pathlib.Path]: - """Create changes files for each distro, yielding the paths""" - for distro in self.distributions.split(): - yield self.changes_file(distro) - - def changes_file(self, distro: str) -> pathlib.Path: - """Create a `changes` file for a specific distro""" - target = self.changes_file_path(distro) - target.write_text(self.src.read_text().replace(self.distributions, distro)) - return target - - def changes_file_path(self, distro: str) -> pathlib.Path: - """Path to write the new changes file to""" - return self.src.with_suffix(f".{distro}.changes") - - -class DebSigningUtil(DirectorySigningUtil): - """Sign all `changes` packages in a given directory - - the `.changes` spec allows a single `.changes` file to have multiple `Distributions` listed. - - but, most package repos require a single signed `.change` file per distribution, with only one - distribution listed. - - this extracts the `.changes` files to -> per-distro `filename.distro.changes`, and removes - the original, before signing the files. - """ - - command_name = "debsign" - ext = "changes" - _package_type = "deb" - - @cached_property - def command_args(self) -> tuple: - return ("-k", self.maintainer.fingerprint) - - @property - def changes_files(self) -> Type[DebChangesFiles]: - return DebChangesFiles - - @cached_property - def pkg_files(self) -> tuple: - """Mangled .changes paths""" - return tuple(chain.from_iterable(self.changes_files(src) for src in super().pkg_files)) - - -# Setup - - -def _register_utils() -> None: - PackageSigningRunner.register_util("deb", DebSigningUtil) - PackageSigningRunner.register_util("rpm", RPMSigningUtil) - - -def main(*args) -> int: - _register_utils() - return PackageSigningRunner(*args).run() - - -if __name__ == "__main__": - sys.exit(main(*sys.argv[1:])) diff --git a/tools/distribution/tests/test_distrotest.py b/tools/distribution/tests/test_distrotest.py deleted file mode 100644 index 0ad033ddd99d9..0000000000000 --- a/tools/distribution/tests/test_distrotest.py +++ /dev/null @@ -1,1527 +0,0 @@ -import collections -import contextlib -from unittest.mock import AsyncMock, MagicMock, PropertyMock - -import pytest - -import aiodocker - -from tools.base import checker -from tools.distribution import distrotest -from tools.docker import utils as docker_utils - - -# # DistroTestConfig - -@pytest.mark.parametrize("config_path", [None, "CONFIG_PATH"]) -def test_config_constructor(config_path): - args = ("DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - if config_path is not None: - args += (config_path, ) - config = distrotest.DistroTestConfig(*args) - - for k in args: - if k in ["KEYFILE", "TESTFILE", "CONFIG_PATH"]: - assert getattr(config, f"_{k.lower()}") == k - else: - assert getattr(config, k.lower()) == k - - assert config._config_path == config_path - - -def test_config_dunder_getitem(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "KEYFILE", "PATH", "TARBALL", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - ("DistroTestConfig.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_config, ): - assert config.__getitem__("X") == m_config.return_value.__getitem__.return_value - - assert ( - list(m_config.return_value.__getitem__.call_args) - == [('X',), {}]) - - -# props - -@pytest.mark.parametrize("isdict", [True, False]) -def test_config_config(patches, isdict): - config = distrotest.DistroTestConfig( - "DOCKER", "KEYFILE", "PATH", "TARBALL", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "utils", - ("DistroTestConfig.config_path", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_utils, m_path): - if isdict: - m_utils.from_yaml.return_value = {} - - if isdict: - assert config.config == m_utils.from_yaml.return_value - else: - with pytest.raises(distrotest.ConfigurationError) as e: - config.config - - assert ( - list(m_utils.from_yaml.call_args) - == [(m_path.return_value,), {}]) - if isdict: - assert "config" in config.__dict__ - else: - assert ( - e.value.args[0] - == f"Unable to parse configuration: {m_path.return_value}") - - -@pytest.mark.parametrize("config_path", [None, "CONFIG_PATH"]) -def test_config_config_path(config_path): - args = ("DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - if config_path is not None: - args += (config_path, ) - config = distrotest.DistroTestConfig(*args) - assert config.config_path == config_path or distrotest.DISTROTEST_CONFIG_PATH - assert "config_path" in config.__dict__ - - -def test_config_ctx_dockerfile(): - path = MagicMock() - config = distrotest.DistroTestConfig( - "DOCKER", path, "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - - assert config.ctx_dockerfile == path.joinpath.return_value - assert ( - list(path.joinpath.call_args) - == [('Dockerfile', ), {}]) - assert "ctx_dockerfile" in config.__dict__ - - -def test_config_ctx_keyfile(patches): - path = MagicMock() - keyfile = MagicMock() - config = distrotest.DistroTestConfig( - "DOCKER", path, "TARBALL", keyfile, "TESTFILE", "MAINTAINER", "VERSION") - - assert config.ctx_keyfile == path.joinpath.return_value - assert ( - list(path.joinpath.call_args) - == [(keyfile.name, ), {}]) - assert "ctx_keyfile" in config.__dict__ - - -def test_config_rel_ctx_packages(): - path = MagicMock() - config = distrotest.DistroTestConfig( - "DOCKER", path, "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - - assert config.rel_ctx_packages == path.joinpath.return_value - assert ( - list(path.joinpath.call_args) - == [(config.packages_name, ), {}]) - assert "rel_ctx_packages" in config.__dict__ - - -def test_config_ctx_testfile(): - path = MagicMock() - testfile = MagicMock() - config = distrotest.DistroTestConfig( - "DOCKER", path, "TARBALL", "KEYFILE", testfile, "MAINTAINER", "VERSION") - - assert config.ctx_testfile == path.joinpath.return_value - - assert ( - list(path.joinpath.call_args) - == [(testfile.name, ), {}]) - assert "ctx_testfile" in config.__dict__ - - -@pytest.mark.parametrize( - "items", - [{}, - {f"TYPE{i}": dict(ext="EXT", images=[f"IMAGE{i}{x}" for x in ["a", "b", "c"]]) for i in range(0, 5)}]) -def test_config_images(patches, items): - config = distrotest.DistroTestConfig( - "DOCKER", "KEYFILE", "PATH", "TARBALL", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "DistroTestConfig.items", - prefix="tools.distribution.distrotest") - - with patched as (m_items, ): - m_items.return_value = items.items() - result = config.images - - expected = {} - - for k, v in items.items(): - for image in v["images"]: - expected[image] = dict(type=k, ext=v["ext"]) - assert result == expected - assert "images" in config.__dict__ - - -def test_config_install_img_path(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "INSTALL", "INSTALL", "MAINTAINER", "VERSION") - patched = patches( - "pathlib", - prefix="tools.distribution.distrotest") - - with patched as (m_plib, ): - assert config.install_img_path == m_plib.PurePosixPath.return_value - - assert ( - list(m_plib.PurePosixPath.call_args) - == [("/tmp/install",), {}]) - assert "install_img_path" in config.__dict__ - - -def test_config_keyfile(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "shutil", - ("DistroTestConfig.ctx_keyfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_shutil, m_key): - assert config.keyfile == m_key.return_value - - assert ( - list(m_shutil.copyfile.call_args) - == [("KEYFILE", m_key.return_value), {}]) - assert "keyfile" in config.__dict__ - - -def test_config_keyfile_img_path(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "KEYFILE", "MAINTAINER", "VERSION") - patched = patches( - "pathlib", - prefix="tools.distribution.distrotest") - - with patched as (m_plib, ): - assert config.keyfile_img_path == m_plib.PurePosixPath.return_value - - assert ( - list(m_plib.PurePosixPath.call_args) - == [("/tmp/gpg/signing.key",), {}]) - assert "keyfile_img_path" in config.__dict__ - - -def test_config_packages_dir(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "utils", - ("DistroTestConfig.rel_ctx_packages", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_utils, m_packages): - assert config.packages_dir == m_packages.return_value - - assert ( - list(m_utils.extract.call_args) - == [(m_packages.return_value, "TARBALL"), {}]) - assert "packages_dir" in config.__dict__ - - -def test_config_testfile(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "TESTFILE", "PATH", "TARBALL", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "shutil", - ("DistroTestConfig.ctx_testfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_shutil, m_key): - assert config.testfile == m_key.return_value - - assert ( - list(m_shutil.copyfile.call_args) - == [("TESTFILE", m_key.return_value), {}]) - assert "testfile" in config.__dict__ - - -def test_config_testfile_img_path(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "pathlib", - ("DistroTestConfig.testfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_plib, m_name): - assert config.testfile_img_path == m_plib.PurePosixPath.return_value.joinpath.return_value - - assert ( - list(m_plib.PurePosixPath.call_args) - == [("/tmp",), {}]) - assert ( - list(m_plib.PurePosixPath.return_value.joinpath.call_args) - == [(m_name.return_value.name,), {}]) - assert "testfile_img_path" in config.__dict__ - - -# methods - -def test_config_get_config(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "DistroTestConfig.get_image_name", - ("DistroTestConfig.images", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_name, m_images): - assert config.get_config("IMAGE") == m_images.return_value.__getitem__.return_value - - assert ( - list(m_images.return_value.__getitem__.call_args) - == [(m_name.return_value, ), {}]) - assert ( - list(m_name.call_args) - == [("IMAGE", ), {}]) - - -def test_config_get_image_name(): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - image = MagicMock() - assert config.get_image_name(image) == image.split.return_value.__getitem__.return_value - assert ( - list(image.split.call_args) - == [(":", ), {}]) - assert ( - list(image.split.return_value.__getitem__.call_args) - == [(0, ), {}]) - - -@pytest.mark.parametrize("pkg_type", ["TYPE1", "TYPE2", "TYPE3"]) -@pytest.mark.parametrize("pkg_types", [[], ["TYPE1", "TYPE2", "TYPE3"], ["TYPE3", "TYPE4"]]) -def test_config_get_package_type(patches, pkg_type, pkg_types): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - "DistroTestConfig.get_image_name", - "DistroTestConfig.items", - prefix="tools.distribution.distrotest") - - with patched as (m_name, m_items): - m_name.return_value = pkg_type - m_items.return_value = [("X", dict(images=["TYPE6", "TYPE7"])), ("Y", dict(images=pkg_types))] - if pkg_type in pkg_types: - assert config.get_package_type("IMAGE") == "Y" - else: - with pytest.raises(distrotest.ConfigurationError) as e: - config.get_package_type("IMAGE") - - assert e.value.args[0] == f"Unrecognized image: {pkg_type}" - - assert ( - list(m_name.call_args) - == [("IMAGE", ), {}]) - assert ( - list(m_items.call_args) - == [(), {}]) - - -@pytest.mark.parametrize("pkg_type", ["TYPE1", "TYPE2"]) -@pytest.mark.parametrize("ext", ["TYPE1", "TYPE2", "TYPE3", "TYPE4"]) -@pytest.mark.parametrize("packages", [[], ["PACKAGE1", "PACKAGE2"]]) -def test_config_get_packages(patches, pkg_type, ext, packages): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - ("DistroTestConfig.packages_dir", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_pkg, ): - m_pkg.return_value.joinpath.return_value.glob.return_value = packages - assert config.get_packages(pkg_type, ext) == packages - - assert ( - list(m_pkg.return_value.joinpath.call_args) - == [(pkg_type,), {}]) - assert ( - list(m_pkg.return_value.joinpath.return_value.glob.call_args) - == [(f'*.{ext}',), {}]) - - -def test_config_items(patches): - config = distrotest.DistroTestConfig( - "DOCKER", "PATH", "TARBALL", "KEYFILE", "TESTFILE", "MAINTAINER", "VERSION") - patched = patches( - ("DistroTestConfig.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_config, ): - assert config.items() == m_config.return_value.items.return_value - - assert ( - list(m_config.return_value.items.call_args) - == [(), {}]) - - -# # DistroTestImage - -@pytest.mark.parametrize("stream", [None, "STREAM"]) -def test_image_constructor(patches, stream): - args = ("CONFIG", "BUILD_IMAGE", "NAME") - if stream is not None: - args += (stream, ) - image = distrotest.DistroTestImage(*args) - assert image.test_config == "CONFIG" - assert image.build_image == "BUILD_IMAGE" - assert image.name == "NAME" - assert image._stream == stream - - assert image.prefix == distrotest.DOCKER_IMAGE_PREFIX - assert "prefix" not in image.__dict__ - assert image.dockerfile_template == distrotest.DOCKERFILE_TEMPLATE - assert "dockerfile_template" not in image.__dict__ - - -# props - -def _check_image_config_property(patches, prop, arg=None): - config = MagicMock() - image = distrotest.DistroTestImage(config, "BUILD_IMAGE", "NAME") - assert getattr(image, prop) == getattr(config, arg or prop) - assert prop not in image.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("ctx_dockerfile",), - ("docker",), - ("install_img_path",), - ("keyfile_img_path", ), - ("keyfile", ), - ("path", ), - ("packages_name", ), - ("testfile", ), - ("testfile_img_path", )]) -def test_image_config_props(patches, prop): - _check_image_config_property(patches, *prop) - - -def test_image_build_command(patches): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME") - patched = patches( - ("DistroTestImage.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_config, ): - assert ( - image.build_command - == m_config.return_value.__getitem__.return_value.__getitem__.return_value.strip.return_value.replace.return_value) - - assert ( - list(m_config.return_value.__getitem__.call_args) - == [('build',), {}]) - assert ( - list(m_config.return_value.__getitem__.return_value.__getitem__.call_args) - == [('command',), {}]) - assert ( - list(m_config.return_value.__getitem__.return_value.__getitem__.return_value.strip.call_args) - == [(), {}]) - assert ( - list(m_config.return_value.__getitem__.return_value.__getitem__.return_value.strip.return_value.replace.call_args) - == [("\n", " && "), {}]) - assert "build_command" not in image.__dict__ - - -def test_image_config(patches): - config = MagicMock() - image = distrotest.DistroTestImage(config, "BUILD_IMAGE", "NAME") - patched = patches( - ("DistroTestImage.package_type", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_type, ): - assert image.config == config.__getitem__.return_value - - assert ( - list(config.__getitem__.call_args) - == [(m_type.return_value,), {}]) - assert "config" in image.__dict__ - - -def test_image_ctx_install_dir(patches): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME") - patched = patches( - "pathlib", - ("DistroTestImage.package_type", dict(new_callable=PropertyMock)), - ("DistroTestImage.packages_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_plib, m_type, m_name): - assert image.ctx_install_dir == m_plib.Path.return_value.joinpath.return_value - - assert ( - list(m_plib.Path.call_args) - == [(m_name.return_value, ), {}]) - assert ( - list(m_plib.Path.return_value.joinpath.call_args) - == [(m_type.return_value, ), {}]) - assert "ctx_install_dir" not in image.__dict__ - - -def test_image_dockerfile(patches): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME") - patched = patches( - ("DistroTestImage.build_command", dict(new_callable=PropertyMock)), - ("DistroTestImage.env", dict(new_callable=PropertyMock)), - ("DistroTestImage.dockerfile_template", dict(new_callable=PropertyMock)), - ("DistroTestImage.ctx_install_dir", dict(new_callable=PropertyMock)), - ("DistroTestImage.keyfile", dict(new_callable=PropertyMock)), - ("DistroTestImage.keyfile_img_path", dict(new_callable=PropertyMock)), - ("DistroTestImage.install_img_path", dict(new_callable=PropertyMock)), - ("DistroTestImage.testfile", dict(new_callable=PropertyMock)), - ("DistroTestImage.testfile_img_path", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as patchy: - m_command, m_env, m_template, m_install, m_kfile, m_kpath, m_minstall, m_tfile, m_tpath = patchy - assert image.dockerfile == m_template.return_value.format.return_value - - assert ( - list(m_template.return_value.format.call_args) - == [(), - {'build_image': 'BUILD_IMAGE', - 'install_dir': m_install.return_value, - 'env': m_env.return_value, - 'install_mount_path': m_minstall.return_value, - 'testfile_name': m_tfile.return_value.name, - 'test_mount_path': m_tpath.return_value, - 'build_command': m_command.return_value, - 'keyfile_name': m_kfile.return_value.name, - 'key_mount_path': m_kpath.return_value}]) - - -@pytest.mark.parametrize("env", [None, "SOMENV=OTHER"]) -def test_image_env(patches, env): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME") - patched = patches( - ("DistroTestImage.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_config, ): - m_config.return_value.__getitem__.return_value.get.return_value = env - if env: - assert image.env == f"ENV {env}" - else: - assert image.env == "" - - assert ( - list(m_config.return_value.__getitem__.call_args) - == [("build", ), {}]) - assert ( - list(m_config.return_value.__getitem__.return_value.get.call_args) - == [("env", ""), {}]) - assert "env" not in image.__dict__ - - -def test_image_keyfile_package_type(): - config = MagicMock() - image = distrotest.DistroTestImage(config, "BUILD_IMAGE", "NAME") - assert image.package_type == config.get_package_type.return_value - assert ( - list(config.get_package_type.call_args) - == [("BUILD_IMAGE", ), {}]) - assert "package_type" in image.__dict__ - - -def test_image_tag(patches): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME") - - patched = patches( - ("DistroTestImage.prefix", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_prefix, ): - assert image.tag == f"{m_prefix.return_value}NAME:latest" - - assert "tag" in image.__dict__ - - -# methods - -def test_image_add_dockerfile(patches): - stream = MagicMock() - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", stream=stream) - patched = patches( - "shutil", - ("DistroTestImage.dockerfile", dict(new_callable=PropertyMock)), - ("DistroTestImage.ctx_dockerfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_shutil, m_dfile, m_ctx_docker): - assert not image.add_dockerfile() - assert ( - list(stream.call_args) - == [(m_dfile.return_value,), {}]) - assert ( - list(m_ctx_docker.return_value.write_text.call_args) - == [(m_dfile.return_value,), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [True, False]) -async def test_image_build(patches, raises): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - "docker_utils.build_image", - "DistroTestImage.add_dockerfile", - "DistroTestImage.stream", - ("DistroTestImage.docker", dict(new_callable=PropertyMock)), - ("DistroTestImage.path", dict(new_callable=PropertyMock)), - ("DistroTestImage.tag", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_build, m_add, m_stream, m_docker, m_path, m_tag): - if raises: - m_build.side_effect = docker_utils.BuildError("AN ERROR OCCURRED") - - with pytest.raises(distrotest.BuildError) as e: - await image.build() - - assert ( - e.value.args - == ('AN ERROR OCCURRED',)) - else: - assert not await image.build() - - assert ( - list(m_add.call_args) - == [(), {}]) - assert ( - list(m_build.call_args) - == [(m_docker.return_value, - m_path.return_value, - m_tag.return_value), - {'stream': m_stream, 'forcerm': True}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tag", ["TAG1", "TAG2"]) -@pytest.mark.parametrize( - "images", - [[], - ["TAG1"], - ["TAG1", "TAG2"], - ["TAG3", "TAG4"]]) -async def test_image_exists(patches, tag, images): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - ("DistroTestImage.images", dict(new_callable=AsyncMock)), - ("DistroTestImage.tag", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_images, m_tag): - m_images.return_value = images - m_tag.return_value = tag - if tag in images: - assert await image.exists() is True - else: - assert await image.exists() is False - - -@pytest.mark.parametrize("items", range(0, 5)) -def test_image_get_environment(patches, items): - config = MagicMock() - image = distrotest.DistroTestImage(config, "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - "dict", - "DistroTestImage.get_install_binary", - "DistroTestImage.installable_img_path", - ("DistroTestImage.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - class MockDict(collections.UserDict): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - type(self).__setitem__ = MagicMock() - - with patched as (m_dict, m_get, m_path, m_config): - _items = [[MagicMock(), MagicMock()]] * items - _dict = MockDict((("A", "B"), )) - m_dict.return_value = _dict - m_config.return_value.__getitem__.return_value.items.return_value = _items - assert image.get_environment("PKG_FNAME", "PKG_NAME", "DISTRO_NAME") == _dict - - assert ( - list(m_dict.call_args) - == [(), - {'ENVOY_MAINTAINER': config.maintainer, - 'ENVOY_VERSION': config.version, - 'ENVOY_INSTALL_BINARY': m_path.return_value, - 'ENVOY_INSTALLABLE': m_path.return_value, - 'PACKAGE': 'PKG_NAME', - 'DISTRO': 'DISTRO_NAME'}]) - assert ( - list(list(c) for c in m_path.call_args_list) - == [[(m_get.return_value,), {}], - [('PKG_FNAME',), {}]]) - assert ( - list(m_get.call_args) - == [('PKG_FNAME',), {}]) - assert ( - list(m_config.return_value.__getitem__.call_args) - == [('test',), {}]) - assert ( - list(m_config.return_value.__getitem__.return_value.items.call_args) - == [(), {}]) - assert ( - list(list(c) for c in _dict.__setitem__.call_args_list) - == [[(m_k.upper.return_value, m_v.format.return_value), {}] for m_k, m_v in _items]) - - for m_k, m_v in _items: - assert ( - list(m_k.upper.call_args) - == [(), {}]) - assert ( - list(m_v.format.call_args) - == [(), {'A': 'B'}]) - - -@pytest.mark.parametrize("contains", [True, False]) -def test_image_get_install_binary(patches, contains): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - "re", - ("DistroTestImage.config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - - with patched as (m_re, m_config): - m_config.return_value.__contains__.return_value = contains - assert ( - image.get_install_binary("PACKAGE") - == (m_re.sub.return_value - if contains - else "PACKAGE")) - - if not contains: - assert not m_re.sub.called - assert not m_config.return_value.__getitem__.called - return - - assert ( - list(list(c) for c in m_config.return_value.__getitem__.call_args_list) - == [[('binary_name',), {}], [('binary_name',), {}]]) - assert ( - list(list(c) for c in m_config.return_value.__getitem__.return_value.__getitem__.call_args_list) - == [[('match',), {}], [('replace',), {}]]) - assert ( - list(m_re.sub.call_args) - == [(m_config.return_value.__getitem__.return_value.__getitem__.return_value, - m_config.return_value.__getitem__.return_value.__getitem__.return_value, - 'PACKAGE'), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "images", - [[], - [dict(RepoTags=["TAGA"])], - [dict(RepoTags=[f"TAG{i}A", f"TAG{i}B"]) for i in range(1, 4)]]) -async def test_image_images(patches, images): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - "chain", - ("DistroTestImage.docker", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_chain, m_docker): - m_docker.return_value.images.list = AsyncMock(return_value=images) - assert await image.images() == m_chain.from_iterable.return_value - - expected = [image["RepoTags"] for image in images] - assert ( - list(m_chain.from_iterable.call_args) - == [(expected,), {}]) - - -def test_image_installable_img_path(patches): - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", "STREAM") - patched = patches( - ("DistroTestImage.install_img_path", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_dir, ): - assert image.installable_img_path("INSTALLABLE") == m_dir.return_value.joinpath.return_value - - assert ( - list(m_dir.return_value.joinpath.call_args) - == [("INSTALLABLE",), {}]) - - -@pytest.mark.parametrize("stream", [True, None]) -def test_image_stream(patches, stream): - if stream: - stream = MagicMock() - image = distrotest.DistroTestImage("CONFIG", "BUILD_IMAGE", "NAME", stream) - assert not image.stream("MESSAGE") - if stream: - assert ( - list(stream.call_args) - == [("MESSAGE", ), {}]) - - -# # DistroTest - -def test_distrotest_constructor(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - assert dtest.checker == check - assert dtest.test_config == "CONFIG" - assert dtest.installable == "INSTALLABLE" - assert dtest.distro == "NAME" - assert dtest.build_image == "IMAGE" - - assert dtest.failures == [] - dtest._failures = ["FAIL"] - assert dtest.failures == ["FAIL"] - assert "failures" not in dtest.__dict__ - assert dtest.prefix == distrotest.DOCKER_CONTAINER_PREFIX - assert "prefix" not in dtest.__dict__ - assert dtest.image_class == distrotest.DistroTestImage - assert "image_class" not in dtest.__dict__ - - -# props - -def _check_distrotest_checker_property(prop, arg=None): - check = AsyncMock() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - assert getattr(dtest, prop) == getattr(check, arg or prop) - assert prop not in dtest.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("errors",), - ("exiting",), - ("log",), - ("stdout",)]) -def test_distrotest_checker_props(prop): - _check_distrotest_checker_property(*prop) - - -def _check_distrotest_config_property(patches, prop, arg=None): - check = AsyncMock() - config = MagicMock() - dtest = distrotest.DistroTest(check, config, "NAME", "IMAGE", "INSTALLABLE") - assert getattr(dtest, prop) == getattr(config, arg or prop) - assert prop not in dtest.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("docker",), - ("testfile", )]) -def test_distrotest_config_props(patches, prop): - _check_distrotest_config_property(patches, *prop) - - -def test_distrotest_config(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.image", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_image, ): - assert dtest.config == dict(Image=m_image.return_value.tag) - - -def test_distrotest_environment(patches): - check = checker.AsyncChecker() - installable = MagicMock() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", installable) - patched = patches( - ("DistroTest.image", dict(new_callable=PropertyMock)), - ("DistroTest.package_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_image, m_name): - assert dtest.environment == m_image.return_value.get_environment.return_value - - assert ( - list(m_image.return_value.get_environment.call_args) - == [(installable.name, m_name.return_value, 'NAME'), {}]) - - -@pytest.mark.parametrize("failures", [[], ["FAIL1", "FAIL2"]]) -def test_distrotest_failed(patches, failures): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.failures", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_failures, ): - m_failures.return_value = failures - assert dtest.failed == (len(failures) > 0) - - -def test_distrotest_image(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.image_class", - ("DistroTest.docker", dict(new_callable=PropertyMock)), - ("DistroTest.stdout", dict(new_callable=PropertyMock)), - ("DistroTest.testfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_class, m_docker, m_stdout, m_test): - assert dtest.image == m_class.return_value - - assert ( - list(m_class.call_args) - == [('CONFIG', - 'IMAGE', - 'NAME'), - {'stream': m_stdout.return_value.info}]) - - -def test_distrotest_name(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.prefix", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_prefix, ): - assert dtest.name == f"{m_prefix.return_value}NAME" - - assert "name" in dtest.__dict__ - - -def test_distrotest_package_name(patches): - check = checker.AsyncChecker() - installable = MagicMock() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", installable) - assert dtest.package_name == installable.name.split.return_value.__getitem__.return_value - assert ( - list(installable.name.split.call_args) - == [('_',), {}]) - assert ( - list(installable.name.split.return_value.__getitem__.call_args) - == [(0,), {}]) - - -def test_distrotest_test_cmd(patches): - check = checker.AsyncChecker() - config = MagicMock() - dtest = distrotest.DistroTest(check, config, "NAME", "IMAGE", "INSTALLABLE") - assert dtest.test_cmd == (str(config.testfile_img_path), ) - - -# methods - -@pytest.mark.asyncio -@pytest.mark.parametrize("exists", [True, False]) -async def test_distrotest_build(patches, exists): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.run_log", - ("DistroTest.image", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_run, m_image): - m_image.return_value.exists = AsyncMock(return_value=exists) - m_image.return_value.build = AsyncMock() - assert not await dtest.build() - - assert ( - list(m_image.return_value.exists.call_args) - == [(), {}]) - - if exists: - assert not m_image.return_value.build.called - assert not m_run.called - return - - assert ( - list(m_image.return_value.build.call_args) - == [(), {}]) - assert ( - list(list(c) for c in m_run.call_args_list) - == [[('Building image',), {'msg_type': 'notice'}], - [('Image built',), {}]]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [True, False]) -async def test_distrotest_cleanup(patches, raises): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.docker", dict(new_callable=AsyncMock)), - ("DistroTest.stop", dict(new_callable=AsyncMock)), - ("DistroTest.name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - class SomeError(Exception): - pass - - with patched as (m_docker, m_stop, m_name): - if raises: - m_stop.side_effect = SomeError("AN ERROR OCCURRED") - assert not await dtest.cleanup() - - assert ( - list(m_docker.containers.get.call_args) - == [(m_name.return_value,), {}]) - assert ( - list(m_stop.call_args) - == [(m_docker.containers.get.return_value,), {}]) - - -@pytest.mark.asyncio -async def test_distrotest_create(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.docker", dict(new_callable=AsyncMock)), - ("DistroTest.config", dict(new_callable=PropertyMock)), - ("DistroTest.name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_docker, m_config, m_name): - assert await dtest.create() == m_docker.containers.create_or_replace.return_value - - assert ( - list(m_docker.containers.create_or_replace.call_args) - == [(), - {'config': m_config.return_value, - 'name': m_name.return_value}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failed", [True, False]) -@pytest.mark.parametrize("returns", [0, 1]) -@pytest.mark.parametrize("msgs", range(0, 5)) -async def test_distrotest_exec(patches, failed, returns, msgs): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.error", - "DistroTest.handle_test_output", - ("DistroTest.environment", dict(new_callable=PropertyMock)), - ("DistroTest.failed", dict(new_callable=PropertyMock)), - ("DistroTest.test_cmd", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - container = AsyncMock() - - class Tracker(object): - counter = 0 - _outs = [] - - async def _out(self): - self.counter += 1 - _mock = MagicMock() - if msgs >= self.counter: - self._outs.append(_mock) - return _mock - return "" - - @contextlib.asynccontextmanager - async def _start(self, *args, **kwargs): - self.stream = AsyncMock() - await self.stream(*args, **kwargs) - self.stream.read_out.side_effect = self._out - yield self.stream - - _tracker = Tracker() - container.exec.return_value.start = _tracker._start - container.exec.return_value.inspect.return_value = dict(ExitCode=returns) - - with patched as (m_error, m_out, m_env, m_failed, m_cmd): - m_failed.return_value = failed - assert not await dtest.exec(container) - - assert ( - list(container.exec.call_args) - == [(m_cmd.return_value,), - {'environment': m_env.return_value}]) - assert ( - list(_tracker.stream.call_args) - == [(), {'detach': False}]) - assert ( - list(container.exec.return_value.inspect.call_args) - == [(), {}]) - - assert _tracker.counter == msgs + 1 - assert len(_tracker._outs) == msgs - - for _out in _tracker._outs: - assert ( - list(_out.data.decode.call_args) - == [('utf-8',), {}]) - assert ( - list(_out.data.decode.return_value.strip.call_args) - == [(), {}]) - - _log_error = (msgs > 0) and (returns and not failed) - - if _log_error: - assert dtest._failures == ['container-start'] - assert ( - list(m_error.call_args) - == [([f"[NAME] Error executing test in container\n{_tracker._outs[-1].data.decode.return_value.strip.return_value}"],), {}]) - assert ( - list(list(c) for c in m_out.call_args_list) - == [[(_out.data.decode.return_value.strip.return_value,), {}] for _out in _tracker._outs[:-1]]) - else: - assert dtest._failures == [] - assert not m_error.called - assert ( - list(list(c) for c in m_out.call_args_list) - == [[(_out.data.decode.return_value.strip.return_value,), {}] for _out in _tracker._outs]) - - -def test_distrotest_error(): - check = MagicMock() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - assert dtest.error(["ERR1", "ERR2"]) == check.error.return_value - assert ( - list(check.error.call_args) - == [(check.active_check, ['ERR1', 'ERR2']), {}]) - - -@pytest.mark.parametrize("msg", ["MESSAGE", "MESSAGE\nEXTRA", "MESSAGE\nEXTRA\nMORE"]) -def test_distrotest_handle_test_error(patches, msg): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.error", - ("DistroTest.stdout", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - _msg = MagicMock() - _splitter = MagicMock() - - def _split(splitter, *args): - if splitter == "\n": - return msg.split("\n", *args) - return _splitter - - with patched as (m_error, m_stdout): - _msg.split.side_effect = _split - _splitter.__getitem__.return_value.strip.return_value.split.return_value = ( - "TESTRUN", "TESTNAME") - assert not dtest.handle_test_error(_msg) - - assert ( - list(list(c) for c in _msg.split.call_args_list) - == [[(']',), {}], [('\n', 1), {}]]) - assert ( - list(_splitter.__getitem__.call_args) - == [(0,), {}]) - assert ( - list(_splitter.__getitem__.return_value.strip.call_args) - == [('[',), {}]) - assert ( - list(_splitter.__getitem__.return_value.strip.return_value.split.call_args) - == [(':',), {}]) - - assert dtest._failures == ['TESTNAME'] - assert ( - list(m_error.call_args) - == [(['[TESTRUN:TESTNAME] Test failed'],), {}]) - - if len(msg.split("\n")) > 1: - assert ( - list(m_stdout.return_value.error.call_args) - == [(msg.split("\n", 1)[1],), {}]) - return - - assert not m_stdout.called - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "start", - ["NAME", "[NAME", "[NOME", "x[NAME", "[NAME]", "ERROR"]) -@pytest.mark.parametrize( - "msg", - ["", - "foo", - " bar", - "NAME", - "fooERROR", - "ERRORfoo", - "ERROR foo", - " fooERROR", - " ERRORfoo", - " ERROR foo", - "\nERROR foo", - " ERROR\nfoo", - "OTHER\nfoo"]) -async def test_distrotest_handle_test_output(patches, start, msg): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.handle_test_error", - ("DistroTest.stdout", dict(new_callable=PropertyMock)), - ("DistroTest.log", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - _msg = f"{start}{msg}" - - with patched as (m_error, m_stdout, m_log): - assert not dtest.handle_test_output(_msg) - - if not _msg.startswith("[NAME") and "\n" in _msg: - _parts = _msg.split("\n", 1) - assert ( - list(m_stdout.return_value.info.call_args_list[0]) - == [(_parts[0],), {}]) - _msg = _parts[1] - - if not start.startswith("[NAME"): - assert ( - list(m_stdout.return_value.info.call_args) - == [(_msg,), {}]) - assert not m_error.called - assert not m_log.called - return - - assert not m_stdout.called - - if "ERROR" not in msg: - assert ( - list(m_log.return_value.info.call_args) - == [(_msg,), {}]) - assert not m_error.called - return - - assert not m_log.called - assert ( - list(m_error.call_args) - == [(_msg,), {}]) - - -@pytest.mark.parametrize("failed", [True, False]) -@pytest.mark.parametrize("failures", [[], ["FAIL1"], ["FAIL1", "FAIL2"]]) -def test_distrotest_log_failures(patches, failed, failures): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.run_log", - ("DistroTest.failed", dict(new_callable=PropertyMock)), - ("DistroTest.failures", dict(new_callable=PropertyMock)), - ("DistroTest.package_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_log, m_failed, m_failures, m_name): - m_failed.return_value = failed - m_failures.return_value = failures - assert not dtest.log_failures() - - if not failed: - assert not m_log.called - return - - assert ( - list(m_log.call_args) - == [(f'Package test had failures: {",".join(failures)}',), - {'msg_type': 'error', 'test': m_name.return_value}]) - - -@pytest.mark.asyncio -async def test_distrotest_logs(): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - container = AsyncMock() - _logs = ["LOG1", "LOG2", "LOG3"] - container.log.return_value = _logs - assert await dtest.logs(container) == "".join(_logs) - assert ( - list(container.log.call_args) - == [(), {'stdout': True, 'stderr': True}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failed", [True, False]) -@pytest.mark.parametrize("self_failed", [True, False]) -async def test_distrotest_on_test_complete(patches, failed, self_failed): - check = MagicMock() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.log_failures", - "DistroTest.run_message", - ("DistroTest.stop", dict(new_callable=AsyncMock)), - ("DistroTest.failed", dict(new_callable=PropertyMock)), - ("DistroTest.package_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_log, m_msg, m_stop, m_failed, m_name): - m_failed.return_value = self_failed - assert not await dtest.on_test_complete("CONTAINER", failed) - - assert ( - list(m_log.call_args) - == [(), {}]) - assert ( - list(m_stop.call_args) - == [('CONTAINER',), {}]) - - if failed or self_failed: - assert not check.succeed.called - assert not m_msg.called - return - - assert ( - list(m_msg.call_args) - == [('Package test passed',), - {'test': m_name.return_value}]) - assert ( - list(check.succeed.call_args) - == [(check.active_check, [m_msg.return_value]), {}]) - - -@pytest.mark.asyncio -async def test_distrotest_run(patches): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.error", - ("DistroTest._run", dict(new_callable=AsyncMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_error, m_run): - assert not await dtest.run() - - assert ( - list(m_run.call_args) - == [(), {}]) - assert ( - list(m_error.call_args) - == [(m_run.return_value,), {}]) - - -@pytest.mark.parametrize("msg_type", [None, "MSG_TYPE"]) -@pytest.mark.parametrize("testname", [None, "TEST"]) -def test_distrotest_run_log(patches, msg_type, testname): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "getattr", - "DistroTest.run_message", - ("DistroTest.log", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - args = ["MESSAGE"] - if msg_type: - args.append(msg_type) - kwargs = {} - if testname: - kwargs["test"] = testname - - with patched as (m_get, m_msg, m_log): - assert not dtest.run_log(*args, **kwargs) - - assert ( - list(m_get.call_args) - == [(m_log.return_value, msg_type or 'info'), {}]) - assert ( - list(m_get.return_value.call_args) - == [(m_msg.return_value,), {}]) - assert ( - list(m_msg.call_args) - == [('MESSAGE',), {'test': testname}]) - - -@pytest.mark.parametrize("testname", [None, "TEST"]) -def test_distrotest_run_message(patches, testname): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - if testname: - assert dtest.run_message("MESSAGE", testname) == f"[NAME/{testname}] MESSAGE" - else: - assert dtest.run_message("MESSAGE") == f"[NAME] MESSAGE" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("running", [True, False]) -async def test_distrotest_start(patches, running): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.run_log", - "DistroTest.run_message", - ("DistroTest.create", dict(new_callable=AsyncMock)), - ("DistroTest.logs", dict(new_callable=AsyncMock)), - ("DistroTest.package_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_log, m_msg, m_create, m_logs, m_name): - m_create.return_value.show.return_value.__getitem__.return_value.__getitem__.return_value = running - - if running: - assert await dtest.start() == m_create.return_value - else: - with pytest.raises(distrotest.ContainerError) as e: - await dtest.start() - - assert e.value.args[0] == m_msg.return_value - - assert ( - list(m_create.call_args) - == [(), {}]) - assert ( - list(m_create.return_value.start.call_args) - == [(), {}]) - assert ( - list(m_create.return_value.show.call_args) - == [(), {}]) - - if not running: - assert not m_log.called - assert ( - list(m_msg.call_args) - == [(f"Container unable to start\n{m_logs.return_value}",), - {'test': m_name.return_value}]) - assert ( - list(m_logs.call_args) - == [(m_create.return_value,), {}]) - return - - assert ( - list(m_log.call_args) - == [('Container started',), - {'test': m_name.return_value}]) - assert not m_msg.called - assert not m_logs.called - - -@pytest.mark.asyncio -@pytest.mark.parametrize("container", [None, "CONTAINER"]) -async def test_distrotest_stop(patches, container): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - "DistroTest.run_log", - ("DistroTest.package_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.distrotest") - - if container: - container = AsyncMock() - - with patched as (m_log, m_pkg): - assert not await dtest.stop(container) - - if not container: - assert not m_log.called - return - - assert ( - list(container.kill.call_args) - == [(), {}]) - assert ( - list(container.delete.call_args) - == [(), {}]) - assert ( - list(m_log.call_args) - == [('Container stopped',), {'test': m_pkg.return_value}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("build_raises", [None, distrotest.ConfigurationError, distrotest.BuildError, aiodocker.exceptions.DockerError, Exception]) -@pytest.mark.parametrize("start_raises", [None, distrotest.ContainerError, aiodocker.exceptions.DockerError, Exception]) -@pytest.mark.parametrize("exec_raises", [None, aiodocker.exceptions.DockerError, Exception]) -@pytest.mark.parametrize("stop_raises", [None, aiodocker.exceptions.DockerError, Exception]) -async def test_distrotest__run(patches, build_raises, start_raises, exec_raises, stop_raises): - check = checker.AsyncChecker() - dtest = distrotest.DistroTest(check, "CONFIG", "NAME", "IMAGE", "INSTALLABLE") - patched = patches( - ("DistroTest.build", dict(new_callable=AsyncMock)), - ("DistroTest.start", dict(new_callable=AsyncMock)), - ("DistroTest.exec", dict(new_callable=AsyncMock)), - ("DistroTest.on_test_complete", dict(new_callable=AsyncMock)), - prefix="tools.distribution.distrotest") - - with patched as (m_build, m_start, m_exec, m_stop): - if build_raises: - if build_raises == aiodocker.exceptions.DockerError: - m_build.side_effect = build_raises("ARG1", dict(message="AN ERROR OCCURRED")) - else: - m_build.side_effect = build_raises("AN ERROR OCCURRED") - if start_raises: - if start_raises == aiodocker.exceptions.DockerError: - m_start.side_effect = start_raises("ARG1", dict(message="AN ERROR OCCURRED")) - else: - m_start.side_effect = start_raises("AN ERROR OCCURRED") - if exec_raises: - if exec_raises == aiodocker.exceptions.DockerError: - m_exec.side_effect = exec_raises("ARG1", dict(message="AN ERROR OCCURRED")) - else: - m_exec.side_effect = exec_raises("AN ERROR OCCURRED") - if stop_raises: - if stop_raises == aiodocker.exceptions.DockerError: - m_stop.side_effect = stop_raises("ARG1", dict(message="AN ERROR OCCURRED")) - else: - m_stop.side_effect = stop_raises("AN ERROR OCCURRED") - - should_fail = ( - build_raises == Exception - or not build_raises and start_raises == Exception - or not (build_raises or start_raises) and exec_raises == Exception - or stop_raises == Exception) - - if should_fail: - with pytest.raises(Exception): - await dtest._run() - else: - result = await dtest._run() - - assert ( - list(m_build.call_args) - == [(), {}]) - - if build_raises or start_raises: - assert ( - list(m_stop.call_args) - == [(None, True), {}]) - elif exec_raises: - assert ( - list(m_stop.call_args) - == [(m_start.return_value, True), {}]) - else: - assert ( - list(m_stop.call_args) - == [(m_start.return_value, False), {}]) - - if build_raises: - assert not m_start.called - assert not m_exec.called - if not should_fail: - assert result == ('AN ERROR OCCURRED',) - return - - assert ( - list(m_start.call_args) - == [(), {}]) - - if start_raises: - assert not m_exec.called - if not should_fail: - assert result == ('AN ERROR OCCURRED',) - return - - assert ( - list(m_exec.call_args) - == [(m_start.return_value,), {}]) - - if exec_raises or stop_raises: - if not should_fail: - assert result == ('AN ERROR OCCURRED',) - return - - assert not result diff --git a/tools/distribution/tests/test_sign.py b/tools/distribution/tests/test_sign.py deleted file mode 100644 index 78833b502555b..0000000000000 --- a/tools/distribution/tests/test_sign.py +++ /dev/null @@ -1,1009 +0,0 @@ -import types -from unittest.mock import MagicMock, PropertyMock - -import pytest - -from tools.base import runner -from tools.distribution import sign -from tools.gpg import identity - - -# DirectorySigningUtil - -@pytest.mark.parametrize("command", ["", None, "COMMAND", "OTHERCOMMAND"]) -def test_util_constructor(command): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - args = ("PATH", maintainer, "LOG") - if command is not None: - args += (command, ) - util = sign.DirectorySigningUtil(*args) - assert util._path == "PATH" - assert util.maintainer == maintainer - assert util.log == "LOG" - assert util._command == (command or "") - assert util.command_args == () - - -@pytest.mark.parametrize("command_name", ["", None, "CMD", "OTHERCMD"]) -@pytest.mark.parametrize("command", ["", None, "COMMAND", "OTHERCOMMAND"]) -@pytest.mark.parametrize("which", ["", None, "PATH", "OTHERPATH"]) -def test_util_command(patches, command_name, command, which): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG", command=command) - patched = patches( - "shutil", - ("DirectorySigningUtil.package_type", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - if command_name is not None: - util.command_name = command_name - - with patched as (m_shutil, m_type): - m_shutil.which.return_value = which - - if not which and not command: - with pytest.raises(sign.SigningError) as e: - util.command - - assert ( - list(m_shutil.which.call_args) - == [(command_name or "",), {}]) - assert ( - e.value.args[0] - == f"Signing software missing ({m_type.return_value}): {command_name or ''}") - return - - result = util.command - - assert "command" in util.__dict__ - assert not m_type.called - - if command: - assert not m_shutil.which.called - assert result == command - return - - assert ( - list(m_shutil.which.call_args) - == [(command_name or "",), {}]) - assert result == m_shutil.which.return_value - - -def test_util_sign(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - patched = patches( - "DirectorySigningUtil.sign_pkg", - ("DirectorySigningUtil.pkg_files", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_sign, m_pkgs): - m_pkgs.return_value = ("PKG1", "PKG2", "PKG3") - assert not util.sign() - - assert ( - list(list(c) for c in m_sign.call_args_list) - == [[('PKG1',), {}], - [('PKG2',), {}], - [('PKG3',), {}]]) - - -def test_util_sign_command(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - patched = patches( - ("DirectorySigningUtil.command", dict(new_callable=PropertyMock)), - ("DirectorySigningUtil.command_args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_command, m_args): - m_args.return_value = ("ARG1", "ARG2", "ARG3") - assert ( - util.sign_command("PACKAGE") - == (m_command.return_value, ) + m_args.return_value + ("PACKAGE", )) - - -@pytest.mark.parametrize("returncode", [0, 1]) -def test_util_sign_pkg(patches, returncode): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - util.log = MagicMock() - pkg_file = MagicMock() - patched = patches( - "subprocess", - "DirectorySigningUtil.sign_command", - ("PackageSigningRunner.log", dict(new_callable=PropertyMock)), - ("DirectorySigningUtil.package_type", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_subproc, m_command, m_log, m_type): - m_subproc.run.return_value.returncode = returncode - if returncode: - with pytest.raises(sign.SigningError) as e: - util.sign_pkg(pkg_file) - else: - assert not util.sign_pkg(pkg_file) - - assert ( - list(util.log.notice.call_args) - == [(f"Sign package ({m_type.return_value}): {pkg_file.name}",), {}]) - assert ( - list(m_command.call_args) - == [(pkg_file,), {}]) - assert ( - list(m_subproc.run.call_args) - == [(m_command.return_value,), - {'capture_output': True, - 'encoding': 'utf-8'}]) - - if not returncode: - assert ( - list(util.log.success.call_args) - == [(f"Signed package ({m_type.return_value}): {pkg_file.name}",), {}]) - return - assert e.value.args[0] == m_subproc.run.return_value.stdout + m_subproc.run.return_value.stderr - - -@pytest.mark.parametrize("ext", ["EXT1", "EXT2"]) -@pytest.mark.parametrize("package_type", [None, "", "TYPE1", "TYPE2"]) -def test_util_package_type(ext, package_type): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - util.ext = ext - util._package_type = package_type - assert util.package_type == package_type or ext - - -def test_util_path(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - patched = patches( - "pathlib", - prefix="tools.distribution.sign") - with patched as (m_plib, ): - assert util.path == m_plib.Path.return_value - - assert ( - list(m_plib.Path.call_args) - == [(util._path,), {}]) - - -@pytest.mark.parametrize( - "files", - [[], - ["abc", "xyz"], - ["abc.EXT", "xyz.EXT", "abc.FOO", "abc.BAR"], - ["abc.NOTEXT", "xyz.NOTEXT"]]) -def test_util_pkg_files(patches, files): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - util = sign.DirectorySigningUtil("PATH", maintainer, "LOG") - patched = patches( - ("DirectorySigningUtil.ext", dict(new_callable=PropertyMock)), - ("DirectorySigningUtil.path", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - with patched as (m_ext, m_path): - _glob = {} - - for _path in files: - _mock = MagicMock() - _mock.name = _path - _glob[_path] = _mock - m_path.return_value.glob.return_value = _glob.values() - - m_ext.return_value = "EXT" - result = util.pkg_files - - expected = [fname for fname in files if fname.endswith(".EXT")] - - assert ( - list(m_path.return_value.glob.call_args) - == [("*",), {}]) - assert "pkg_files" not in util.__dict__ - assert ( - result - == tuple(_glob[k] for k in expected)) - - -# PackageSigningRunner - -def test_packager_constructor(): - packager = sign.PackageSigningRunner("x", "y", "z") - assert isinstance(packager, runner.Runner) - assert packager.maintainer_class == identity.GPGIdentity - assert packager._signing_utils == () - - -def test_packager_cls_register_util(): - assert sign.PackageSigningRunner._signing_utils == () - - class Util1(object): - pass - - class Util2(object): - pass - - sign.PackageSigningRunner.register_util("util1", Util1) - assert ( - sign.PackageSigningRunner._signing_utils - == (('util1', Util1),)) - - sign.PackageSigningRunner.register_util("util2", Util2) - assert ( - sign.PackageSigningRunner._signing_utils - == (('util1', Util1), - ('util2', Util2),)) - - -def test_packager_extract(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_args, ): - assert packager.extract == m_args.return_value.extract - - assert "extract" not in packager.__dict__ - - -def test_packager_maintainer(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - ("PackageSigningRunner.log", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.maintainer_class", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.maintainer_email", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.maintainer_name", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_log, m_class, m_email, m_name): - assert packager.maintainer == m_class.return_value.return_value - - assert ( - list(m_class.return_value.call_args) - == [(m_name.return_value, m_email.return_value, m_log.return_value), {}]) - - assert "maintainer" in packager.__dict__ - - -def test_packager_maintainer_email(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_args, ): - assert packager.maintainer_email == m_args.return_value.maintainer_email - - assert "maintainer_email" not in packager.__dict__ - - -def test_packager_maintainer_name(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - - patched = patches( - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_args, ): - assert packager.maintainer_name == m_args.return_value.maintainer_name - - assert "maintainer_name" not in packager.__dict__ - - -def test_packager_package_type(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - - patched = patches( - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_args, ): - assert packager.package_type == m_args.return_value.package_type - - assert "package_type" not in packager.__dict__ - - -def test_packager_path(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "pathlib", - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_plib, m_args): - assert packager.path == m_plib.Path.return_value - - assert ( - list(m_plib.Path.call_args) - == [(m_args.return_value.path, ), {}]) - assert "path" not in packager.__dict__ - - -def test_packager_tar(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - ("PackageSigningRunner.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_args, ): - assert packager.tar == m_args.return_value.tar - - assert "tar" not in packager.__dict__ - - -def test_packager_signing_utils(): - packager = sign.PackageSigningRunner("x", "y", "z") - _utils = (("NAME1", "UTIL1"), ("NAME2", "UTIL2")) - packager._signing_utils = _utils - assert packager.signing_utils == dict(_utils) - - -def test_packager_add_arguments(): - packager = sign.PackageSigningRunner("x", "y", "z") - parser = MagicMock() - packager.add_arguments(parser) - assert ( - list(list(c) for c in parser.add_argument.call_args_list) - == [[('--log-level', '-l'), - {'choices': ['debug', 'info', 'warn', 'error'], - 'default': 'info', - 'help': 'Log level to display'}], - [('path',), - {'default': '', - 'help': 'Path to the directory containing packages to sign'}], - [('--extract',), - {'action': 'store_true', - 'help': 'If set, treat the path as a tarball containing directories ' - 'according to package_type'}], - [('--tar',), - {'help': 'Path to save the signed packages as tar file'}], - [('--type',), - {'choices': ['util1', 'util2', ''], - 'default': '', - 'help': 'Package type to sign'}], - [('--maintainer-name',), - {'default': '', 'help': 'Maintainer name to match when searching for a GPG key to match with'}], - [('--maintainer-email',), - {'default': '', - 'help': 'Maintainer email to match when searching for a GPG key to match with'}]]) - - -def test_packager_archive(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "tarfile", - ("PackageSigningRunner.tar", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_tarfile, m_tar): - assert not packager.archive("PATH") - - assert ( - list(m_tarfile.open.call_args) - == [(m_tar.return_value, 'w'), {}]) - assert ( - list(m_tarfile.open.return_value.__enter__.return_value.add.call_args) - == [('PATH',), {'arcname': '.'}]) - - -def test_packager_get_signing_util(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - ("PackageSigningRunner.log", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.maintainer", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.signing_utils", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - path = MagicMock() - - with patched as (m_log, m_maintainer, m_utils): - assert packager.get_signing_util(path) == m_utils.return_value.__getitem__.return_value.return_value - - assert ( - list(m_utils.return_value.__getitem__.call_args) - == [(path.name,), {}]) - assert ( - list(m_utils.return_value.__getitem__.return_value.call_args) - == [(path, m_maintainer.return_value, m_log.return_value), {}]) - - -@pytest.mark.parametrize("extract", [True, False]) -def test_packager_run(patches, extract): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "PackageSigningRunner.sign_tarball", - "PackageSigningRunner.sign_directory", - ("PackageSigningRunner.extract", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.log", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - assert ( - packager.run.__wrapped__.__catches__ - == (identity.GPGError, sign.SigningError)) - - with patched as (m_tarb, m_dir, m_extract, m_log): - m_extract.return_value = extract - assert not packager.run() - - assert ( - list(m_log.return_value.success.call_args) - == [('Successfully signed packages',), {}]) - - if extract: - assert ( - list(m_tarb.call_args) - == [(), {}]) - assert not m_dir.called - return - assert not m_tarb.called - assert ( - list(m_dir.call_args) - == [(), {}]) - - -def test_packager_sign(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "PackageSigningRunner.get_signing_util", - ("PackageSigningRunner.log", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.maintainer", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - path = MagicMock() - - with patched as (m_util, m_log, m_maintainer): - assert not packager.sign(path) - - assert ( - list(m_log.return_value.notice.call_args) - == [(f"Signing {path.name}s ({m_maintainer.return_value}) {path}",), {}]) - assert ( - list(m_util.call_args) - == [(path, ), {}]) - assert ( - list(m_util.return_value.sign.call_args) - == [(), {}]) - - -@pytest.mark.parametrize("utils", [[], ["a", "b", "c"]]) -@pytest.mark.parametrize("listdir", [[], ["a", "b"], ["b", "c"], ["c", "d"]]) -def test_packager_sign_all(patches, listdir, utils): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "PackageSigningRunner.sign", - ("PackageSigningRunner.signing_utils", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - path = MagicMock() - - with patched as (m_sign, m_utils): - _glob = {} - - for _path in listdir: - _mock = MagicMock() - _mock.name = _path - _glob[_path] = _mock - path.glob.return_value = _glob.values() - m_utils.return_value = utils - assert not packager.sign_all(path) - - assert ( - list(path.glob.call_args) - == [('*',), {}]) - expected = [x for x in listdir if x in utils] - assert ( - list(list(c) for c in m_sign.call_args_list) - == [[(_glob[k], ), {}] for k in expected]) - - -@pytest.mark.parametrize("tar", [True, False]) -def test_packager_sign_directory(patches, tar): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "PackageSigningRunner.archive", - "PackageSigningRunner.sign", - ("PackageSigningRunner.path", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.tar", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_archive, m_sign, m_path, m_tar): - m_tar.return_value = tar - assert not packager.sign_directory() - - assert ( - list(m_sign.call_args) - == [(m_path.return_value, ), {}]) - if not tar: - assert not m_archive.called - return - - assert ( - list(m_archive.call_args) - == [(m_path.return_value, ), {}]) - - -@pytest.mark.parametrize("tar", [True, False]) -def test_packager_sign_tarball(patches, tar): - packager = sign.PackageSigningRunner("x", "y", "z") - patched = patches( - "utils", - "PackageSigningRunner.archive", - "PackageSigningRunner.sign_all", - ("PackageSigningRunner.path", dict(new_callable=PropertyMock)), - ("PackageSigningRunner.tar", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_utils, m_archive, m_sign, m_path, m_tar): - m_tar.return_value = tar - if not tar: - with pytest.raises(sign.SigningError) as e: - packager.sign_tarball() - else: - assert not packager.sign_tarball() - - if not tar: - assert ( - e.value.args[0] - == 'You must set a `--tar` file to save to when `--extract` is set') - assert not m_utils.untar.called - assert not m_sign.called - assert not m_archive.called - return - - assert ( - list(m_utils.untar.call_args) - == [(m_path.return_value,), {}]) - assert ( - list(m_sign.call_args) - == [(m_utils.untar.return_value.__enter__.return_value,), {}]) - assert ( - list(m_archive.call_args) - == [(m_utils.untar.return_value.__enter__.return_value,), {}]) - - -# RPMMacro - -@pytest.mark.parametrize("overwrite", [[], None, True, False]) -@pytest.mark.parametrize("kwargs", [{}, dict(K1="V1", K2="V2")]) -def test_rpmmacro_constructor(patches, overwrite, kwargs): - rpmmacro = ( - sign.RPMMacro("HOME", overwrite=overwrite, **kwargs) - if overwrite != [] - else sign.RPMMacro("HOME", **kwargs)) - assert rpmmacro._macro_filename == ".rpmmacros" - assert rpmmacro._home == "HOME" - assert rpmmacro.overwrite == bool(overwrite or False) - assert rpmmacro.kwargs == kwargs - assert rpmmacro.template == sign.RPMMACRO_TEMPLATE - - -def test_rpmmacro_home(patches): - rpmmacro = sign.RPMMacro("HOME") - patched = patches( - "pathlib", - prefix="tools.distribution.sign") - with patched as (m_plib, ): - assert rpmmacro.home == m_plib.Path.return_value - - assert ( - list(m_plib.Path.call_args) - == [(rpmmacro._home,), {}]) - - -def test_rpmmacro_path(patches): - rpmmacro = sign.RPMMacro("HOME") - patched = patches( - ("RPMMacro.home", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - with patched as (m_home, ): - assert rpmmacro.path == m_home.return_value.joinpath.return_value - - assert ( - list(m_home.return_value.joinpath.call_args) - == [(rpmmacro._macro_filename, ), {}]) - - -@pytest.mark.parametrize("kwargs", [{}, dict(K1="V1", K2="V2")]) -def test_rpmmacro_macro(patches, kwargs): - rpmmacro = sign.RPMMacro("HOME", **kwargs) - patched = patches( - ("RPMMacro.template", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - with patched as (m_template, ): - result = rpmmacro.macro - - expected = m_template.return_value - for k, v in kwargs.items(): - assert ( - list(expected.replace.call_args) - == [(f"__{k.upper()}__", v), {}]) - expected = expected.replace.return_value - - assert result == expected - assert "macro" not in rpmmacro.__dict__ - - -@pytest.mark.parametrize("overwrite", [True, False]) -@pytest.mark.parametrize("exists", [True, False]) -def test_rpmmacro_write(patches, overwrite, exists): - rpmmacro = sign.RPMMacro("HOME") - patched = patches( - ("RPMMacro.macro", dict(new_callable=PropertyMock)), - ("RPMMacro.path", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - rpmmacro.overwrite = overwrite - - with patched as (m_macro, m_path): - m_path.return_value.exists.return_value = exists - assert not rpmmacro.write() - - if not overwrite: - assert ( - list(m_path.return_value.exists.call_args) - == [(), {}]) - else: - assert not m_path.return_value.exists.join.called - - if not overwrite and exists: - assert not m_path.return_value.write_text.called - return - - assert ( - list(m_path.return_value.write_text.call_args) - == [(m_macro.return_value,), {}]) - - -# RPMSigningUtil - -@pytest.mark.parametrize("args", [(), ("ARG1", "ARG2")]) -@pytest.mark.parametrize("kwargs", [{}, dict(K1="V1", K2="V2")]) -def test_rpmsign_constructor(patches, args, kwargs): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - patched = patches( - "RPMSigningUtil.setup", - "DirectorySigningUtil.__init__", - prefix="tools.distribution.sign") - - with patched as (m_setup, m_super): - rpmsign = sign.RPMSigningUtil("PATH", maintainer, *args, **kwargs) - - assert isinstance(rpmsign, sign.DirectorySigningUtil) - assert rpmsign.ext == "rpm" - assert rpmsign.command_name == "rpmsign" - assert ( - list(m_setup.call_args) - == [(), {}]) - assert ( - list(m_super.call_args) - == [('PATH', maintainer) + args, kwargs]) - assert rpmsign.rpmmacro == sign.RPMMacro - - -@pytest.mark.parametrize("gpg2", [True, False]) -def test_rpmsign_command(patches, gpg2): - maintainer = identity.GPGIdentity() - patched = patches( - "RPMSigningUtil.__init__", - ("DirectorySigningUtil.command", dict(new_callable=PropertyMock)), - ("identity.GPGIdentity.gpg_bin", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_init, m_super, m_gpg): - m_gpg.return_value.name = "gpg2" if gpg2 else "notgpg2" - m_init.return_value = None - rpmsign = sign.RPMSigningUtil("PATH", maintainer, "LOG") - rpmsign.maintainer = maintainer - - if gpg2: - assert rpmsign.command == m_super.return_value - else: - with pytest.raises(sign.SigningError) as e: - rpmsign.command - - assert ( - e.value.args[0] - == 'GPG2 is required to sign RPM packages') - - if gpg2: - assert "command" in rpmsign.__dict__ - else: - assert "command" not in rpmsign.__dict__ - - -def test_rpmsign_command_args(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - patched = patches( - "RPMSigningUtil.setup", - ("identity.GPGIdentity.fingerprint", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_setup, m_fingerprint): - rpmsign = sign.RPMSigningUtil("PATH", maintainer, "LOG") - assert ( - rpmsign.command_args - == ("--key-id", m_fingerprint.return_value, - "--addsign")) - - assert "command_args" in rpmsign.__dict__ - - -class DummyRPMSigningUtil(sign.RPMSigningUtil): - - def __init__(self, path, maintainer): - self._path = path - self.maintainer = maintainer - - -def test_rpmsign_setup(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = MagicMock() - - rpmsign = DummyRPMSigningUtil("PATH", maintainer) - - patched = patches( - ("RPMSigningUtil.rpmmacro", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_macro, ): - assert not rpmsign.setup() - - assert ( - list(m_macro.return_value.call_args) - == [(maintainer.home,), - {'maintainer': maintainer.name, - 'gpg_bin': maintainer.gpg_bin, - 'gpg_config': maintainer.gnupg_home}]) - - -def test_rpmsign_sign_pkg(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - rpmsign = DummyRPMSigningUtil("PATH", maintainer) - patched = patches( - "DirectorySigningUtil.sign_pkg", - prefix="tools.distribution.sign") - file = MagicMock() - - with patched as (m_sign, ): - assert not rpmsign.sign_pkg(file) - - assert ( - list(file.chmod.call_args) - == [(0o755, ), {}]) - assert ( - list(m_sign.call_args) - == [(file,), {}]) - - -# DebChangesFiles - -def test_changes_constructor(): - changes = sign.DebChangesFiles("SRC") - assert changes.src == "SRC" - - -def test_changes_dunder_iter(patches): - path = MagicMock() - changes = sign.DebChangesFiles(path) - - patched = patches( - ("DebChangesFiles.files", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - _files = ["FILE1", "FILE2", "FILE3"] - - with patched as (m_files, ): - m_files.return_value = _files - result = changes.__iter__() - assert list(result) == _files - - assert isinstance(result, types.GeneratorType) - assert ( - list(path.unlink.call_args) - == [(), {}]) - - -@pytest.mark.parametrize( - "lines", - [([], None), - (["FOO", "BAR"], None), - (["FOO", "BAR", "Distribution: distro1"], "distro1"), - (["FOO", "BAR", "Distribution: distro1 distro2"], "distro1 distro2"), - (["FOO", "BAR", "Distribution: distro1 distro2", "BAZ"], "distro1 distro2"), - (["FOO", "BAR", "", "Distribution: distro1 distro2"], None)]) -def test_changes_distributions(patches, lines): - lines, expected = lines - changes = sign.DebChangesFiles("SRC") - patched = patches( - "open", - prefix="tools.distribution.sign") - - class DummyFile(object): - line = 0 - - def __init__(self, lines): - self.lines = lines - - def readline(self): - if len(self.lines) > self.line: - line = self.lines[self.line] - self.line += 1 - return line - - _file = DummyFile(lines) - - with patched as (m_open, ): - m_open.return_value.__enter__.return_value.readline.side_effect = _file.readline - if expected: - assert changes.distributions == expected - else: - with pytest.raises(sign.SigningError) as e: - changes.distributions - assert ( - e.value.args[0] - == "Did not find Distribution field in changes file SRC") - - if "" in lines: - lines = lines[:lines.index("")] - - if expected: - breakon = 0 - for line in lines: - if line.startswith("Distribution:"): - break - breakon += 1 - lines = lines[:breakon] - count = len(lines) + 1 - assert ( - list(list(c) for c in m_open.return_value.__enter__.return_value.readline.call_args_list) - == [[(), {}]] * count) - - -def test_changes_files(patches): - changes = sign.DebChangesFiles("SRC") - - patched = patches( - "DebChangesFiles.changes_file", - ("DebChangesFiles.distributions", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_changes, m_distros): - m_distros.return_value = "DISTRO1 DISTRO2 DISTRO3" - result = changes.files - assert list(result) == [m_changes.return_value] * 3 - - assert isinstance(result, types.GeneratorType) - assert ( - list(list(c) for c in m_changes.call_args_list) - == [[('DISTRO1',), {}], - [('DISTRO2',), {}], - [('DISTRO3',), {}]]) - - -def test_changes_changes_file(patches): - path = MagicMock() - changes = sign.DebChangesFiles(path) - patched = patches( - "DebChangesFiles.changes_file_path", - ("DebChangesFiles.distributions", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_path, m_distros): - assert ( - changes.changes_file("DISTRO") - == m_path.return_value) - - assert ( - list(m_path.call_args) - == [('DISTRO',), {}]) - assert ( - list(m_path.return_value.write_text.call_args) - == [(path.read_text.return_value.replace.return_value,), {}]) - assert ( - list(path.read_text.call_args) - == [(), {}]) - assert ( - list(path.read_text.return_value.replace.call_args) - == [(m_distros.return_value, "DISTRO"), {}]) - - -def test_changes_file_path(): - path = MagicMock() - changes = sign.DebChangesFiles(path) - assert changes.changes_file_path("DISTRO") == path.with_suffix.return_value - assert ( - list(path.with_suffix.call_args) - == [('.DISTRO.changes',), {}]) - - -# DebSigningUtil - -@pytest.mark.parametrize("args", [(), ("ARG1", ), ("ARG2", )]) -def test_debsign_constructor(patches, args): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - debsign = sign.DebSigningUtil("PATH", maintainer, "LOG", *args) - - assert isinstance(debsign, sign.DirectorySigningUtil) - assert debsign.ext == "changes" - assert debsign.command_name == "debsign" - assert debsign._package_type == "deb" - assert debsign.changes_files == sign.DebChangesFiles - assert debsign._path == "PATH" - assert debsign.maintainer == maintainer - assert debsign.log == "LOG" - - -def test_debsign_command_args(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - patched = patches( - ("identity.GPGIdentity.fingerprint", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_fingerprint, ): - debsign = sign.DebSigningUtil("PATH", maintainer, "LOG") - assert ( - debsign.command_args - == ("-k", m_fingerprint.return_value)) - - assert "command_args" in debsign.__dict__ - - -def test_debsign_pkg_files(patches): - packager = sign.PackageSigningRunner("x", "y", "z") - maintainer = identity.GPGIdentity(packager) - debsign = sign.DebSigningUtil("PATH", maintainer, "LOG") - patched = patches( - "chain", - ("DirectorySigningUtil.pkg_files", dict(new_callable=PropertyMock)), - ("DebSigningUtil.changes_files", dict(new_callable=PropertyMock)), - prefix="tools.distribution.sign") - - with patched as (m_chain, m_pkg, m_changes): - m_pkg.return_value = ("FILE1", "FILE2", "FILE3") - m_chain.from_iterable.side_effect = lambda _iter: list(_iter) - assert ( - debsign.pkg_files - == (m_changes.return_value.return_value, ) * 3) - - assert m_chain.from_iterable.called - assert ( - list(list(c) for c in m_changes.return_value.call_args_list) - == [[('FILE1',), {}], [('FILE2',), {}], [('FILE3',), {}]]) - - -# Module - -def test_sign_main(patches, command_main): - patched = patches( - "_register_utils", - prefix="tools.distribution.sign") - - with patched as (m_reg, ): - command_main( - sign.main, - "tools.distribution.sign.PackageSigningRunner") - - assert ( - list(m_reg.call_args) - == [(), {}]) - - -def test_sign_register_utils(patches, command_main): - patched = patches( - "PackageSigningRunner.register_util", - prefix="tools.distribution.sign") - - with patched as (m_reg, ): - sign._register_utils() - - assert ( - list(list(c) for c in m_reg.call_args_list) - == [[('deb', sign.DebSigningUtil), {}], - [('rpm', sign.RPMSigningUtil), {}]]) diff --git a/tools/distribution/tests/test_verify.py b/tools/distribution/tests/test_verify.py deleted file mode 100644 index 5a3666770a1d2..0000000000000 --- a/tools/distribution/tests/test_verify.py +++ /dev/null @@ -1,472 +0,0 @@ -from itertools import chain -from unittest.mock import AsyncMock, MagicMock, PropertyMock - -import pytest - -from tools.base.checker import AsyncChecker -from tools.distribution import distrotest, verify - - -def test_checker_constructor(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - assert isinstance(checker, AsyncChecker) - assert checker._active_distrotest is None - assert checker.checks == ("distros", ) - - assert checker.test_class == distrotest.DistroTest - assert "test_class" not in checker.__dict__ - assert checker.test_config_class == distrotest.DistroTestConfig - assert "test_config_class" not in checker.__dict__ - - -def _check_arg_property(patches, prop, arg=None): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - - patched = patches( - ("PackagesDistroChecker.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_args, ): - assert getattr(checker, prop) == getattr(m_args.return_value, arg or prop) - - assert prop not in checker.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("rebuild",), - ("filter_distributions", "distribution")]) -def test_checker_arg_props(patches, prop): - _check_arg_property(patches, *prop) - - -def _check_arg_path_property(patches, prop, arg=None): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "pathlib", - ("PackagesDistroChecker.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_plib, m_args): - assert getattr(checker, prop) == m_plib.Path.return_value - - assert ( - list(m_plib.Path.call_args) - == [(getattr(m_args.return_value, arg or prop), ), {}]) - assert prop not in checker.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("testfile",), - ("keyfile",), - ("packages_tarball", "packages")]) -def test_checker_arg_path_props(patches, prop): - _check_arg_path_property(patches, *prop) - - -def test_checker_active_distrotest(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - assert checker.active_distrotest is None - checker._active_distrotest = "ATEST" - assert checker.active_distrotest == "ATEST" - assert "active_distrotest" not in checker.__dict__ - - -@pytest.mark.parametrize("is_dict", [True, False]) -def test_checker_config(patches, is_dict): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "isinstance", - "utils", - ("PackagesDistroChecker.args", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_inst, m_utils, m_args): - m_inst.return_value = is_dict - if is_dict: - assert checker.config == m_utils.from_yaml.return_value - else: - - with pytest.raises(verify.PackagesConfigurationError) as e: - checker.config - - assert ( - list(m_utils.from_yaml.call_args) - == [(m_args.return_value.config,), {}]) - - if is_dict: - assert "config" in checker.__dict__ - else: - assert ( - e.value.args[0] - == f"Unable to parse configuration {m_args.return_value.config}") - - -def test_checker_docker(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "aiodocker", - prefix="tools.distribution.verify") - - with patched as (m_docker, ): - assert checker.docker == m_docker.Docker.return_value - - assert ( - list(m_docker.Docker.call_args) - == [(), {}]) - assert "docker" in checker.__dict__ - - -def test_checker_path(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "pathlib", - ("PackagesDistroChecker.tempdir", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_plib, m_temp): - assert checker.path == m_plib.Path.return_value - - assert ( - list(m_plib.Path.call_args) - == [(m_temp.return_value.name, ), {}]) - assert "path" not in checker.__dict__ - - -def test_checker_test_config(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.docker", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.keyfile", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.packages_tarball", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.path", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.test_config_class", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.testfile", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_docker, m_key, m_tar, m_path, m_class, m_test): - assert checker.test_config == m_class.return_value.return_value - - assert ( - list(m_class.return_value.call_args) - == [(), - {'docker': m_docker.return_value, - 'keyfile': m_key.return_value, - 'path': m_path.return_value, - 'tarball': m_tar.return_value, - 'testfile': m_test.return_value, - 'maintainer': verify.ENVOY_MAINTAINER, - 'version': verify.ENVOY_VERSION}]) - assert "test_config" in checker.__dict__ - - -@pytest.mark.parametrize( - "config", - [{}, - {f"DISTRO{i}": dict(image="SOMEIMAGE", ext="EXT1", foo="FOO", bar="BAR") for i in range(1, 4)}, - {f"DISTRO{i}": dict(image="OTHERIMAGE", ext="EXT2", foo="FOO", bar="BAR") for i in range(1, 4)}]) -@pytest.mark.parametrize( - "distributions", - [None, - [], - ["DISTRO1", "DISTRO2", "DISTRO3"], - ["DISTRO1", "DISTRO3"]]) -def test_checker_tests(patches, config, distributions): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "PackagesDistroChecker.get_test_config", - "PackagesDistroChecker.get_test_packages", - ("PackagesDistroChecker.config", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.filter_distributions", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_tconfig, m_pkgs, m_config, m_tests): - m_config.return_value = config.copy() - m_tests.return_value = distributions - result = checker.tests - - if distributions: - config = {k: v for k, v in config.items() if k in distributions} - - assert ( - len(result) - == len(config) - == len(m_pkgs.call_args_list) - == len(m_tconfig.call_args_list)) - - for i, k in enumerate(result): - assert k == (list(config)[i]) - assert result[k] == m_tconfig.return_value - - assert ( - list(list(c) for c in m_tconfig.call_args_list) - == [[(_config["image"], ), {}] for _config in config.values()]) - assert ( - list(list(c) for c in m_tconfig.return_value.update.call_args_list) - == [[(_conf,), {}] for _conf in config.values()]) - assert ( - list(list(c) for c in m_tconfig.return_value.__getitem__.call_args_list) - == [[('type',), {}], [('ext',), {}]] * len(config)) - assert ( - list(list(c) for c in m_pkgs.call_args_list) - == [[(m_tconfig.return_value.__getitem__.return_value, ) * 2, {}]] * len(config)) - - -def test_checker_add_arguments(): - checker = verify.PackagesDistroChecker("x", "y", "z") - parser = MagicMock() - checker.add_arguments(parser) - assert ( - list(list(c) for c in parser.add_argument.call_args_list) - == [[('--log-level', '-l'), - {'choices': ['debug', 'info', 'warn', 'error'], - 'default': 'info', - 'help': 'Log level to display'}], - [('--fix',), - {'action': 'store_true', - 'default': False, - 'help': 'Attempt to fix in place'}], - [('--diff',), - {'action': 'store_true', - 'default': False, - 'help': 'Display a diff in the console where available'}], - [('--warning', '-w'), - {'choices': ['warn', 'error'], - 'default': 'warn', - 'help': 'Handle warnings as warnings or errors'}], - [('--summary',), - {'action': 'store_true', - 'default': False, - 'help': 'Show a summary of check runs'}], - [('--summary-errors',), - {'type': int, - 'default': 5, - 'help': 'Number of errors to show in the summary, -1 shows all'}], - [('--summary-warnings',), - {'type': int, - 'default': 5, - 'help': 'Number of warnings to show in the summary, -1 shows all'}], - [('--check', '-c'), - {'choices': ('distros',), - 'nargs': '*', - 'help': 'Specify which checks to run, can be specified for multiple checks'}], - [('--config-distros',), - {'default': '', - 'help': 'Custom configuration for the distros check'}], - [('--path', '-p'), - {'default': None, - 'help': 'Path to the test root (usually Envoy source dir). If not specified the first path of paths is used'}], - [('paths',), - {'nargs': '*', - 'help': 'Paths to check. At least one path must be specified, or the `path` argument should be provided'}], - [('testfile',), - {'help': 'Path to the test file that will be run inside the distribution containers'}], - [('config',), - {'help': 'Path to a YAML configuration with distributions for testing'}], - [('packages',), - {'help': 'Path to a tarball containing packages to test'}], - [('--keyfile', '-k'), - {'help': 'Specify the path to a file containing a gpg key for verifying packages.'}], - [('--distribution', '-d'), - {'nargs': '?', - 'help': 'Specify distribution to test. Can be specified multiple times.'}], - [('--rebuild',), - {'action': 'store_true', - 'help': 'Rebuild test images before running the tests.'}]]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "tests", - [{}, - {f"DISTRO{i}": dict(image=f"IMAGE{i}") - for i in range(1, 4)}]) -@pytest.mark.parametrize("rebuild", [True, False]) -async def test_checker_check_distros(patches, tests, rebuild): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "PackagesDistroChecker.run_test", - ("PackagesDistroChecker.log", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.rebuild", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.tests", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - _items = {} - for i, (k, v) in enumerate(tests.items()): - v["packages"] = [] - for x in range(0, 3): - _mock = MagicMock() - _mock.name = f"P{i}{x}" - v["packages"].append(_mock) - _items[k] = v - - with patched as (m_dtest, m_log, m_rebuild, m_tests): - m_tests.return_value.items.return_value = _items.items() - m_rebuild.return_value = rebuild - assert not await checker.check_distros() - - assert ( - list(list(c) for c in m_log.return_value.info.call_args_list) - == [[(f'[{name}] Testing with: {",".join(n.name for n in tests[name]["packages"])}',), {}] - for name in tests]) - expected = list( - chain.from_iterable( - [[(name, tests[name]["image"], package, (i == 0 and rebuild)), {}] - for i, package in enumerate(tests[name]["packages"])] - for name in tests)) - assert ( - list(list(c) for c in m_dtest.call_args_list) - == expected) - - -def test_checker_get_test_config(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.test_config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_config, ): - assert checker.get_test_config("IMAGE") == m_config.return_value.get_config.return_value - - assert ( - list(m_config.return_value.get_config.call_args) - == [('IMAGE',), {}]) - - -def test_checker_get_test_packages(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.test_config", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_config, ): - assert checker.get_test_packages("TYPE", "EXT") == m_config.return_value.get_packages.return_value - - assert ( - list(m_config.return_value.get_packages.call_args) - == [('TYPE', 'EXT'), {}]) - - -@pytest.mark.asyncio -async def test_checker_on_checks_complete(patches): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - "PackagesDistroChecker._cleanup_test", - "PackagesDistroChecker._cleanup_docker", - "checker.BaseChecker.on_checks_complete", - prefix="tools.distribution.verify") - order_mock = MagicMock() - - with patched as (m_test, m_docker, m_complete): - m_test.side_effect = lambda: order_mock("TEST") - m_docker.side_effect = lambda: order_mock("DOCKER") - m_complete.side_effect = lambda: (order_mock('COMPLETE') and "COMPLETE") - assert await checker.on_checks_complete() == "COMPLETE" - - assert ( - (list(list(c) for c in order_mock.call_args_list)) - == [[('TEST',), {}], - [('DOCKER',), {}], - [('COMPLETE',), {}]]) - - for m in m_test, m_docker, m_complete: - assert ( - list(m.call_args) - == [(), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("exiting", [True, False]) -@pytest.mark.parametrize("errors", [None, (), ("ERR1", "ERR")]) -@pytest.mark.parametrize("rebuild", [True, False]) -async def test_checker_run_test(patches, exiting, errors, rebuild): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.test_class", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.test_config", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.exiting", dict(new_callable=PropertyMock)), - ("PackagesDistroChecker.log", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - config = dict( - type="TESTTYPE", - image="IMAGE") - - with patched as (m_test, m_config, m_exit, m_log): - m_exit.return_value = exiting - m_test.return_value.return_value.run = AsyncMock( - return_value=errors) - assert not await checker.run_test("NAME", "IMAGE", "PACKAGE", rebuild) - - if exiting: - assert not m_log.called - assert not m_test.called - assert not checker._active_distrotest - return - - assert ( - checker._active_distrotest - == m_test.return_value.return_value) - assert ( - list(m_log.return_value.info.call_args) - == [('[NAME] Testing package: PACKAGE',), {}]) - assert ( - list(m_test.return_value.call_args) - == [(checker, m_config.return_value, 'NAME', 'IMAGE', 'PACKAGE'), {"rebuild": rebuild}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("exists", [True, False]) -async def test_checker__cleanup_docker(patches, exists): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.docker", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - if exists: - checker.__dict__["docker"] = "DOCKER" - - with patched as (m_docker, ): - m_docker.return_value.close = AsyncMock() - await checker._cleanup_docker() - - assert "docker" not in checker.__dict__ - - if not exists: - assert not m_docker.return_value.close.called - return - - assert ( - list(m_docker.return_value.close.call_args) - == [(), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("exists", [True, False]) -async def test_checker__cleanup_test(patches, exists): - checker = verify.PackagesDistroChecker("path1", "path2", "path3") - patched = patches( - ("PackagesDistroChecker.active_distrotest", dict(new_callable=PropertyMock)), - prefix="tools.distribution.verify") - - with patched as (m_active, ): - if not exists: - m_active.return_value = None - else: - m_active.return_value.cleanup = AsyncMock() - await checker._cleanup_test() - - if not exists: - return - - assert ( - list(m_active.return_value.cleanup.call_args) - == [(), {}]) - - -# Module - -def test_verify_main(patches, command_main): - command_main( - verify.main, - "tools.distribution.verify.PackagesDistroChecker") diff --git a/tools/distribution/verify.py b/tools/distribution/verify.py deleted file mode 100644 index 6eb6de41a7d26..0000000000000 --- a/tools/distribution/verify.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python3 - -# -# This tool allows you to test a tarball of built packages (eg debs, rpms) -# against a configurable set of OS distributions. -# -# usage -# -# with bazel: -# -# bazel run //tools/distribution:verify -- -h -# -# alternatively, if you have the necessary python deps available -# -# PYTHONPATH=. ./tools/distribution/verify.py -h -# -# python requires: aiodocker, coloredlogs, frozendict, verboselogs -# - -import argparse -import pathlib -import sys -from functools import cached_property -from typing import Optional, Type - -import aiodocker - -from tools.base import checker, utils -from tools.distribution import distrotest - -# TODO(phlax): make this configurable -ENVOY_MAINTAINER = "Envoy maintainers " -ENVOY_VERSION = "1.20.0" - - -class PackagesConfigurationError(Exception): - pass - - -class PackagesDistroChecker(checker.AsyncChecker): - _active_distrotest = None - checks = ("distros",) - - @property - def active_distrotest(self) -> Optional[distrotest.DistroTest]: - """Currently active test""" - return self._active_distrotest - - @cached_property - def config(self) -> dict: - """Config parsed from the provided path - - Expects a yaml file with distributions in the following format: - - ```yaml - - debian_buster: - # Docker image tag name - image: debian:buster-slim - # File extension of installable packages, for packages signed for - # particular distributions, this can be the distribution name and `.changes` - # extension. - ext: buster.changes - - ubuntu_foo: - image: ubuntu:foo - ext: foo.changes - - redhat_8.1: - image: registry.access.redhat.com/ubi8/ubi:8.1 - ``` - """ - config = utils.from_yaml(self.args.config) - if not isinstance(config, dict): - raise PackagesConfigurationError(f"Unable to parse configuration {self.args.config}") - return config - - @cached_property - def docker(self) -> aiodocker.Docker: - """An instance of `aiodocker.Docker`""" - return aiodocker.Docker() - - @property - def filter_distributions(self) -> list: - """List of distributions to filter the tests to be run with""" - return self.args.distribution - - @property - def keyfile(self) -> pathlib.Path: - """Path to a keyfile to to include in the Docker images for verifying - package signatures - """ - return pathlib.Path(self.args.keyfile) - - @property - def packages_tarball(self) -> pathlib.Path: - """Path to the packages tarball""" - return pathlib.Path(self.args.packages) - - @property - def path(self) -> pathlib.Path: - """Path to a temporary directory to run the tests from""" - return pathlib.Path(self.tempdir.name) - - @property - def rebuild(self) -> bool: - """Flag to rebuild the test images even if they exist""" - return self.args.rebuild - - @property - def test_class(self) -> Type[distrotest.DistroTest]: - """The test class to run the tests with""" - return distrotest.DistroTest - - @cached_property - def test_config(self) -> distrotest.DistroTestConfig: - """The test config - - Parses global and provided configs to store and resolve configurations - for the test runner. - - Also extracts the packages to the temporary directory and provides info - on available packages to test with. - """ - return self.test_config_class( - docker=self.docker, - path=self.path, - tarball=self.packages_tarball, - keyfile=self.keyfile, - testfile=self.testfile, - maintainer=ENVOY_MAINTAINER, - version=ENVOY_VERSION) - - @property - def test_config_class(self) -> Type[distrotest.DistroTestConfig]: - """The test config class""" - return distrotest.DistroTestConfig - - @property - def testfile(self) -> pathlib.Path: - """Path to a testfile to run inside the test containers""" - return pathlib.Path(self.args.testfile) - - @cached_property - def tests(self) -> dict: - """A dictionary of tests and test configuration, filtered according - to provided args - """ - _ret = {} - for name, config in self.config.items(): - if self.filter_distributions and name not in self.filter_distributions: - continue - _ret[name] = self.get_test_config(config["image"]) - _ret[name].update(config) - _ret[name]["packages"] = self.get_test_packages(_ret[name]["type"], _ret[name]["ext"]) - return _ret - - def add_arguments(self, parser: argparse.ArgumentParser) -> None: - super().add_arguments(parser) - parser.add_argument( - "testfile", - help="Path to the test file that will be run inside the distribution containers") - parser.add_argument( - "config", help="Path to a YAML configuration with distributions for testing") - parser.add_argument("packages", help="Path to a tarball containing packages to test") - parser.add_argument( - "--keyfile", - "-k", - help="Specify the path to a file containing a gpg key for verifying packages.") - parser.add_argument( - "--distribution", - "-d", - nargs="?", - help="Specify distribution to test. Can be specified multiple times.") - parser.add_argument( - "--rebuild", action="store_true", help="Rebuild test images before running the tests.") - - async def check_distros(self) -> None: - """Check runner""" - for name, config in self.tests.items(): - self.log.info( - f"[{name}] Testing with: " - f"{','.join(p.name for p in config['packages'])}") - for i, package in enumerate(config["packages"]): - await self.run_test(name, config["image"], package, (i == 0 and self.rebuild)) - - def get_test_config(self, image: str) -> dict: - """Get the type/ext config for a given image name""" - return self.test_config.get_config(image) - - def get_test_packages(self, type: str, ext: str) -> list: - """Get the packages to test for a given type/ext""" - return self.test_config.get_packages(type, ext) - - async def on_checks_complete(self) -> int: - """Cleanup and return the test result""" - await self._cleanup_test() - await self._cleanup_docker() - return await super().on_checks_complete() - - async def run_test(self, name: str, image: str, package: pathlib.Path, rebuild: bool) -> None: - """Runs a test for each of the packages against a particular distro""" - if self.exiting: - return - self.log.info(f"[{name}] Testing package: {package}") - self._active_distrotest = self.test_class( - self, self.test_config, name, image, package, rebuild=rebuild) - await self._active_distrotest.run() - - async def _cleanup_docker(self) -> None: - """Close the docker connection""" - if "docker" in self.__dict__: - await self.docker.close() - del self.__dict__["docker"] - - async def _cleanup_test(self) -> None: - """Cleanup test containers""" - if self.active_distrotest: - await self.active_distrotest.cleanup() - - -def main(*args) -> int: - return PackagesDistroChecker(*args).run() - - -if __name__ == "__main__": - sys.exit(main(*sys.argv[1:])) diff --git a/tools/docker/BUILD b/tools/docker/BUILD deleted file mode 100644 index 9fd20eedd3b1f..0000000000000 --- a/tools/docker/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -load("//bazel:envoy_build_system.bzl", "envoy_package") -load("@docker_pip3//:requirements.bzl", "requirement") -load("//tools/base:envoy_python.bzl", "envoy_py_library") - -licenses(["notice"]) # Apache 2 - -envoy_package() - -envoy_py_library( - name = "tools.docker.utils", - deps = [ - requirement("aiodocker"), - ], -) diff --git a/tools/docker/requirements.txt b/tools/docker/requirements.txt deleted file mode 100644 index 3d00aa942a2d2..0000000000000 --- a/tools/docker/requirements.txt +++ /dev/null @@ -1,152 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes tools/docker/requirements.txt -# -aiodocker==0.19.1 \ - --hash=sha256:59dfae91b5acbfa953baf4a3553b7c5ff375346b0f3bbfd8cae11c3b93adce04 \ - --hash=sha256:bfbb44dbee185dbc8943be68d1f51358af3ec473c463bdee68a25e33d70ae3ad - # via -r tools/docker/requirements.txt -aiohttp==3.7.4.post0 \ - --hash=sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe \ - --hash=sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe \ - --hash=sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5 \ - --hash=sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8 \ - --hash=sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd \ - --hash=sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb \ - --hash=sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c \ - --hash=sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87 \ - --hash=sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0 \ - --hash=sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290 \ - --hash=sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5 \ - --hash=sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287 \ - --hash=sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde \ - --hash=sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf \ - --hash=sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8 \ - --hash=sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16 \ - --hash=sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf \ - --hash=sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809 \ - --hash=sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213 \ - --hash=sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f \ - --hash=sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013 \ - --hash=sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b \ - --hash=sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9 \ - --hash=sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5 \ - --hash=sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb \ - --hash=sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df \ - --hash=sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4 \ - --hash=sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439 \ - --hash=sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f \ - --hash=sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22 \ - --hash=sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f \ - --hash=sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5 \ - --hash=sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970 \ - --hash=sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009 \ - --hash=sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc \ - --hash=sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a \ - --hash=sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95 - # via aiodocker -async-timeout==3.0.1 \ - --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \ - --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3 - # via aiohttp -attrs==21.2.0 \ - --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ - --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb - # via aiohttp -chardet==4.0.0 \ - --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ - --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 - # via aiohttp -idna==3.2 \ - --hash=sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a \ - --hash=sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3 - # via yarl -multidict==5.1.0 \ - --hash=sha256:018132dbd8688c7a69ad89c4a3f39ea2f9f33302ebe567a879da8f4ca73f0d0a \ - --hash=sha256:051012ccee979b2b06be928a6150d237aec75dd6bf2d1eeeb190baf2b05abc93 \ - --hash=sha256:05c20b68e512166fddba59a918773ba002fdd77800cad9f55b59790030bab632 \ - --hash=sha256:07b42215124aedecc6083f1ce6b7e5ec5b50047afa701f3442054373a6deb656 \ - --hash=sha256:0e3c84e6c67eba89c2dbcee08504ba8644ab4284863452450520dad8f1e89b79 \ - --hash=sha256:0e929169f9c090dae0646a011c8b058e5e5fb391466016b39d21745b48817fd7 \ - --hash=sha256:1ab820665e67373de5802acae069a6a05567ae234ddb129f31d290fc3d1aa56d \ - --hash=sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5 \ - --hash=sha256:2e68965192c4ea61fff1b81c14ff712fc7dc15d2bd120602e4a3494ea6584224 \ - --hash=sha256:2f1a132f1c88724674271d636e6b7351477c27722f2ed789f719f9e3545a3d26 \ - --hash=sha256:37e5438e1c78931df5d3c0c78ae049092877e5e9c02dd1ff5abb9cf27a5914ea \ - --hash=sha256:3a041b76d13706b7fff23b9fc83117c7b8fe8d5fe9e6be45eee72b9baa75f348 \ - --hash=sha256:3a4f32116f8f72ecf2a29dabfb27b23ab7cdc0ba807e8459e59a93a9be9506f6 \ - --hash=sha256:46c73e09ad374a6d876c599f2328161bcd95e280f84d2060cf57991dec5cfe76 \ - --hash=sha256:46dd362c2f045095c920162e9307de5ffd0a1bfbba0a6e990b344366f55a30c1 \ - --hash=sha256:4b186eb7d6ae7c06eb4392411189469e6a820da81447f46c0072a41c748ab73f \ - --hash=sha256:54fd1e83a184e19c598d5e70ba508196fd0bbdd676ce159feb412a4a6664f952 \ - --hash=sha256:585fd452dd7782130d112f7ddf3473ffdd521414674c33876187e101b588738a \ - --hash=sha256:5cf3443199b83ed9e955f511b5b241fd3ae004e3cb81c58ec10f4fe47c7dce37 \ - --hash=sha256:6a4d5ce640e37b0efcc8441caeea8f43a06addace2335bd11151bc02d2ee31f9 \ - --hash=sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359 \ - --hash=sha256:806068d4f86cb06af37cd65821554f98240a19ce646d3cd24e1c33587f313eb8 \ - --hash=sha256:830f57206cc96ed0ccf68304141fec9481a096c4d2e2831f311bde1c404401da \ - --hash=sha256:929006d3c2d923788ba153ad0de8ed2e5ed39fdbe8e7be21e2f22ed06c6783d3 \ - --hash=sha256:9436dc58c123f07b230383083855593550c4d301d2532045a17ccf6eca505f6d \ - --hash=sha256:9dd6e9b1a913d096ac95d0399bd737e00f2af1e1594a787e00f7975778c8b2bf \ - --hash=sha256:ace010325c787c378afd7f7c1ac66b26313b3344628652eacd149bdd23c68841 \ - --hash=sha256:b47a43177a5e65b771b80db71e7be76c0ba23cc8aa73eeeb089ed5219cdbe27d \ - --hash=sha256:b797515be8743b771aa868f83563f789bbd4b236659ba52243b735d80b29ed93 \ - --hash=sha256:b7993704f1a4b204e71debe6095150d43b2ee6150fa4f44d6d966ec356a8d61f \ - --hash=sha256:d5c65bdf4484872c4af3150aeebe101ba560dcfb34488d9a8ff8dbcd21079647 \ - --hash=sha256:d81eddcb12d608cc08081fa88d046c78afb1bf8107e6feab5d43503fea74a635 \ - --hash=sha256:dc862056f76443a0db4509116c5cd480fe1b6a2d45512a653f9a855cc0517456 \ - --hash=sha256:ecc771ab628ea281517e24fd2c52e8f31c41e66652d07599ad8818abaad38cda \ - --hash=sha256:f200755768dc19c6f4e2b672421e0ebb3dd54c38d5a4f262b872d8cfcc9e93b5 \ - --hash=sha256:f21756997ad8ef815d8ef3d34edd98804ab5ea337feedcd62fb52d22bf531281 \ - --hash=sha256:fc13a9524bc18b6fb6e0dbec3533ba0496bbed167c56d0aabefd965584557d80 - # via - # aiohttp - # yarl -typing-extensions==3.10.0.1 \ - --hash=sha256:8bbffbd37fbeb9747a0241fdfde5ae99d4531ad1d1a41ccaea62100e15a5814c \ - --hash=sha256:045dd532231acfa03628df5e0c66dba64e2cc8fc8b844538d4ad6d5dd6cb82dc \ - --hash=sha256:83af6730a045fda60f46510f7f1f094776d90321caa4d97d20ef38871bef4bd3 - # via - # aiodocker - # aiohttp -yarl==1.6.3 \ - --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \ - --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \ - --hash=sha256:15263c3b0b47968c1d90daa89f21fcc889bb4b1aac5555580d74565de6836366 \ - --hash=sha256:2ce4c621d21326a4a5500c25031e102af589edb50c09b321049e388b3934eec3 \ - --hash=sha256:31ede6e8c4329fb81c86706ba8f6bf661a924b53ba191b27aa5fcee5714d18ec \ - --hash=sha256:324ba3d3c6fee56e2e0b0d09bf5c73824b9f08234339d2b788af65e60040c959 \ - --hash=sha256:329412812ecfc94a57cd37c9d547579510a9e83c516bc069470db5f75684629e \ - --hash=sha256:4736eaee5626db8d9cda9eb5282028cc834e2aeb194e0d8b50217d707e98bb5c \ - --hash=sha256:4953fb0b4fdb7e08b2f3b3be80a00d28c5c8a2056bb066169de00e6501b986b6 \ - --hash=sha256:4c5bcfc3ed226bf6419f7a33982fb4b8ec2e45785a0561eb99274ebbf09fdd6a \ - --hash=sha256:547f7665ad50fa8563150ed079f8e805e63dd85def6674c97efd78eed6c224a6 \ - --hash=sha256:5b883e458058f8d6099e4420f0cc2567989032b5f34b271c0827de9f1079a424 \ - --hash=sha256:63f90b20ca654b3ecc7a8d62c03ffa46999595f0167d6450fa8383bab252987e \ - --hash=sha256:68dc568889b1c13f1e4745c96b931cc94fdd0defe92a72c2b8ce01091b22e35f \ - --hash=sha256:69ee97c71fee1f63d04c945f56d5d726483c4762845400a6795a3b75d56b6c50 \ - --hash=sha256:6d6283d8e0631b617edf0fd726353cb76630b83a089a40933043894e7f6721e2 \ - --hash=sha256:72a660bdd24497e3e84f5519e57a9ee9220b6f3ac4d45056961bf22838ce20cc \ - --hash=sha256:73494d5b71099ae8cb8754f1df131c11d433b387efab7b51849e7e1e851f07a4 \ - --hash=sha256:7356644cbed76119d0b6bd32ffba704d30d747e0c217109d7979a7bc36c4d970 \ - --hash=sha256:8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10 \ - --hash=sha256:8aa3decd5e0e852dc68335abf5478a518b41bf2ab2f330fe44916399efedfae0 \ - --hash=sha256:97b5bdc450d63c3ba30a127d018b866ea94e65655efaf889ebeabc20f7d12406 \ - --hash=sha256:9ede61b0854e267fd565e7527e2f2eb3ef8858b301319be0604177690e1a3896 \ - --hash=sha256:b2e9a456c121e26d13c29251f8267541bd75e6a1ccf9e859179701c36a078643 \ - --hash=sha256:b5dfc9a40c198334f4f3f55880ecf910adebdcb2a0b9a9c23c9345faa9185721 \ - --hash=sha256:bafb450deef6861815ed579c7a6113a879a6ef58aed4c3a4be54400ae8871478 \ - --hash=sha256:c49ff66d479d38ab863c50f7bb27dee97c6627c5fe60697de15529da9c3de724 \ - --hash=sha256:ce3beb46a72d9f2190f9e1027886bfc513702d748047b548b05dab7dfb584d2e \ - --hash=sha256:d26608cf178efb8faa5ff0f2d2e77c208f471c5a3709e577a7b3fd0445703ac8 \ - --hash=sha256:d597767fcd2c3dc49d6eea360c458b65643d1e4dbed91361cf5e36e53c1f8c96 \ - --hash=sha256:d5c32c82990e4ac4d8150fd7652b972216b204de4e83a122546dce571c1bdf25 \ - --hash=sha256:d8d07d102f17b68966e2de0e07bfd6e139c7c02ef06d3a0f8d2f0f055e13bb76 \ - --hash=sha256:e46fba844f4895b36f4c398c5af062a9808d1f26b2999c58909517384d5deda2 \ - --hash=sha256:e6b5460dc5ad42ad2b36cca524491dfcaffbfd9c8df50508bddc354e787b8dc2 \ - --hash=sha256:f040bcc6725c821a4c0665f3aa96a4d0805a7aaf2caf266d256b8ed71b9f041c \ - --hash=sha256:f0b059678fd549c66b89bed03efcabb009075bd131c248ecdf087bdb6faba24a \ - --hash=sha256:fcbb48a93e8699eae920f8d92f7160c03567b421bc17362a9ffbbd706a816f71 - # via aiohttp diff --git a/tools/docker/tests/test_utils.py b/tools/docker/tests/test_utils.py deleted file mode 100644 index dba3026160b27..0000000000000 --- a/tools/docker/tests/test_utils.py +++ /dev/null @@ -1,145 +0,0 @@ -from unittest.mock import AsyncMock, MagicMock - -import pytest - -from tools.docker import utils - - -class MockAsyncIterator: - def __init__(self, seq): - self.iter = iter(seq) - self.count = 0 - - def __aiter__(self): - return self - - async def __anext__(self): - self.count += 1 - try: - return next(self.iter) - except StopIteration: - raise StopAsyncIteration - - -@pytest.mark.asyncio -@pytest.mark.parametrize("args", [(), ("ARG1", ), ("ARG1", "ARG2")]) -@pytest.mark.parametrize("kwargs", [{}, dict(kkey1="VVAR1", kkey2="VVAR2")]) -async def test_util_build_image(patches, args, kwargs): - patched = patches( - "_build_image", - "tempfile", - prefix="tools.docker.utils") - - with patched as (m_build, m_temp): - assert not await utils.build_image(*args, **kwargs) - - assert ( - list(m_temp.NamedTemporaryFile.call_args) - == [(), {}]) - - assert ( - list(m_build.call_args) - == [(m_temp.NamedTemporaryFile.return_value.__enter__.return_value, ) + args, - kwargs]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("buildargs", [None, dict(key1="VAR1", key2="VAR2")]) -@pytest.mark.parametrize("error", [None, "SOMETHING WENT WRONG"]) -async def test_util__build_image(patches, stream, buildargs, error): - lines = ( - dict(notstream=f"NOTLINE{i}", - stream=f"LINE{i}") - for i in range(1, 4)) - - if error: - lines = list(lines) - lines[1]["errorDetail"] = dict(message=error) - lines = iter(lines) - - docker = AsyncMock() - docker.images.build = MagicMock(return_value=MockAsyncIterator(lines)) - - _stream = MagicMock() - tar = MagicMock() - patched = patches( - "tarfile", - prefix="tools.docker.utils") - - with patched as (m_tar, ): - args = (tar, docker, "CONTEXT", "TAG") - kwargs = {} - if stream: - kwargs["stream"] = _stream - if buildargs: - kwargs["buildargs"] = buildargs - - if error: - with pytest.raises(utils.BuildError) as e: - await utils._build_image(*args, **kwargs) - else: - assert not await utils._build_image(*args, **kwargs) - - assert ( - list(m_tar.open.call_args) - == [(tar.name,), {'fileobj': tar, 'mode': 'w'}]) - assert ( - list(m_tar.open.return_value.__enter__.return_value.add.call_args) - == [('CONTEXT',), {'arcname': '.'}]) - assert ( - list(tar.seek.call_args) - == [(0,), {}]) - assert ( - list(docker.images.build.call_args) - == [(), - {'fileobj': tar, - 'encoding': 'gzip', - 'tag': 'TAG', - 'stream': True, - 'buildargs': buildargs or {}}]) - if stream and error: - assert ( - list(list(c) for c in _stream.call_args_list) - == [[('LINE1',), {}]]) - return - elif stream: - assert ( - list(list(c) for c in _stream.call_args_list) - == [[(f'LINE{i}',), {}] for i in range(1, 4)]) - return - # the iterator should be called n + 1 for the n of items - # if there was an error it should stop at the error - assert docker.images.build.return_value.count == 2 if error else 4 - assert not _stream.called - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [True, False]) -@pytest.mark.parametrize("url", [None, "URL"]) -async def test_util_docker_client(patches, raises, url): - - class DummyError(Exception): - pass - - patched = patches( - "aiodocker", - prefix="tools.docker.utils") - - with patched as (m_docker, ): - m_docker.Docker.return_value.close = AsyncMock() - if raises: - with pytest.raises(DummyError): - async with utils.docker_client(url) as docker: - raise DummyError() - else: - async with utils.docker_client(url) as docker: - pass - - assert ( - list(m_docker.Docker.call_args) - == [(url,), {}]) - assert docker == m_docker.Docker.return_value - assert ( - list(m_docker.Docker.return_value.close.call_args) - == [(), {}]) diff --git a/tools/docker/utils.py b/tools/docker/utils.py deleted file mode 100644 index b28cce2876f2d..0000000000000 --- a/tools/docker/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -import tarfile -import tempfile -from contextlib import asynccontextmanager -from typing import AsyncIterator, Callable, Optional - -import aiodocker - - -class BuildError(Exception): - pass - - -async def _build_image( - tar, #: IO[bytes] (`docker.images.build` expects `BinaryIO`) - docker: aiodocker.Docker, - context: str, - tag: str, - buildargs: Optional[dict] = None, - stream: Optional[Callable] = None, - **kwargs) -> None: - """Docker image builder - - if a `stream` callable arg is supplied, logs are output there. - - raises `tools.docker.utils.BuildError` with any error output. - """ - # create a tarfile from the supplied directory - with tarfile.open(tar.name, fileobj=tar, mode="w") as tarball: - tarball.add(context, arcname=".") - tar.seek(0) - - # build the docker image - build = docker.images.build( - fileobj=tar, encoding="gzip", tag=tag, stream=True, buildargs=buildargs or {}, **kwargs) - - async for line in build: - if line.get("errorDetail"): - raise BuildError( - f"Docker image failed to build {tag} {buildargs}\n{line['errorDetail']['message']}") - if stream and "stream" in line: - stream(line["stream"].strip()) - - -async def build_image(*args, **kwargs) -> None: - """Creates a Docker context by tarballing a directory, and then building an image with it - - aiodocker doesn't provide an in-built way to build docker images from a directory, only - a file, so you can't include artefacts. - - this adds the ability to include artefacts. - - as an example, assuming you have a directory containing a `Dockerfile` and some artefacts at - `/tmp/mydockercontext` - and wanted to build the image `envoy:foo` you could: - - ```python - - import asyncio - - from tools.docker import utils - - - async def myimage(): - async with utils.docker_client() as docker: - await utils.build_image( - docker, - "/tmp/mydockerbuildcontext", - "envoy:foo", - buildargs={}) - - asyncio.run(myimage()) - ``` - """ - with tempfile.NamedTemporaryFile() as tar: - await _build_image(tar, *args, **kwargs) - - -@asynccontextmanager -async def docker_client(url: Optional[str] = "") -> AsyncIterator[aiodocker.Docker]: - """Aiodocker client - - For example to dump the docker image data: - - ```python - - import asyncio - - from tools.docker import utils - - - async def docker_images(): - async with utils.docker_client() as docker: - print(await docker.images.list()) - - asyncio.run(docker_images()) - ``` - """ - - docker = aiodocker.Docker(url) - try: - yield docker - finally: - await docker.close() diff --git a/tools/github/release/BUILD b/tools/github/release/BUILD deleted file mode 100644 index 2f68200ffc6b2..0000000000000 --- a/tools/github/release/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -load("@github_pip3//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_library") -load("//bazel:envoy_build_system.bzl", "envoy_package") -load("//tools/base:envoy_python.bzl", "envoy_py_library") - -licenses(["notice"]) # Apache 2 - -envoy_package() - -py_library( - name = "abstract", - srcs = ["abstract.py"], - deps = [ - "//tools/base:abstract", - "//tools/base:functional", - "//tools/base:utils", - requirement("aiohttp"), - requirement("gidgethub"), - requirement("packaging"), - ], -) - -py_library( - name = "exceptions", - srcs = ["exceptions.py"], -) - -envoy_py_library( - "tools.github.release.manager", - deps = [ - ":abstract", - ":exceptions", - "//tools/base:abstract", - "//tools/base:functional", - "//tools/base:utils", - requirement("aiohttp"), - requirement("gidgethub"), - requirement("packaging"), - ], -) - -envoy_py_library( - "tools.github.release.release", - deps = [ - ":abstract", - ":exceptions", - "//tools/base:aio", - "//tools/base:functional", - "//tools/base:utils", - requirement("gidgethub"), - ], -) diff --git a/tools/github/release/abstract.py b/tools/github/release/abstract.py deleted file mode 100644 index 1542e0a7db149..0000000000000 --- a/tools/github/release/abstract.py +++ /dev/null @@ -1,313 +0,0 @@ -import pathlib -from abc import abstractmethod -from functools import cached_property -from typing import ( - Any, AsyncGenerator, Awaitable, Coroutine, Dict, Iterable, Iterator, List, Optional, Pattern, - Set, Type, Union) - -import verboselogs # type:ignore - -import packaging.version - -import aiohttp - -import gidgethub.abc - -from tools.base import abstract -from tools.base.functional import async_property - - -class AGithubReleaseAssets(metaclass=abstract.Abstraction): - """Base class for Github release assets pusher/fetcher""" - - @abstractmethod - def __init__(self, release: "AGithubRelease", path: pathlib.Path) -> None: - raise NotImplementedError - - @abstractmethod - async def __aiter__(self) -> AsyncGenerator[Dict[str, Union[str, pathlib.Path]], Awaitable]: - if False: - yield - raise NotImplementedError - - @abstractmethod - def __enter__(self) -> "AGithubReleaseAssets": - raise NotImplementedError - - @async_property - @abstractmethod - async def assets(self) -> Dict: - """Github release asset dictionaries""" - raise NotImplementedError - - @async_property - @abstractmethod - async def awaitables( - self) -> AsyncGenerator[Coroutine[Any, Any, Dict[str, Union[str, pathlib.Path]]], Dict]: - raise NotImplementedError - - -class AGithubReleaseAssetsFetcher(AGithubReleaseAssets, metaclass=abstract.Abstraction): - """Fetcher of Github release assets""" - - @abstractmethod - def __init__( - self, - release: "AGithubRelease", - path: pathlib.Path, - asset_types: Optional[Dict[str, Pattern[str]]] = None, - append: Optional[bool] = False) -> None: - raise NotImplementedError - - @cached_property - @abstractmethod - def asset_types(self) -> Dict[str, Pattern[str]]: - """Patterns for grouping assets""" - raise NotImplementedError - - @abstractmethod - def asset_type(self, asset: Dict) -> Optional[str]: - """Categorization of an asset into an asset type - - The default `asset_types` matcher will just match all files. - - A dictionary of `re` matchers can be provided, eg: - - ``` - asset_types = dict( - deb=re.compile(".*(\\.deb|\\.changes)$"), - rpm=re.compile(".*\\.rpm$")) - ``` - """ - raise NotImplementedError - - @abstractmethod - async def download(self, asset: Dict) -> Dict[str, Union[str, pathlib.Path]]: - """Download an asset""" - raise NotImplementedError - - @abstractmethod - async def save(self, asset_type: str, name: str, - download: aiohttp.ClientResponse) -> Dict[str, Union[str, pathlib.Path]]: - """Save an asset of given type to disk""" - raise NotImplementedError - - -class AGithubReleaseAssetsPusher(AGithubReleaseAssets, metaclass=abstract.Abstraction): - """Pusher of Github release assets""" - - @abstractmethod - def artefacts(self) -> Iterator[pathlib.Path]: - """Iterator of matching (ie release file type) artefacts found in a given path""" - raise NotImplementedError - - @abstractmethod - async def upload(self, artefact: pathlib.Path, url: str) -> Dict[str, Union[str, pathlib.Path]]: - """Upload an artefact from a filepath to a given URL""" - raise NotImplementedError - - -class AGithubRelease(metaclass=abstract.Abstraction): - """A Github tagged release version - - Provides CRUD operations for a release and its assets, and therefore - can exist already, or be created. - """ - - @abstractmethod - def __init__(self, manager: "AGithubReleaseManager", version: str): - raise NotImplementedError - - @async_property(cache=True) - @abstractmethod - async def asset_names(self) -> Set[str]: - """Set of the names of assets for this release version""" - raise NotImplementedError - - @async_property(cache=True) - @abstractmethod - async def assets(self) -> Dict: - raise NotImplementedError - - @property - @abstractmethod - def fetcher(self) -> Type[AGithubReleaseAssetsFetcher]: - """An instance of `AGithubReleaseAssetsFetcher` for fetching release - assets. - """ - raise NotImplementedError - - @property - @abstractmethod - def github(self) -> gidgethub.abc.GitHubAPI: - raise NotImplementedError - - @property - @abstractmethod - def pusher(self) -> Type[AGithubReleaseAssetsPusher]: - """An instance of `AGithubReleaseAssetsPusher` for pushing release - assets. - """ - raise NotImplementedError - - @property - @abstractmethod - def session(self) -> aiohttp.ClientSession: - raise NotImplementedError - - @property - @abstractmethod - def version(self) -> str: - raise NotImplementedError - - @abstractmethod - async def create( - self, - assets: Optional[List[pathlib.Path]] = None - ) -> Dict[str, Union[List[Dict[str, Union[str, pathlib.Path]]], Dict]]: - """Create this release version and optionally upload provided assets""" - raise NotImplementedError - - @abstractmethod - async def delete(self) -> None: - """Delete this release version""" - raise NotImplementedError - - @abstractmethod - def fail(self, message: str) -> str: - raise NotImplementedError - - @abstractmethod - async def fetch( - self, - path: pathlib.Path, - asset_types: Optional[Dict[str, Pattern[str]]] = None, - append: Optional[bool] = False) -> Dict[str, List[Dict[str, Union[str, pathlib.Path]]]]: - """Fetch assets for this version, saving either to a directory or - tarball - """ - raise NotImplementedError - - @abstractmethod - async def get(self) -> Dict: - """Get the release information for this Github release.""" - raise NotImplementedError - - @abstractmethod - async def push( - self, artefacts: Iterable[pathlib.Path] - ) -> Dict[str, List[Dict[str, Union[str, pathlib.Path]]]]: - """Push assets from a list of paths, either directories or tarballs.""" - raise NotImplementedError - - @async_property(cache=True) - @abstractmethod - async def upload_url(self) -> str: - """Upload URL for this release version""" - raise NotImplementedError - - -class AGithubReleaseManager(metaclass=abstract.Abstraction): - """This utility wraps the github API to provide the ability to - create and manage releases and release assets. - - A github client connection and/or aiohttp session can be provided if you - wish to reuse the client or session. - - If you do not provide a session, one will be created and the async - `.close()` method should called after use. - - For this reason, instances of this class can be used as an async - contextmanager, and the session will be automatically closed on exit, for - example: - - ```python - - from tools.github.release.manager import GithubReleaseManager - - async with GithubReleaseManager(...) as manager: - await manager["1.19.0"].create() - ``` - """ - - @abstractmethod - async def __aenter__(self) -> "AGithubReleaseManager": - raise NotImplementedError - - @abstractmethod - async def __aexit__(self, *args) -> None: - raise NotImplementedError - - @abstractmethod - def __getitem__(self, version) -> AGithubRelease: - """Accessor for a specific Github release""" - raise NotImplementedError - - @cached_property - @abstractmethod - def github(self) -> gidgethub.abc.GitHubAPI: - """An instance of the gidgethub GitHubAPI""" - raise NotImplementedError - - @async_property - @abstractmethod - async def latest(self) -> Dict[str, packaging.version.Version]: - """Returns a dictionary of latest minor and patch versions - - For example, given the following versions: - - 1.19.2, 1.19.1, 1.20.3 - - It would return: - - 1.19 -> 1.19.2 - 1.19.1 -> 1.19.1 - 1.19.2 -> 1.19.2 - 1.20 -> 1.20.3 - 1.20.3 -> 1.20.3 - - """ - raise NotImplementedError - - @cached_property - @abstractmethod - def log(self) -> verboselogs.VerboseLogger: - """A verbose logger""" - raise NotImplementedError - - @async_property - @abstractmethod - async def releases(self) -> List[Dict]: - """List of dictionaries containing information about available releases, - as returned by the Github API - """ - raise NotImplementedError - - @cached_property - @abstractmethod - def releases_url(self) -> pathlib.PurePosixPath: - """Github API releases URL""" - raise NotImplementedError - - @cached_property - @abstractmethod - def session(self) -> aiohttp.ClientSession: - """Aiohttp Client session, also used for Github API client""" - raise NotImplementedError - - @abstractmethod - def fail(self, message: str) -> str: - """Either raise an error or log a warning and return the message, - dependent on the value of `self.continues`. - """ - raise NotImplementedError - - @abstractmethod - def format_version(self, version: Union[str, packaging.version.Version]) -> str: - """Formatted version name - eg `1.19.0` -> `v1.19.0`""" - raise NotImplementedError - - @abstractmethod - def parse_version(self, version: str) -> Optional[packaging.version.Version]: - """Parsed version - eg `v1.19.0` -> `Version(1.19.0)`""" - raise NotImplementedError diff --git a/tools/github/release/exceptions.py b/tools/github/release/exceptions.py deleted file mode 100644 index 796ee0011119e..0000000000000 --- a/tools/github/release/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class GithubReleaseError(Exception): - pass diff --git a/tools/github/release/manager.py b/tools/github/release/manager.py deleted file mode 100644 index 7c465e421d5cf..0000000000000 --- a/tools/github/release/manager.py +++ /dev/null @@ -1,145 +0,0 @@ -import pathlib -import re -from functools import cached_property -from typing import Dict, List, Optional, Pattern, Type, Union - -import verboselogs # type:ignore - -import packaging.version - -import aiohttp - -import gidgethub.abc -import gidgethub.aiohttp - -from tools.base import abstract -from tools.base.functional import async_property - -from tools.github.release.abstract import AGithubRelease, AGithubReleaseManager -from tools.github.release.exceptions import GithubReleaseError -# from tools.github.release.release import GithubRelease - -VERSION_MIN = packaging.version.Version("0") - - -@abstract.implementer(AGithubReleaseManager) -class GithubReleaseManager: - - _version_re = r"v(\w+)" - _version_format = "v{version}" - - def __init__( - self, - path: Union[str, pathlib.Path], - repository: str, - continues: Optional[bool] = False, - create: Optional[bool] = True, - user: Optional[str] = None, - oauth_token: Optional[str] = None, - version: Optional[str] = None, - log: Optional[verboselogs.VerboseLogger] = None, - asset_types: Optional[Dict[str, Pattern[str]]] = None, - github: Optional[gidgethub.abc.GitHubAPI] = None, - session: Optional[aiohttp.ClientSession] = None) -> None: - self.version = version - self._path = path - self.repository = repository - self.continues = continues - self._log = log - self.oauth_token = oauth_token - self.user = user - self._asset_types = asset_types - self._github = github - self._session = session - self.create = create - - async def __aenter__(self) -> AGithubReleaseManager: - return self - - async def __aexit__(self, *args) -> None: - await self.close() - - def __getitem__(self, version) -> AGithubRelease: - # return self.release_class(self, version) - raise NotImplementedError - - @property - def release_class(self) -> Type[AGithubRelease]: - # return GithubRelease - raise NotImplementedError - - @cached_property - def github(self) -> gidgethub.abc.GitHubAPI: - return ( - self._github - or gidgethub.aiohttp.GitHubAPI(self.session, self.user, oauth_token=self.oauth_token)) - - @cached_property - def log(self) -> verboselogs.VerboseLogger: - return self._log or verboselogs.VerboseLogger(__name__) - - @cached_property - def path(self) -> pathlib.Path: - return pathlib.Path(self._path) - - @async_property - async def latest(self) -> Dict[str, packaging.version.Version]: - latest = {} - for release in await self.releases: - version = self.parse_version(release["tag_name"]) - if not version: - continue - latest[str(version)] = version - minor = f"{version.major}.{version.minor}" - if version > latest.get(minor, self.version_min): - latest[minor] = version - return latest - - @async_property - async def releases(self) -> List[Dict]: - results = [] - # By iterating the results from the releases url here, - # gidgethub will paginate the release information. - async for result in self.github.getiter(str(self.releases_url)): - results.append(result) - return results - - @cached_property - def releases_url(self) -> pathlib.PurePosixPath: - return pathlib.PurePosixPath(f"/repos/{self.repository}/releases") - - @cached_property - def session(self) -> aiohttp.ClientSession: - return self._session or aiohttp.ClientSession() - - @property - def version_min(self) -> packaging.version.Version: - return VERSION_MIN - - @cached_property - def version_re(self) -> Pattern[str]: - return re.compile(self._version_re) - - async def close(self) -> None: - if not "session" in self.__dict__: - return - await self.session.close() - del self.__dict__["session"] - - def fail(self, message: str) -> str: - if not self.continues: - raise GithubReleaseError(message) - self.log.warning(message) - return message - - def format_version(self, version: Union[str, packaging.version.Version]) -> str: - return self._version_format.format(version=version) - - def parse_version(self, version: str) -> Optional[packaging.version.Version]: - parsed_version = self.version_re.sub(r"\1", version) - if parsed_version: - try: - return packaging.version.Version(parsed_version) - except packaging.version.InvalidVersion: - pass - self.log.warning(f"Unable to parse version: {version}") diff --git a/tools/github/release/release.py b/tools/github/release/release.py deleted file mode 100644 index 8963af080f316..0000000000000 --- a/tools/github/release/release.py +++ /dev/null @@ -1,196 +0,0 @@ -import pathlib -from functools import cached_property -from typing import Dict, Iterable, List, Optional, Pattern, Set, Tuple, Type, Union - -import verboselogs # type:ignore - -import aiohttp - -import gidgethub.abc -import gidgethub.aiohttp - -from tools.base import abstract, aio -from tools.base.functional import async_property - -from tools.github.release.abstract import ( - AGithubRelease, AGithubReleaseAssetsFetcher, AGithubReleaseAssetsPusher, AGithubReleaseManager) -# from tools.github.release.assets import GithubReleaseAssetsFetcher, GithubReleaseAssetsPusher -from tools.github.release.exceptions import GithubReleaseError - - -@abstract.implementer(AGithubRelease) -class GithubRelease: - file_exts = {"deb", "changes", "rpm"} - - def __init__(self, manager: AGithubReleaseManager, version: str): - self.manager = manager - self._version = version - - @async_property(cache=True) - async def asset_names(self) -> Set[str]: - """Set of the names of assets for this release version""" - return set(asset["name"] for asset in await self.assets) - - @async_property(cache=True) - async def assets(self) -> Dict: - """Assets dictionary as returned by Github Release API""" - try: - return await self.github.getitem(await self.assets_url) - except gidgethub.GitHubException as e: - raise GithubReleaseError(e) - - @async_property(cache=True) - async def assets_url(self) -> str: - """URL for retrieving this version's assets information from""" - return (await self.release)["assets_url"] - - @async_property(cache=True) - async def delete_url(self) -> pathlib.PurePosixPath: - """Github API-relative URL for deleting this release version""" - return self.releases_url.joinpath(str(await self.release_id)) - - @async_property - async def exists(self) -> bool: - return self.version_name in await self.release_names - - @property - def fetcher(self) -> Type[AGithubReleaseAssetsFetcher]: - # return GithubReleaseAssetsFetcher - raise NotImplementedError - - @property - def github(self) -> gidgethub.abc.GitHubAPI: - return self.manager.github - - @property - def log(self) -> verboselogs.VerboseLogger: - return self.manager.log - - @property - def pusher(self) -> Type[AGithubReleaseAssetsPusher]: - # return GithubReleaseAssetsPusher - raise NotImplementedError - - @async_property(cache=True) - async def release(self) -> Dict: - """Dictionary of release version information as returned by the Github Release API""" - return await self.get() - - @async_property(cache=True) - async def release_id(self) -> int: - """The Github release ID for this version, required for some URLs""" - return (await self.release)["id"] - - @async_property - async def release_names(self) -> Tuple[str, ...]: - """Tuple of release tag names as returned by the Github Release API - - This is used to check whether the release exists already. - """ - return tuple(release["tag_name"] for release in await self.manager.releases) - - @property - def releases_url(self) -> pathlib.PurePosixPath: - return self.manager.releases_url - - @property - def session(self) -> aiohttp.ClientSession: - return self.manager.session - - @async_property(cache=True) - async def upload_url(self) -> str: - """Upload URL for this release version""" - return (await self.release)["upload_url"].split("{")[0] - - @property - def version(self) -> str: - return self._version - - @property - def version_name(self) -> str: - return self.manager.format_version(self.version) - - @cached_property - def version_url(self) -> pathlib.PurePosixPath: - """Github API-relative URL to retrieve release version information from""" - return self.releases_url.joinpath("tags", self.version_name) - - async def create( - self, - assets: Optional[List[pathlib.Path]] = None - ) -> Dict[str, Union[List[Dict[str, Union[str, pathlib.Path]]], Dict]]: - results: Dict[str, Union[List[Dict], Dict]] = {} - if await self.exists: - self.fail(f"Release {self.version_name} already exists") - else: - self.log.notice(f"Creating release {self.version}") - try: - results["release"] = await self.github.post( - str(self.releases_url), data=dict(tag_name=self.version_name)) - except gidgethub.GitHubException as e: - raise GithubReleaseError(e) - self.log.success(f"Release created {self.version}") - if assets: - results.update(await self.push(assets)) - return results - - async def delete(self) -> None: - if not await self.exists: - raise GithubReleaseError( - f"Unable to delete version {self.version_name} as it does not exist") - self.log.notice(f"Deleting release version: {self.version_name}") - try: - await self.github.delete(str(await self.delete_url)) - except gidgethub.GitHubException as e: - raise GithubReleaseError(e) - self.log.success(f"Release version deleted: {self.version_name}") - - async def fetch( - self, - path: pathlib.Path, - asset_types: Optional[Dict[str, Pattern[str]]] = None, - append: Optional[bool] = False) -> Dict[str, List[Dict[str, Union[str, pathlib.Path]]]]: - self.log.notice(f"Downloading assets for release version: {self.version_name} -> {path}") - assets: List[Dict[str, Union[str, pathlib.Path]]] = [] - errors: List[Dict[str, Union[str, pathlib.Path]]] = [] - response = dict(assets=assets, errors=errors) - async for result in self.fetcher(self, path, asset_types, append=append): - if result.get("error"): - response["errors"].append(result) - continue - response["assets"].append(result) - self.log.info(f"Asset saved: {result['name']} -> {result['outfile']}") - if not response["errors"]: - self.log.success( - f"Assets downloaded for release version: {self.version_name} -> {path}") - return response - - def fail(self, message: str) -> str: - return self.manager.fail(message) - - async def get(self) -> Dict: - try: - return await self.github.getitem(str(self.version_url)) - except gidgethub.GitHubException as e: - raise GithubReleaseError(e) - - async def push( - self, artefacts: Iterable[pathlib.Path] - ) -> Dict[str, List[Dict[str, Union[str, pathlib.Path]]]]: - self.log.notice(f"Pushing assets for {self.version}") - assets: List[Dict[str, Union[str, pathlib.Path]]] = [] - errors: List[Dict[str, Union[str, pathlib.Path]]] = [] - response = dict(assets=assets, errors=errors) - try: - for path in artefacts: - async for result in self.pusher(self, path): - if result.get("error"): - response["errors"].append(result) - continue - response["assets"].append(result) - self.log.info(f"Release file uploaded {result['name']}") - except aio.ConcurrentError as e: - raise e.args[0] - if not response["errors"]: - self.log.success(f"Assets uploaded: {self.version}") - return response diff --git a/tools/github/release/tests/test_manager.py b/tools/github/release/tests/test_manager.py deleted file mode 100644 index d0ae588529284..0000000000000 --- a/tools/github/release/tests/test_manager.py +++ /dev/null @@ -1,358 +0,0 @@ - -from unittest.mock import AsyncMock, MagicMock, PropertyMock - -import pytest - -import packaging.version - -from tools.base.functional import async_property -from tools.github.release import manager, exceptions as github_errors - - -@pytest.mark.parametrize("continues", [None, True, False]) -@pytest.mark.parametrize("create", [None, True, False]) -@pytest.mark.parametrize("user", [None, "USER"]) -@pytest.mark.parametrize("oauth_token", [None, "OAUTH TOKEN"]) -@pytest.mark.parametrize("log", [None, "LOG"]) -@pytest.mark.parametrize("asset_types", [None, "ASSET TYPES"]) -@pytest.mark.parametrize("github", [None, "GITHUB"]) -@pytest.mark.parametrize("session", [None, "SESSION"]) -def test_release_manager_constructor(continues, create, user, oauth_token, log, asset_types, github, session): - kwargs = dict( - continues=continues, - create=create, - user=user, - oauth_token=oauth_token, - log=log, - asset_types=asset_types, - github=github, - session=session) - kwargs = {k: v for k, v in kwargs.items() if v is not None} - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY", **kwargs) - assert releaser._path == "PATH" - assert releaser.repository == "REPOSITORY" - assert releaser.continues == (continues if continues is not None else False) - assert releaser.create == (create if create is not None else True) - assert releaser._log == log - assert releaser.oauth_token == oauth_token - assert releaser.user == user - assert releaser._asset_types == asset_types - assert releaser._github == github - assert releaser._session == session - - assert releaser._version_re == r"v(\w+)" - assert ( - releaser.version_min - == manager.VERSION_MIN - == packaging.version.Version("0")) - assert "version_min" not in releaser.__dict__ - - -@pytest.mark.asyncio -async def test_release_manager_async_contextmanager(patches): - patched = patches( - ("GithubReleaseManager.close", dict(new_callable=AsyncMock)), - prefix="tools.github.release.manager") - - with patched as (m_close, ): - async with manager.GithubReleaseManager("PATH", "REPOSITORY") as releaser: - assert isinstance(releaser, manager.GithubReleaseManager) - assert not m_close.called - assert ( - list(m_close.call_args) - == [(), {}]) - - -def test_release_manager_dunder_getitem(): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - - with pytest.raises(NotImplementedError): - releaser["X.Y.Z"] - - -@pytest.mark.parametrize("oauth_token", [None, "OAUTH_TOKEN"]) -@pytest.mark.parametrize("user", [None, "USER"]) -@pytest.mark.parametrize("github", [True, False]) -def test_release_manager_github(patches, oauth_token, user, github): - kwargs = {} - if oauth_token: - kwargs["oauth_token"] = oauth_token - if user: - kwargs["user"] = user - if github: - kwargs["github"] = "GITHUB" - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY", **kwargs) - patched = patches( - "gidgethub", - ("GithubReleaseManager.session", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - - with patched as (m_api, m_session): - assert ( - releaser.github - == (m_api.aiohttp.GitHubAPI.return_value - if not github - else "GITHUB")) - - assert "github" in releaser.__dict__ - if github: - assert not m_api.aiohttp.GitHubAPI.called - return - assert ( - list(m_api.aiohttp.GitHubAPI.call_args) - == [(m_session.return_value, user), - {'oauth_token': oauth_token}]) - - -@pytest.mark.parametrize("log", [True, False]) -def test_release_manager_log(patches, log): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "verboselogs", - prefix="tools.github.release.manager") - if log: - releaser._log = "LOG" - - with patched as (m_log, ): - assert ( - releaser.log - == (m_log.VerboseLogger.return_value - if not log - else "LOG")) - - assert "log" in releaser.__dict__ - - if log: - assert not m_log.VerboseLogger.called - return - - assert ( - list(m_log.VerboseLogger.call_args) - == [('tools.github.release.manager',), {}]) - - -def test_release_manager_path(patches): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "pathlib", - prefix="tools.github.release.manager") - - with patched as (m_plib, ): - assert ( - releaser.path - == m_plib.Path.return_value) - - assert "path" in releaser.__dict__ - assert ( - list(m_plib.Path.call_args) - == [('PATH',), {}]) - - -@pytest.mark.asyncio -async def test_release_manager_latest(patches): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "GithubReleaseManager.parse_version", - ("GithubReleaseManager.releases", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - - versions = [dict(tag_name=v) for v in ("1.19.2", "X", "1.19.1", "Y_Z", "1.20.3", "", "0.0.1")] - - with patched as (m_version, m_releases): - m_version.side_effect = lambda version: (packaging.version.Version(version) if "." in version else None) - m_releases.side_effect = AsyncMock(return_value=versions) - result = await releaser.latest - - assert ( - result - == {'0.0.1': packaging.version.Version('0.0.1'), - '0.0': packaging.version.Version('0.0.1'), - '1.19.2': packaging.version.Version('1.19.2'), - '1.19': packaging.version.Version('1.19.2'), - '1.19.1': packaging.version.Version('1.19.1'), - '1.20.3': packaging.version.Version('1.20.3'), - '1.20': packaging.version.Version('1.20.3')}) - assert not hasattr(releaser, "__async_prop_cache__") - - -@pytest.mark.asyncio -async def test_release_manager_releases(patches): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - ("GithubReleaseManager.github", dict(new_callable=PropertyMock)), - ("GithubReleaseManager.releases_url", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - getiter_mock = MagicMock() - - async def getiter(url): - getiter_mock(url) - for i in range(0, 5): - yield i - - with patched as (m_github, m_releases): - m_github.return_value.getiter = getiter - assert await releaser.releases == list(range(0, 5)) - - assert ( - list(getiter_mock.call_args) - == [(str(m_releases.return_value), ), {}]) - assert not hasattr(releaser, async_property.cache_name) - - -def test_release_manager_releases_url(patches): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "pathlib", - prefix="tools.github.release.manager") - - with patched as (m_plib, ): - assert releaser.releases_url == m_plib.PurePosixPath.return_value - - assert ( - list(m_plib.PurePosixPath.call_args) - == [(f"/repos/REPOSITORY/releases", ), {}]) - assert "releases_url" in releaser.__dict__ - - -@pytest.mark.parametrize("session", [True, False]) -def test_release_manager_session(patches, session): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "aiohttp", - prefix="tools.github.release.manager") - if session: - releaser._session = "SESSION" - - with patched as (m_http, ): - assert ( - releaser.session - == (m_http.ClientSession.return_value - if not session - else "SESSION")) - - assert "session" in releaser.__dict__ - if session: - assert not m_http.ClientSession.called - return - assert ( - list(m_http.ClientSession.call_args) - == [(), {}]) - - -def test_release_manager_version_re(patches): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "re", - prefix="tools.github.release.manager") - releaser._version_re = "VERSION RE" - - with patched as (m_re, ): - assert releaser.version_re == m_re.compile.return_value - - assert ( - list(m_re.compile.call_args) - == [("VERSION RE", ), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("session", [True, False]) -async def test_release_manager_close(patches, session): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - ("GithubReleaseManager.session", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - - if session: - releaser.__dict__["session"] = "SESSION" - - with patched as (m_session, ): - m_session.return_value.close = AsyncMock() - assert not await releaser.close() - - assert "session" not in releaser.__dict__ - - if not session: - assert not m_session.called - return - - assert ( - list(m_session.return_value.close.call_args) - == [(), {}]) - - -@pytest.mark.parametrize("continues", [True, False]) -def test_release_manager_fail(patches, continues): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY", continues=continues) - parser = MagicMock() - patched = patches( - ("GithubReleaseManager.log", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - - with patched as (m_log, ): - if continues: - assert ( - releaser.fail("MESSAGE") - == "MESSAGE") - else: - with pytest.raises(github_errors.GithubReleaseError): - releaser.fail("MESSAGE") - - if not continues: - assert not m_log.return_value.warning.called - return - - assert ( - list(m_log.return_value.warning.call_args) - == [("MESSAGE", ), {}]) - - -def test_release_manager_format_version(): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - releaser._version_format = MagicMock() - assert releaser.format_version("VERSION") == releaser._version_format.format.return_value - assert ( - list(releaser._version_format.format.call_args) - == [(), dict(version="VERSION")]) - - -@pytest.mark.parametrize("version", [None, 0, "", "1.2.3"]) -@pytest.mark.parametrize("raises", [None, BaseException, packaging.version.InvalidVersion]) -def test_release_manager_parse_version(patches, version, raises): - releaser = manager.GithubReleaseManager("PATH", "REPOSITORY") - patched = patches( - "packaging.version.Version", - ("GithubReleaseManager.log", dict(new_callable=PropertyMock)), - ("GithubReleaseManager.version_re", dict(new_callable=PropertyMock)), - prefix="tools.github.release.manager") - - with patched as (m_packaging, m_log, m_version): - m_version.return_value.sub.return_value = version - if raises: - m_packaging.side_effect = raises() - - if version and raises == BaseException: - with pytest.raises(BaseException): - releaser.parse_version("VERSION") - else: - assert ( - releaser.parse_version("VERSION") - == (None - if not version or raises - else m_packaging.return_value)) - - assert ( - list(m_version.return_value.sub.call_args) - == [(r"\1", "VERSION"), {}]) - if version: - assert ( - list(m_packaging.call_args) - == [(m_version.return_value.sub.return_value, ), {}]) - else: - assert not m_packaging.called - - if not version or raises and raises != BaseException: - assert ( - list(m_log.return_value.warning.call_args) - == [("Unable to parse version: VERSION", ), {}]) - else: - assert not m_log.called diff --git a/tools/github/release/tests/test_release.py b/tools/github/release/tests/test_release.py deleted file mode 100644 index 8edfee419039e..0000000000000 --- a/tools/github/release/tests/test_release.py +++ /dev/null @@ -1,525 +0,0 @@ - -from unittest.mock import AsyncMock, MagicMock, PropertyMock - -import pytest - -import gidgethub - -from tools.base import aio -from tools.base.functional import async_property -from tools.github.release import exceptions as github_errors, release as github_release - - -def test_release_constructor(): - release = github_release.GithubRelease("MANAGER", "VERSION") - assert release.manager == "MANAGER" - assert release.version == "VERSION" - - # assert release.fetcher == github_release.GithubReleaseAssetsFetcher - # assert "fetcher" not in release.__dict__ - # assert release.pusher == github_release.GithubReleaseAssetsPusher - # assert "pusher" not in release.__dict__ - - -def _check_manager_property(prop, arg=None): - manager = MagicMock() - checker = github_release.GithubRelease(manager, "VERSION") - assert getattr(checker, prop) == getattr(manager, arg or prop) - assert prop not in checker.__dict__ - - -@pytest.mark.parametrize( - "prop", - [("github",), - ("log",), - ("releases_url",), - ("session",)]) -def test_release_manager_props(prop): - _check_manager_property(*prop) - - -@pytest.mark.asyncio -async def test_release_asset_names(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.assets", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - assets = [MagicMock(), MagicMock()] - - with patched as (m_assets, ): - m_assets.side_effect = AsyncMock(return_value=assets) - assert ( - await release.asset_names - == set(m.__getitem__.return_value for m in assets)) - - for asset in assets: - assert ( - list(asset.__getitem__.call_args) - == [('name',), {}]) - - assert "asset_names" in release.__async_prop_cache__ - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [None, BaseException, gidgethub.GitHubException]) -async def test_release_assets(patches, raises): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.assets_url", dict(new_callable=PropertyMock)), - ("GithubRelease.github", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_url, m_github): - get = AsyncMock() - url = AsyncMock() - m_github.return_value.getitem.side_effect = get - m_url.side_effect = url - if raises: - get.side_effect = raises("AN ERROR OCCURRED") - exception = ( - github_errors.GithubReleaseError - if raises == gidgethub.GitHubException - else raises) - with pytest.raises(exception): - await release.assets - else: - assert ( - await release.assets - == get.return_value) - - assert ( - list(get.call_args) - == [(url.return_value,), {}]) - if not raises: - assert "assets" in release.__async_prop_cache__ - - -@pytest.mark.asyncio -async def test_release_assets_url(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.release", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_release, ): - mock_release = AsyncMock() - m_release.side_effect = mock_release - assert ( - await release.assets_url - == mock_release.return_value.__getitem__.return_value) - - assert ( - list(mock_release.return_value.__getitem__.call_args) - == [('assets_url',), {}]) - assert "assets_url" in getattr(release, async_property.cache_name) - - -@pytest.mark.asyncio -async def test_release_delete_url(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.releases_url", dict(new_callable=PropertyMock)), - ("GithubRelease.release_id", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_url, m_id): - mock_id = AsyncMock() - m_id.side_effect = mock_id - assert ( - await release.delete_url - == m_url.return_value.joinpath.return_value) - - assert ( - list(m_url.return_value.joinpath.call_args) - == [(str(mock_id.return_value), ), {}]) - assert "delete_url" in getattr(release, async_property.cache_name) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("version", [f"VERSION{i}" for i in range(0, 7)]) -async def test_release_exists(patches, version): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.release_names", dict(new_callable=PropertyMock)), - ("GithubRelease.version_name", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - versions = [f"VERSION{i}" for i in range(3, 5)] - - with patched as (m_release, m_name): - m_name.return_value = version - m_release.side_effect = AsyncMock(return_value=versions) - assert await release.exists == (version in versions) - - assert not hasattr(release, async_property.cache_name) - - -@pytest.mark.asyncio -async def test_release_release(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.get", dict(new_callable=AsyncMock)), - prefix="tools.github.release.release") - - with patched as (m_get, ): - assert await release.release == m_get.return_value - - assert "release" in getattr(release, async_property.cache_name) - - -@pytest.mark.asyncio -async def test_release_release_id(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.release", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_release, ): - mock_release = AsyncMock() - m_release.side_effect = mock_release - assert await release.release_id == mock_release.return_value.__getitem__.return_value - - assert ( - list(mock_release.return_value.__getitem__.call_args) - == [('id',), {}]) - assert "release_id" in getattr(release, async_property.cache_name) - - -@pytest.mark.asyncio -async def test_release_release_names(patches): - manager = MagicMock() - - release_names = [dict(tag_name=f"TAG{i}") for i in range(0, 3)] - - async def releases_fun(): - return release_names - - manager.releases = releases_fun() - release = github_release.GithubRelease(manager, "VERSION") - assert ( - await release.release_names - == tuple(t["tag_name"] for t in release_names)) - - -@pytest.mark.asyncio -async def test_release_upload_url(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.release", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_release, ): - mock_release = AsyncMock() - m_release.side_effect = mock_release - assert ( - await release.upload_url - == mock_release.return_value.__getitem__.return_value.split.return_value.__getitem__.return_value) - - assert ( - list(mock_release.return_value.__getitem__.call_args) - == [('upload_url',), {}]) - assert ( - list(mock_release.return_value.__getitem__.return_value.split.call_args) - == [('{',), {}]) - assert ( - list(mock_release.return_value.__getitem__.return_value.split.return_value.__getitem__.call_args) - == [(0,), {}]) - assert "upload_url" in release.__async_prop_cache__ - - -def test_release_version_name(patches): - manager = MagicMock() - release = github_release.GithubRelease(manager, "VERSION") - release.version_name == manager.format_version.return_value - assert ( - list(manager.format_version.call_args) - == [("VERSION",), {}]) - - -def test_release_version_url(patches): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.releases_url", dict(new_callable=PropertyMock)), - ("GithubRelease.version_name", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_releases, m_version): - assert release.version_url == m_releases.return_value.joinpath.return_value - - assert ( - list(m_releases.return_value.joinpath.call_args) - == [("tags", m_version.return_value), {}]) - assert "version_url" in release.__dict__ - - -@pytest.mark.asyncio -@pytest.mark.parametrize("exists", [True, False]) -@pytest.mark.parametrize("assets", [None, [], [f"ASSET{i}" for i in range(0, 3)]]) -@pytest.mark.parametrize("raises", [None, BaseException, gidgethub.GitHubException]) -async def test_release_create(patches, exists, assets, raises): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - "GithubRelease.fail", - ("GithubRelease.push", dict(new_callable=AsyncMock)), - ("GithubRelease.exists", dict(new_callable=PropertyMock)), - ("GithubRelease.github", dict(new_callable=PropertyMock)), - ("GithubRelease.log", dict(new_callable=PropertyMock)), - ("GithubRelease.releases_url", dict(new_callable=PropertyMock)), - ("GithubRelease.version_name", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - args = ( - (assets, ) - if assets is not None - else ()) - - with patched as (m_fail, m_push, m_exists, m_github, m_log, m_url, m_version): - m_exists.side_effect = AsyncMock(return_value=exists) - m_push.return_value = dict(PUSHED=True) - m_github.return_value.post = AsyncMock() - if raises: - m_github.return_value.post.side_effect = raises("AN ERROR OCCURRED") - if raises and not exists: - exception = ( - github_errors.GithubReleaseError - if raises == gidgethub.GitHubException - else raises) - with pytest.raises(exception): - await release.create(*args) - else: - result = await release.create(*args) - - expected = {} - if not exists: - assert ( - list(m_log.return_value.notice.call_args) - == [(f"Creating release VERSION", ), {}]) - assert ( - list(m_github.return_value.post.call_args) - == [(str(m_url.return_value), ), dict(data=dict(tag_name=m_version.return_value))]) - assert not m_fail.called - if not raises: - expected["release"] = m_github.return_value.post.return_value - assert ( - list(m_log.return_value.success.call_args) - == [(f"Release created VERSION", ), {}]) - else: - assert not m_log.return_value.success.called - else: - assert not m_github.return_value.post.called - assert not m_log.called - assert ( - list(m_fail.call_args) - == [(f"Release {m_version.return_value} already exists", ), {}]) - - if not exists and raises: - assert not m_push.called - return - if assets: - expected["PUSHED"] = True - assert ( - list(m_push.call_args) - == [(assets, ), {}]) - else: - assert not m_push.called - assert result == expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize("exists", [True, False]) -@pytest.mark.parametrize("raises", [None, BaseException, gidgethub.GitHubException]) -async def test_release_delete(patches, exists, raises): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.delete_url", dict(new_callable=PropertyMock)), - ("GithubRelease.exists", dict(new_callable=PropertyMock)), - ("GithubRelease.github", dict(new_callable=PropertyMock)), - ("GithubRelease.log", dict(new_callable=PropertyMock)), - ("GithubRelease.version_name", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_url, m_exists, m_github, m_log, m_version): - url = AsyncMock() - m_url.side_effect = url - m_exists.side_effect = AsyncMock(return_value=exists) - m_github.return_value.delete = AsyncMock() - if raises: - m_github.return_value.delete.side_effect = raises("AN ERROR OCCURRED") - - if exists and not raises: - assert not await release.delete() - elif raises == BaseException: - with pytest.raises(BaseException) as e: - await release.delete() - else: - with pytest.raises(github_errors.GithubReleaseError) as e: - await release.delete() - - if not exists: - assert ( - e.value.args[0] - == f"Unable to delete version {m_version.return_value} as it does not exist") - assert not m_log.called - assert not m_github.called - return - assert ( - list(m_log.return_value.notice.call_args) - == [(f"Deleting release version: {m_version.return_value}", ), {}]) - assert ( - list(m_github.return_value.delete.call_args) - == [(str(url.return_value), ), {}]) - if raises: - assert not m_log.return_value.success.called - return - assert ( - list(m_log.return_value.success.call_args) - == [(f"Release version deleted: {m_version.return_value}", ), {}]) - - -def test_release_fail(): - manager = MagicMock() - release = github_release.GithubRelease(manager, "VERSION") - assert release.fail("FAILURE") == manager.fail.return_value - assert ( - list(manager.fail.call_args) - == [("FAILURE", ), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("asset_types", [None, (), tuple(f"ASSET_TYPE{i}" for i in range(0, 3))]) -@pytest.mark.parametrize("errors", [[], [0], [2, 4], range(0, 5)]) -async def test_release_fetch(patches, asset_types, errors): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.fetcher", dict(new_callable=PropertyMock)), - ("GithubRelease.log", dict(new_callable=PropertyMock)), - ("GithubRelease.version_name", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - kwargs = {} if asset_types is None else dict(asset_types=asset_types) - fetched = MagicMock() - - async def fetcher(_releaser, path, asset_types, append=False): - fetched(_releaser, path, asset_types, append) - for x in range(0, 5): - response = dict(name=f"FETCHED{x}") - if x in errors: - response["error"] = f"ERROR{x}" - else: - response["outfile"] = f"OUTFILE{x}" - yield response - expected = dict( - errors=[ - dict(name=f"FETCHED{i}", error=f"ERROR{i}") - for i in errors], - assets=[ - dict(name=f"FETCHED{i}", outfile=f"OUTFILE{i}") - for i in range(0, 5) if i not in errors]) - - with patched as (m_fetcher, m_log, m_version): - m_fetcher.return_value = fetcher - assert await release.fetch("PATH", **kwargs) == expected - - assert ( - list(m_log.return_value.notice.call_args) - == [(f"Downloading assets for release version: {m_version.return_value} -> PATH", ), {}]) - assert ( - list(fetched.call_args) - == [(release, 'PATH', asset_types, False), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [None, BaseException, gidgethub.GitHubException]) -async def test_release_get(patches, raises): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.version_url", dict(new_callable=PropertyMock)), - ("GithubRelease.github", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - - with patched as (m_url, m_github): - m_github.return_value.getitem = AsyncMock() - if raises: - m_github.return_value.getitem.side_effect = raises("AN ERROR OCCURRED") - exception = ( - github_errors.GithubReleaseError - if raises == gidgethub.GitHubException - else raises) - with pytest.raises(exception): - await release.get() - else: - assert await release.get() == m_github.return_value.getitem.return_value - assert ( - list(m_github.return_value.getitem.call_args) - == [(str(m_url.return_value), ), {}]) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [None, BaseException, aio.ConcurrentError]) -@pytest.mark.parametrize("errors", [[], [1, 3], range(0, 5)]) -async def test_release_push(patches, raises, errors): - release = github_release.GithubRelease("MANAGER", "VERSION") - patched = patches( - ("GithubRelease.log", dict(new_callable=PropertyMock)), - ("GithubRelease.pusher", dict(new_callable=PropertyMock)), - prefix="tools.github.release.release") - artefacts = [f"ARTEFACTS{i}" for i in range(0, 5)] - expected = dict(assets=[], errors=[]) - - for i in range(0, 5): - for x in range(0, 5): - result = dict( - name=f"ARTEFACTS{i}_ASSET{x}", - foo=f"ARTEFACTS{i}_BAR{x}") - if x in errors: - result["error"] = f"GOT AN ERROR ARTEFACTS{i} {x}" - expected["errors"].append(result) - else: - expected["assets"].append(result) - - - class SomeError(Exception): - pass - - async def pusher(path): - if raises: - raise raises(SomeError("AN ERROR OCCURRED")) - for i in range(0, 5): - response = dict( - name=f"{path}_ASSET{i}", - foo=f"{path}_BAR{i}") - if i in errors: - response["error"] = f"GOT AN ERROR {path} {i}" - yield response - - with patched as (m_log, m_pusher): - m_pusher.return_value.side_effect = lambda _self, path: pusher(path) - if raises: - with pytest.raises(BaseException if raises == BaseException else SomeError): - await release.push(artefacts) - else: - assert await release.push(artefacts) == expected - - assert ( - list(m_log.return_value.notice.call_args) - == [(f"Pushing assets for VERSION", ), {}]) - - if raises: - assert ( - list(list(c) for c in m_pusher.return_value.call_args_list) - == [[(release, 'ARTEFACTS0'), {}]]) - assert not m_log.return_value.info.called - else: - assert ( - list(list(c) for c in m_pusher.return_value.call_args_list) - == [[(release, f'ARTEFACTS{x}'), {}] for x in range(0, 5)]) - - if raises or errors: - assert not m_log.return_value.success.called - else: - assert ( - list(m_log.return_value.success.call_args) - == [(f"Assets uploaded: VERSION", ), {}]) - assert ( - list(list(c) for c in m_log.return_value.info.call_args_list) - == [[(f'Release file uploaded ARTEFACTS{i}_ASSET{x}',), {}] - for i in range(0, 5) - for x in range(0, 5)]) diff --git a/tools/gpg/BUILD b/tools/gpg/BUILD deleted file mode 100644 index 50a3dd91ff14e..0000000000000 --- a/tools/gpg/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -load("//bazel:envoy_build_system.bzl", "envoy_package") -load("@gpg_pip3//:requirements.bzl", "requirement") -load("//tools/base:envoy_python.bzl", "envoy_py_library") - -licenses(["notice"]) # Apache 2 - -envoy_package() - -envoy_py_library( - name = "tools.gpg.identity", - deps = [requirement("python-gnupg")], -) diff --git a/tools/gpg/identity.py b/tools/gpg/identity.py deleted file mode 100644 index 6d9c99607fa2f..0000000000000 --- a/tools/gpg/identity.py +++ /dev/null @@ -1,151 +0,0 @@ -import logging -import os -import pathlib -import pwd -import shutil -from functools import cached_property -from email.utils import formataddr, parseaddr -from typing import Iterable, Optional - -import gnupg # type:ignore - - -class GPGError(Exception): - pass - - -class GPGIdentity(object): - """A GPG identity with a signing key - - The signing key is found either by matching provided name/email, - or by retrieving the first private key. - """ - - def __init__( - self, - name: Optional[str] = None, - email: Optional[str] = None, - log: Optional[logging.Logger] = None): - self._provided_name = name - self._provided_email = email - self._log = log - - def __str__(self) -> str: - return self.uid - - @cached_property - def email(self) -> str: - """Email parsed from the signing key""" - return parseaddr(self.uid)[1] - - @property - def fingerprint(self) -> str: - """GPG key fingerprint""" - return self.signing_key["fingerprint"] - - @cached_property - def gpg(self) -> gnupg.GPG: - return gnupg.GPG() - - @cached_property - def gpg_bin(self) -> Optional[pathlib.Path]: - gpg_bin = shutil.which("gpg2") or shutil.which("gpg") - return pathlib.Path(gpg_bin) if gpg_bin else None - - @property - def gnupg_home(self) -> pathlib.Path: - return self.home.joinpath(".gnupg") - - @cached_property - def home(self) -> pathlib.Path: - """Gets *and sets if required* the `HOME` env var""" - home_dir = os.environ.get("HOME", pwd.getpwuid(os.getuid()).pw_dir) - os.environ["HOME"] = home_dir - return pathlib.Path(home_dir) - - @cached_property - def log(self) -> logging.Logger: - return self._log or logging.getLogger(self.__class__.__name__) - - @property - def provided_email(self) -> str: - """Provided email for the identity""" - return self._provided_email or "" - - @cached_property - def provided_id(self) -> Optional[str]: - """Provided name and/or email for the identity""" - if not (self.provided_name or self.provided_email): - return None - return ( - formataddr((self.provided_name, self.provided_email)) if - (self.provided_name and self.provided_email) else - (self.provided_name or self.provided_email)) - - @property - def provided_name(self) -> Optional[str]: - """Provided name for the identity""" - return self._provided_name - - @cached_property - def name(self) -> str: - """Name parsed from the signing key""" - return parseaddr(self.uid)[0] - - @cached_property - def signing_key(self) -> dict: - """A `dict` representing the GPG key to sign with""" - # if name and/or email are provided the list of keys is pre-filtered - # but we still need to figure out which uid matched for the found key - for key in self.gpg.list_keys(True, keys=self.provided_id): - key = self.match(key) - if key: - return key - raise GPGError( - f"No key found for '{self.provided_id}'" if self.provided_id else "No available key") - - @property - def uid(self) -> str: - """UID of the identity's signing key""" - return self.signing_key["uid"] - - def match(self, key: dict) -> Optional[dict]: - """Match a signing key - - The key is found either by matching provided name/email - or the first available private key - - the matching `uid` (or first) is added as `uid` to the dict - """ - if self.provided_id: - key["uid"] = self._match_key(key["uids"]) - return key if key["uid"] else None - if self.log: - self.log.warning("No GPG name/email supplied, signing with first available key") - key["uid"] = key["uids"][0] - return key - - def _match_email(self, uids: Iterable) -> Optional[str]: - """Match only the email""" - for uid in uids: - if parseaddr(uid)[1] == self.provided_email: - return uid - - def _match_key(self, uids: Iterable) -> Optional[str]: - """If either/both name or email are supplied it tries to match either/both""" - if self.provided_name and self.provided_email: - return self._match_uid(uids) - elif self.provided_name: - return self._match_name(uids) - elif self.provided_email: - return self._match_email(uids) - - def _match_name(self, uids: Iterable) -> Optional[str]: - """Match only the name""" - for uid in uids: - if parseaddr(uid)[0] == self.provided_name: - return uid - - def _match_uid(self, uids: Iterable) -> Optional[str]: - """Match the whole uid - ie `Name `""" - return self.provided_id if self.provided_id in uids else None diff --git a/tools/gpg/requirements.txt b/tools/gpg/requirements.txt deleted file mode 100644 index f405a325f70be..0000000000000 --- a/tools/gpg/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --generate-hashes tools/gpg/requirements.txt -# -python-gnupg==0.4.7 \ - --hash=sha256:2061f56b1942c29b92727bf9aecbd3cea3893acc9cccbdc7eb4604285efe4ac7 \ - --hash=sha256:3ff5b1bf5e397de6e1fe41a7c0f403dad4e242ac92b345f440eaecfb72a7ebae - # via -r tools/gpg/requirements.txt diff --git a/tools/gpg/tests/test_identity.py b/tools/gpg/tests/test_identity.py deleted file mode 100644 index 0cd3086f5a6ee..0000000000000 --- a/tools/gpg/tests/test_identity.py +++ /dev/null @@ -1,442 +0,0 @@ -from unittest.mock import MagicMock, PropertyMock - -import pytest - -from tools.gpg import identity - - -@pytest.mark.parametrize("name", ["NAME", None]) -@pytest.mark.parametrize("email", ["EMAIL", None]) -@pytest.mark.parametrize("log", ["LOG", None]) -def test_identity_constructor(name, email, log): - gpg = identity.GPGIdentity(name, email, log) - assert gpg.provided_name == name - assert gpg.provided_email == (email or "") - assert gpg._log == log - - -def test_identity_dunder_str(patches): - gpg = identity.GPGIdentity() - patched = patches( - ("GPGIdentity.uid", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_uid, ): - m_uid.return_value = "SOME BODY" - assert str(gpg) == "SOME BODY" - - -def test_identity_email(patches): - gpg = identity.GPGIdentity() - patched = patches( - "parseaddr", - ("GPGIdentity.uid", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_parse, m_uid): - assert gpg.email == m_parse.return_value.__getitem__.return_value - - assert ( - list(m_parse.return_value.__getitem__.call_args) - == [(1,), {}]) - assert ( - list(m_parse.call_args) - == [(m_uid.return_value,), {}]) - assert "email" in gpg.__dict__ - - -def test_identity_fingerprint(patches): - gpg = identity.GPGIdentity() - patched = patches( - ("GPGIdentity.signing_key", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_key, ): - assert gpg.fingerprint == m_key.return_value.__getitem__.return_value - - assert ( - list(m_key.return_value.__getitem__.call_args) - == [('fingerprint',), {}]) - - assert "fingerprint" not in gpg.__dict__ - - -def test_identity_gpg(patches): - gpg = identity.GPGIdentity() - patched = patches( - "gnupg.GPG", - prefix="tools.gpg.identity") - - with patched as (m_gpg, ): - assert gpg.gpg == m_gpg.return_value - - assert ( - list(m_gpg.call_args) - == [(), {}]) - - assert "gpg" in gpg.__dict__ - - -def test_identity_gnupg_home(patches): - gpg = identity.GPGIdentity() - patched = patches( - ("GPGIdentity.home", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_home, ): - assert gpg.gnupg_home == m_home.return_value.joinpath.return_value - - assert ( - list(m_home.return_value.joinpath.call_args) - == [('.gnupg', ), {}]) - - assert "gnupg_home" not in gpg.__dict__ - - -@pytest.mark.parametrize("gpg1", [None, "GPG"]) -@pytest.mark.parametrize("gpg2", [None, "GPG2"]) -def test_identity_gpg_bin(patches, gpg1, gpg2): - gpg = identity.GPGIdentity() - patched = patches( - "pathlib", - "shutil", - prefix="tools.gpg.identity") - - def _get_bin(_cmd): - if _cmd == "gpg2" and gpg2: - return gpg2 - if _cmd == "gpg" and gpg1: - return gpg1 - - with patched as (m_plib, m_shutil): - m_shutil.which.side_effect = _get_bin - if gpg2 or gpg1: - assert gpg.gpg_bin == m_plib.Path.return_value - else: - assert not gpg.gpg_bin - - if gpg2 or gpg1: - assert ( - list(m_plib.Path.call_args) - == [(gpg2 or gpg1, ), {}]) - else: - assert not m_plib.Path.called - - if gpg2: - assert ( - list(list(c) for c in m_shutil.which.call_args_list) - == [[('gpg2',), {}]]) - return - assert ( - list(list(c) for c in m_shutil.which.call_args_list) - == [[('gpg2',), {}], [('gpg',), {}]]) - - -def test_identity_home(patches): - gpg = identity.GPGIdentity() - patched = patches( - "os", - "pathlib", - "pwd", - prefix="tools.gpg.identity") - - with patched as (m_os, m_plib, m_pwd): - assert gpg.home == m_plib.Path.return_value - - # m_os.environ.__getitem__.return_value - assert ( - list(m_plib.Path.call_args) - == [(m_os.environ.get.return_value, ), {}]) - assert ( - list(m_os.environ.get.call_args) - == [('HOME', m_pwd.getpwuid.return_value.pw_dir), {}]) - assert ( - list(m_pwd.getpwuid.call_args) - == [(m_os.getuid.return_value,), {}]) - assert ( - list(m_os.getuid.call_args) - == [(), {}]) - - assert "home" in gpg.__dict__ - - -@pytest.mark.parametrize("log", ["LOGGER", None]) -def test_identity_log(patches, log): - gpg = identity.GPGIdentity() - patched = patches( - "logging", - prefix="tools.gpg.identity") - - gpg._log = log - - with patched as (m_log, ): - if log: - assert gpg.log == log - assert not m_log.getLogger.called - else: - assert gpg.log == m_log.getLogger.return_value - assert ( - list(m_log.getLogger.call_args) - == [(gpg.__class__.__name__, ), {}]) - - -@pytest.mark.parametrize("name", ["NAME", None]) -@pytest.mark.parametrize("email", ["EMAIL", None]) -def test_identity_identity_id(patches, name, email): - gpg = identity.GPGIdentity() - patched = patches( - "formataddr", - ("GPGIdentity.provided_name", dict(new_callable=PropertyMock)), - ("GPGIdentity.provided_email", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_format, m_name, m_email): - m_name.return_value = name - m_email.return_value = email - result = gpg.provided_id - - assert "provided_id" in gpg.__dict__ - - if name and email: - assert ( - list(m_format.call_args) - == [(('NAME', 'EMAIL'),), {}]) - assert result == m_format.return_value - return - - assert not m_format.called - assert result == name or email - - -def test_identity_name(patches): - gpg = identity.GPGIdentity() - patched = patches( - "parseaddr", - ("GPGIdentity.uid", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_parse, m_uid): - assert gpg.name == m_parse.return_value.__getitem__.return_value - - assert ( - list(m_parse.return_value.__getitem__.call_args) - == [(0,), {}]) - assert ( - list(m_parse.call_args) - == [(m_uid.return_value,), {}]) - assert "name" in gpg.__dict__ - - -@pytest.mark.parametrize("key", ["KEY1", "KEY2", "KEY5"]) -@pytest.mark.parametrize("name", ["NAME", None]) -@pytest.mark.parametrize("email", ["EMAIL", None]) -def test_identity_signing_key(patches, key, name, email): - packager = MagicMock() - gpg = identity.GPGIdentity() - _keys = ["KEY1", "KEY2", "KEY3"] - patched = patches( - "GPGIdentity.match", - ("GPGIdentity.gpg", dict(new_callable=PropertyMock)), - ("GPGIdentity.provided_id", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_match, m_gpg, m_id): - if not name and not email: - m_id.return_value = None - m_match.side_effect = lambda k: (k == key and f"MATCH {k}") - m_gpg.return_value.list_keys.return_value = _keys - if key in _keys: - assert gpg.signing_key == f"MATCH {key}" - _match_attempts = _keys[:_keys.index(key) + 1] - else: - with pytest.raises(identity.GPGError) as e: - gpg.signing_key - if name or email: - assert ( - e.value.args[0] - == f"No key found for '{m_id.return_value}'") - else: - assert ( - e.value.args[0] - == 'No available key') - _match_attempts = _keys - - assert ( - list(m_gpg.return_value.list_keys.call_args) - == [(True, ), dict(keys=m_id.return_value)]) - assert ( - list(list(c) for c in m_match.call_args_list) - == [[(k,), {}] for k in _match_attempts]) - - -def test_identity_uid(patches): - gpg = identity.GPGIdentity() - patched = patches( - ("GPGIdentity.signing_key", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_key, ): - assert gpg.uid == m_key.return_value.__getitem__.return_value - - assert ( - list(m_key.return_value.__getitem__.call_args) - == [('uid',), {}]) - - assert "uid" not in gpg.__dict__ - - -@pytest.mark.parametrize("name", ["NAME", None]) -@pytest.mark.parametrize("email", ["EMAIL", None]) -@pytest.mark.parametrize("match", ["MATCH", None]) -@pytest.mark.parametrize("log", [True, False]) -def test_identity_match(patches, name, email, match, log): - gpg = identity.GPGIdentity() - _keys = ["KEY1", "KEY2", "KEY3"] - patched = patches( - "GPGIdentity._match_key", - ("GPGIdentity.provided_id", dict(new_callable=PropertyMock)), - ("GPGIdentity.log", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - key = dict(uids=["UID1", "UID2"]) - - with patched as (m_match, m_id, m_log): - if not log: - m_log.return_value = None - m_match.return_value = match - m_id.return_value = name or email - result = gpg.match(key) - - if not name and not email: - assert not m_match.called - if log: - assert ( - list(m_log.return_value.warning.call_args) - == [('No GPG name/email supplied, signing with first available key',), {}]) - assert ( - result - == {'uids': ['UID1', 'UID2'], 'uid': 'UID1'}) - return - assert ( - list(m_match.call_args) - == [(key["uids"],), {}]) - if log: - assert not m_log.return_value.warning.called - if match: - assert ( - result - == {'uids': ['UID1', 'UID2'], 'uid': 'MATCH'}) - else: - assert not result - - -@pytest.mark.parametrize("uids", [[], ["UID1"], ["UID1", "UID2"]]) -@pytest.mark.parametrize("email", [None, "UID1", "UID1", "UID2", "UID3"]) -def test_identity__match_email(patches, uids, email): - gpg = identity.GPGIdentity() - patched = patches( - "parseaddr", - ("GPGIdentity.provided_email", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_parse, m_email): - m_parse.side_effect = lambda _email: ("NAME", _email) - m_email.return_value = email - result = gpg._match_email(uids) - - if email in uids: - assert result == email - assert ( - list(list(c) for c in m_parse.call_args_list) - == [[(uid,), {}] for uid in uids[:uids.index(email) + 1]]) - return - - assert not result - assert ( - list(list(c) for c in m_parse.call_args_list) - == [[(uid,), {}] for uid in uids]) - - -@pytest.mark.parametrize("name", ["NAME", None]) -@pytest.mark.parametrize("email", ["EMAIL", None]) -def test_identity__match_key(patches, name, email): - gpg = identity.GPGIdentity() - _keys = ["KEY1", "KEY2", "KEY3"] - patched = patches( - "GPGIdentity._match_email", - "GPGIdentity._match_name", - "GPGIdentity._match_uid", - ("GPGIdentity.provided_email", dict(new_callable=PropertyMock)), - ("GPGIdentity.provided_name", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - key = dict(uids=["UID1", "UID2"]) - - with patched as (m_email, m_name, m_uid, m_pemail, m_pname): - m_pemail.return_value = email - m_pname.return_value = name - result = gpg._match_key(key) - - if name and email: - assert ( - list(m_uid.call_args) - == [(dict(uids=key["uids"]),), {}]) - assert not m_email.called - assert not m_name.called - assert result == m_uid.return_value - elif name: - assert ( - list(m_name.call_args) - == [(dict(uids=key["uids"]),), {}]) - assert not m_email.called - assert not m_uid.called - assert result == m_name.return_value - elif email: - assert ( - list(m_email.call_args) - == [(dict(uids=key["uids"]),), {}]) - assert not m_name.called - assert not m_uid.called - assert result == m_email.return_value - - -@pytest.mark.parametrize("uids", [[], ["UID1"], ["UID1", "UID2"]]) -@pytest.mark.parametrize("name", [None, "UID1", "UID1", "UID2", "UID3"]) -def test_identity__match_name(patches, uids, name): - gpg = identity.GPGIdentity() - patched = patches( - "parseaddr", - ("GPGIdentity.provided_name", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_parse, m_name): - m_parse.side_effect = lambda _name: (_name, "EMAIL") - m_name.return_value = name - result = gpg._match_name(uids) - - if name in uids: - assert result == name - assert ( - list(list(c) for c in m_parse.call_args_list) - == [[(uid,), {}] for uid in uids[:uids.index(name) + 1]]) - return - - assert not result - assert ( - list(list(c) for c in m_parse.call_args_list) - == [[(uid,), {}] for uid in uids]) - - -@pytest.mark.parametrize("uid", ["UID1", "UID7"]) -def test_identity__match_uid(patches, uid): - gpg = identity.GPGIdentity() - uids = [f"UID{i}" for i in range(5)] - matches = uid in uids - patched = patches( - ("GPGIdentity.provided_id", dict(new_callable=PropertyMock)), - prefix="tools.gpg.identity") - - with patched as (m_id, ): - m_id.return_value = uid - if matches: - assert gpg._match_uid(uids) == uid - else: - assert not gpg._match_uid(uids)