libs: Add envoy.dependency.check[cves]

README.md

@@ -110,11 +110,11 @@ pypi: https://pypi.org/project/envoy.code_format.python_check
 ---
 
 
-#### [envoy.dependency.cve_scan](envoy.dependency.cve_scan)
+#### [envoy.dependency.check](envoy.dependency.check)
 
-version: 0.0.5.dev0
+version: 0.0.1.dev0
 
-pypi: https://pypi.org/project/envoy.dependency.cve_scan
+pypi: https://pypi.org/project/envoy.dependency.check
 
 ##### requirements:
 

envoy.dependency.check/BUILD

@@ -0,0 +1,2 @@
+
+pytooling_package("envoy.dependency.check")

envoy.dependency.check/README.rst

@@ -0,0 +1,5 @@
+
+envoy.dependency.check
+======================
+
+Dependency checker used in Envoy proxy's CI

envoy.dependency.check/VERSION

@@ -0,0 +1 @@
+0.0.1-dev

envoy.dependency.cve_scan/envoy/dependency/cve_scan/BUILD → envoy.dependency.check/envoy/dependency/check/BUILD

@@ -1,23 +1,27 @@
 
 pytooling_library(
-    "envoy.dependency.cve_scan",
+    "envoy.dependency.check",
     dependencies=[
         "//deps:abstracts",
         "//deps:aio.core",
         "//deps:aio.run.checker",
         "//deps:envoy.base.utils",
-        "//deps:aiohttp",
         "//deps:jinja2",
+        "//deps:aiohttp",
         "//deps:packaging",
     ],
     sources=[
         "abstract/__init__.py",
         "abstract/checker.py",
-        "abstract/cpe.py",
-        "abstract/cve.py",
+        "abstract/cves/__init__.py",
+        "abstract/cves/cpe.py",
+        "abstract/cves/cve.py",
+        "abstract/cves/cves.py",
+        "abstract/cves/version_matcher.py",
         "abstract/dependency.py",
-        "abstract/typing.py",
-        "abstract/version_matcher.py",
+        "checker.py",
+        "cmd.py",
         "exceptions.py",
+        "typing.py",
     ],
 )

envoy.dependency.check/envoy/dependency/check/__init__.py

@@ -0,0 +1,39 @@
+
+from . import abstract, exceptions, typing
+from .abstract import (
+    ADependency,
+    ADependencyChecker,
+    ADependencyCPE,
+    ADependencyCVE,
+    ADependencyCVEs,
+    ADependencyCVEVersionMatcher)
+from .checker import (
+    Dependency,
+    DependencyChecker,
+    DependencyCPE,
+    DependencyCVE,
+    DependencyCVEs,
+    DependencyCVEVersionMatcher)
+from .cmd import run, main
+from . import checker
+
+
+__all__ = (
+    "abstract",
+    "ADependency",
+    "ADependencyChecker",
+    "ADependencyCPE",
+    "ADependencyCVE",
+    "ADependencyCVEs",
+    "ADependencyCVEVersionMatcher",
+    "checker",
+    "Dependency",
+    "DependencyChecker",
+    "DependencyCPE",
+    "DependencyCVE",
+    "DependencyCVEs",
+    "DependencyCVEVersionMatcher",
+    "exceptions",
+    "main",
+    "run",
+    "typing")

envoy.dependency.check/envoy/dependency/check/abstract/__init__.py

@@ -0,0 +1,17 @@
+
+from .checker import ADependencyChecker
+from .cves import (
+    ADependencyCPE,
+    ADependencyCVE,
+    ADependencyCVEs,
+    ADependencyCVEVersionMatcher)
+from .dependency import ADependency
+
+
+__all__ = (
+    "ADependency",
+    "ADependencyChecker",
+    "ADependencyCPE",
+    "ADependencyCVE",
+    "ADependencyCVEs",
+    "ADependencyCVEVersionMatcher")

envoy.dependency.check/envoy/dependency/check/abstract/checker.py

@@ -0,0 +1,107 @@
+"""Abstract dependency checker."""
+
+import abc
+import argparse
+import json
+import pathlib
+from functools import cached_property
+from typing import Tuple, Type
+
+import aiohttp
+
+import abstracts
+
+from aio.run import checker
+
+from envoy.dependency.check import abstract, exceptions, typing
+
+
+class ADependencyChecker(
+        checker.Checker,
+        metaclass=abstracts.Abstraction):
+    """Dependency checker."""
+
+    checks = ("cves", )
+
+    @property
+    def cve_config(self):
+        return self.args.cve_config
+
+    @cached_property
+    def cves(self):
+        return self.cves_class(
+            self.dependencies,
+            config_path=self.cve_config,
+            session=self.session)
+
+    @property  # type:ignore
+    @abstracts.interfacemethod
+    def cves_class(self) -> "abstract.ADependencyCVEs":
+        """CVEs class."""
+        raise NotImplementedError
+
+    @cached_property
+    def dependencies(self) -> Tuple["abstract.ADependency", ...]:
+        """Tuple of dependencies."""
+        deps = []
+        for k, v in self.dependency_metadata.items():
+            deps.append(self.dependency_class(k, v))
+        return tuple(sorted(deps))
+
+    @property  # type:ignore
+    @abstracts.interfacemethod
+    def dependency_class(self) -> Type["abstract.ADependency"]:
+        """Dependency class."""
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def dependency_metadata(self) -> typing.DependenciesDict:
+        """Dependency metadata (derived in Envoy's case from
+        `repository_locations.bzl`)."""
+        return json.loads(self.repository_locations_path.read_text())
+
+    @property
+    def repository_locations_path(self) -> pathlib.Path:
+        return pathlib.Path(self.args.repository_locations)
+
+    @cached_property
+    def session(self) -> aiohttp.ClientSession:
+        """HTTP client session."""
+        return aiohttp.ClientSession()
+
+    def add_arguments(self, parser: argparse.ArgumentParser) -> None:
+        super().add_arguments(parser)
+        parser.add_argument('--repository_locations')
+        parser.add_argument('--cve_config')
+
+    async def check_cves(self) -> None:
+        """Scan for CVEs in a parsed NIST CVE database."""
+        for dep in self.dependencies:
+            await self.dep_cve_check(dep)
+
+    async def dep_cve_check(
+            self,
+            dep: "abstract.ADependency") -> None:
+        if not dep.cpe:
+            self.log.info(f"No CPE listed for: {dep.id}")
+            return
+        warnings = []
+        async for failing_cve in self.cves.dependency_check(dep):
+            warnings.append(
+                f'{failing_cve.format_failure(dep)}')
+        if warnings:
+            self.warn("cves", warnings)
+        else:
+            self.succeed("cves", [f"No CVEs found for: {dep.id}"])
+
+    async def on_checks_complete(self) -> int:
+        await self.session.close()
+        return await super().on_checks_complete()
+
+    @checker.preload(
+        when=["cves"],
+        catches=[exceptions.CVECheckError])
+    async def preload_cves(self) -> None:
+        async for download in self.cves.downloads:
+            self.log.debug(f"Preloaded cve data: {download}")

envoy.dependency.check/envoy/dependency/check/abstract/cves/__init__.py

@@ -0,0 +1,12 @@
+
+from .cpe import ADependencyCPE
+from .cve import ADependencyCVE
+from .cves import ADependencyCVEs
+from .version_matcher import ADependencyCVEVersionMatcher
+
+
+__all__ = (
+    "ADependencyCPE",
+    "ADependencyCVE",
+    "ADependencyCVEs",
+    "ADependencyCVEVersionMatcher")

envoy.dependency.cve_scan/envoy/dependency/cve_scan/abstract/cpe.py → envoy.dependency.check/envoy/dependency/check/abstract/cves/cpe.py

@@ -6,23 +6,22 @@
 
 from envoy.base import utils
 
-from envoy.dependency.cve_scan.exceptions import CPEError
-from . import dependency
+from envoy.dependency.check import abstract, exceptions
 
 
 FUZZY_DATE_RE = re.compile(r'(\d{4}).?(\d{2}).?(\d{2})')
 FUZZY_SEMVER_RE = re.compile(r'(\d+)[:\.\-_](\d+)[:\.\-_](\d+)')
 
 
-class ACPE(metaclass=abstracts.Abstraction):
+class ADependencyCPE(metaclass=abstracts.Abstraction):
     """Model a subset of CPE fields that are used in CPE matching."""
 
     @classmethod
-    def from_string(cls, cpe_str: str) -> "ACPE":
+    def from_string(cls, cpe_str: str) -> "ADependencyCPE":
         """Generate a CPE object from a CPE string."""
         components = cpe_str.split(':')
         if len(components) < 6 or not cpe_str.startswith('cpe:2.3:'):
-            raise CPEError(
+            raise exceptions.CPEError(
                 f"CPE string ({cpe_str}) must be a valid CPE v2.3 string")
         return cls(*components[2:6])
 
@@ -48,11 +47,9 @@ def vendor_normalized(self) -> str:
         significant."""
         return str(self.__class__(self.part, self.vendor, '*', '*'))
 
-    def dependency_match(self, dep: dependency.ADependency) -> bool:
+    def dependency_match(self, dep: "abstract.ADependency") -> bool:
         """Heuristically match dependency metadata against CPE."""
-
-        dep_cpe = self.__class__.from_string(
-            utils.typed(str, dep.cpe))
+        dep_cpe = self.__class__.from_string(utils.typed(str, dep.cpe))
 
         # We allow Envoy dependency CPEs to wildcard the 'product', this is
         # useful for

envoy.dependency.cve_scan/envoy/dependency/cve_scan/abstract/cve.py → envoy.dependency.check/envoy/dependency/check/abstract/cves/cve.py

@@ -9,8 +9,7 @@
 
 import abstracts
 
-from envoy.dependency.cve_scan.exceptions import CVEError
-from . import cpe, dependency, typing, version_matcher
+from envoy.dependency.check import abstract, exceptions, typing
 
 
 CVE_FAIL_TPL = """
@@ -27,7 +26,7 @@
 """
 
 
-class ACVE(metaclass=abstracts.Abstraction):
+class ADependencyCVE(metaclass=abstracts.Abstraction):
 
     def __init__(
             self,
@@ -41,17 +40,17 @@ def __gt__(self, other) -> bool:
 
     @property  # type:ignore
     @abstracts.interfacemethod
-    def cpe_class(self) -> "cpe.ACPE":
+    def cpe_class(self) -> "abstract.ADependencyCPE":
         """CPE class.
 
         Should match class specified in CVEChecker.
         """
         raise NotImplementedError
 
     @cached_property
-    def cpes(self) -> Set["cpe.ACPE"]:
+    def cpes(self) -> Set["abstract.ADependencyCPE"]:
         """Associated CPEs."""
-        cpe_set: Set["cpe.ACPE"] = set()
+        cpe_set: Set["abstract.ADependencyCPE"] = set()
         self.gather_cpes(self.nodes, cpe_set)
         return cpe_set
 
@@ -115,11 +114,11 @@ def severity(self) -> str:
     @property  # type:ignore
     @abstracts.interfacemethod
     def version_matcher_class(self) -> Type[
-            version_matcher.ACVEVersionMatcher]:
+            "abstract.ADependencyCVEVersionMatcher"]:
         """Version matcher class."""
         raise NotImplementedError
 
-    def dependency_match(self, dep: "dependency.ADependency") -> bool:
+    def dependency_match(self, dep: "abstract.ADependency") -> bool:
         """Heuristically match dependency metadata against CVE.
 
         In general, we allow false positives but want to keep the noise
@@ -137,7 +136,7 @@ def dependency_match(self, dep: "dependency.ADependency") -> bool:
                     if cve_cpe.version == '*'
                     else True)
 
-    def format_failure(self, dep: "dependency.ADependency") -> str:
+    def format_failure(self, dep: "abstract.ADependency") -> str:
         """Format CVE failure for a given dependency."""
         return self.fail_template.render(
             cve=self,
@@ -146,7 +145,7 @@ def format_failure(self, dep: "dependency.ADependency") -> str:
     def gather_cpes(
             self,
             nodes: List["typing.CVENodeDict"],
-            cpe_set: Set["cpe.ACPE"]) -> None:
+            cpe_set: Set["abstract.ADependencyCPE"]) -> None:
         """Recursively gather CPE data from CVE nodes."""
         for node in nodes:
             for cpe_match in node.get('cpe_match', []):
@@ -160,7 +159,7 @@ def gather_cpes(
     def include_version(
             self,
             cpe_match: "typing.CVENodeMatchDict",
-            cpe: "cpe.ACPE") -> bool:
+            cpe: "abstract.ADependencyCPE") -> bool:
         """Determine whether a CPE matches according to installed version of a
         dependency."""
         return (
@@ -171,10 +170,11 @@ def include_version(
 
     def parse_cve_date(self, date_str: str) -> date:
         if not date_str.endswith('Z'):
-            raise CVEError("CVE dates should be UTC and in isoformat")
+            raise exceptions.CVEError(
+                "CVE dates should be UTC and in isoformat")
         return date.fromisoformat(date_str.split('T')[0])
 
-    def wildcard_version_match(self, dep: "dependency.ADependency") -> bool:
+    def wildcard_version_match(self, dep: "abstract.ADependency") -> bool:
         # If the CVE was published after the dependency was last updated, it's
         # a potential match.
         return (