diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml index 9cdc3508284b9..23054755c0ef6 100644 --- a/.github/workflows/check-deps.yml +++ b/.github/workflows/check-deps.yml @@ -1,36 +1,27 @@ -name: Check for latest_release of deps +name: Check dependencies -on : - schedule : - - cron : '0 8 * * *' +on: + schedule: + - cron: '0 8 * * *' - workflow_dispatch : + workflow_dispatch: -jobs : - build : - runs-on : ubuntu-latest +jobs: + build: + runs-on: ubuntu-latest if: github.repository_owner == 'envoyproxy' - - steps : - - name : checkout - uses : actions/checkout/@v2 - with : - ref : ${{ github.head_ref }} - + steps: + - name: Checkout repository + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.3.4 + with: + ref: ${{ github.head_ref }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Run dependency checker run: | - python -m pip install --upgrade pip - pip install virtualenv - - - name: setting up virtualenv - run : | export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - # --create_issues flag to create issue only in github action - # and not interfere with the CI - ./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl --create_issues - ./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl --create_issues + bazel run //tools/dependency:check -- -c release_issues + bazel run //tools/dependency:check -- -c cves -w error diff --git a/ci/check_repository_locations.sh b/ci/check_repository_locations.sh deleted file mode 100755 index bd799f1b05d90..0000000000000 --- a/ci/check_repository_locations.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -function no_change { - echo "No change to **/repository_locations.bzl" - exit 0 -} - -(./tools/git/modified_since_last_github_commit.sh . bzl | grep repository_locations) || no_change - -./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl -./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 463f40642fe82..a3c27d115daaa 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -454,9 +454,12 @@ elif [[ "$CI_TARGET" == "deps" ]]; then # Validate repository metadata. echo "check repositories..." "${ENVOY_SRCDIR}"/tools/check_repositories.sh - "${ENVOY_SRCDIR}"/ci/check_repository_locations.sh + + echo "check dependencies..." + bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:check # Run pip requirements tests + echo "check pip..." bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:pip_check exit 0 diff --git a/tools/base/requirements.in b/tools/base/requirements.in index a4e11a9caaae7..52a50b874949d 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -1,13 +1,14 @@ abstracts>=0.0.12 aio.api.bazel aio.core>=0.2.0 -aio.run.runner>=0.2.1 +aio.run.runner>=0.2.2 aio.run.checker>=0.2.1 colorama coloredlogs coverage envoy.base.utils>=0.0.14 envoy.code_format.python_check>=0.0.7 +envoy.dependency.check envoy.dependency.cve_scan>=0.0.4 envoy.dependency.pip_check>=0.1.0 envoy.distribution.release>=0.0.7 diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 3d28bdb9db15f..2682806132678 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -9,11 +9,13 @@ abstracts==0.0.12 \ # via # -r requirements.in # aio.api.bazel + # aio.api.github # aio.core # aio.run.checker # aio.run.runner # envoy.base.utils # envoy.code-format.python-check + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.release @@ -24,12 +26,18 @@ aio.api.bazel==0.0.1 \ --hash=sha256:21094c7f8ed038d4668d93efa908d0770cf4bb781373a1300f152b211ff3dc81 \ --hash=sha256:d110ab219de520c911bd1505f516cf208fea75fe66529f638c9b4ac182b20ab8 # via -r requirements.in +aio.api.github==0.0.3 \ + --hash=sha256:8054c3d023eb5c1dfe9ad93341f5e184c9835b2e502b737195b2539388a5af56 \ + --hash=sha256:d7b03d5e4fc3363603ec75fed2c77f6fdbff3ac596e79328500416e2d6c7a0e9 + # via envoy.dependency.check aio.core==0.2.0 \ --hash=sha256:40a6d6495eaf11a9333847e5d74ed84452da5dfbc785c65f022d7c1343126f4c \ --hash=sha256:a174f73793b57050c53463dde4a06f2655f613c72f4789f568dd4f32bc54af2c # via # -r requirements.in + # aio.api.github # envoy.code-format.python-check + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.distribution.release # envoy.distribution.repo @@ -41,13 +49,14 @@ aio.run.checker==0.2.1 \ # via # -r requirements.in # envoy.code-format.python-check + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.distrotest # envoy.distribution.verify -aio.run.runner==0.2.1 \ - --hash=sha256:80062b417b127b433224fd889673b06167cd9fbaaf24c4148f799e3f9993632d \ - --hash=sha256:87feef7303efba78908dde07482999daa2a0254b37a7c972b4fdfab685eb416b +aio.run.runner==0.2.2 \ + --hash=sha256:8d1c265076c01f5ffe6b26c08575a98db0f7bd1e38805bda98f1a7b4983619e8 \ + --hash=sha256:92e67031877cec36fc46ec33b7ab9158464ca1b5870e13dc2fb4374200ad4878 # via # -r requirements.in # aio.run.checker @@ -109,6 +118,7 @@ aiohttp==3.7.4.post0 \ # via # aio.core # aiodocker + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.github.abstract # envoy.github.release @@ -280,6 +290,7 @@ envoy.base.utils==0.0.14 \ # via # -r requirements.in # envoy.code-format.python-check + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.distrotest @@ -293,6 +304,10 @@ envoy.code-format.python-check==0.0.7 \ --hash=sha256:c34f12946c908d2c7deb9faefecb044d8d5a9458755b40ed2537b04184fc8a21 \ --hash=sha256:c4758e9da6d5cba437f8948becadb4c3ab5f9f01a07f1a0965944873ae724963 # via -r requirements.in +envoy.dependency.check==0.0.1 \ + --hash=sha256:52118226ff7f46698ad6b93ce6b04dabad4edbccc7a60db73dd1da83c12647a2 \ + --hash=sha256:f8738db62ed52ea439e10467cd2bdc0a6469f5f63711230d5dce4656d83d1023 + # via -r requirements.in envoy.dependency.cve-scan==0.0.4 \ --hash=sha256:036bc115f09b3e14151708a33f9fe4c4ee32e911c0096ff44c140492fd3bcc9a \ --hash=sha256:087abcbc5a366d0ef27359829096451b1578feb11f87920cc63dc853ebf2ac71 @@ -381,6 +396,8 @@ gidgethub==5.0.1 \ --hash=sha256:3efbd6998600254ec7a2869318bd3ffde38edc3a0d37be0c14bc46b45947b682 \ --hash=sha256:67245e93eb0918b37df038148af675df43b62e832c529d7f859f6b90d9f3e70d # via + # aio.api.github + # envoy.dependency.check # envoy.distribution.release # envoy.github.abstract # envoy.github.release @@ -419,6 +436,7 @@ jinja2==3.0.3 \ --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 # via # -r requirements.in + # envoy.dependency.check # envoy.dependency.cve-scan # sphinx markupsafe==2.0.1 \ @@ -530,6 +548,8 @@ packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 # via + # aio.api.github + # envoy.dependency.check # envoy.dependency.cve-scan # envoy.github.abstract # envoy.github.release diff --git a/tools/dependency/BUILD b/tools/dependency/BUILD index 9d2c7264b8321..887966042c23d 100644 --- a/tools/dependency/BUILD +++ b/tools/dependency/BUILD @@ -1,7 +1,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("//bazel:envoy_build_system.bzl", "envoy_package") -load("@base_pip3//:requirements.bzl", "requirement") load("//tools/base:envoy_python.bzl", "envoy_entry_point") +load("@base_pip3//:requirements.bzl", "requirement") load("@envoy_repo//:path.bzl", "PATH") licenses(["notice"]) # Apache 2 @@ -13,18 +13,17 @@ py_library( srcs = ["utils.py"], ) -py_binary( - name = "cve_scan", - srcs = ["cve_scan.py"], +envoy_entry_point( + name = "check", args = [ - "$(location :cve.yaml)", "--repository_locations=$(location //bazel:all_repository_locations)", + "--cve_config=$(location :cve.yaml)", ], data = [ ":cve.yaml", "//bazel:all_repository_locations", ], - deps = [requirement("envoy.dependency.cve_scan")], + pkg = "envoy.dependency.check", ) envoy_entry_point( diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py deleted file mode 100644 index 72b917f6b4098..0000000000000 --- a/tools/dependency/release_dates.py +++ /dev/null @@ -1,316 +0,0 @@ -# CLI tool to query GitHub API and compare with repository_locations.bzl. It: -# - Computes the release date of dependencies and fails if there is a mismatch -# with the metdata release date. -# - Looks up the latest release tag on GitHub and warns if this is later than the -# dependency version in the .bzl. -# -# Usage: -# tools/dependency/release_dates.sh -# -# You will need to set a GitHub access token in the GITHUB_TOKEN environment -# variable. You can generate personal access tokens under developer settings on -# GitHub. You should restrict the scope of the token to "repo: public_repo". - -import os -import sys -import argparse -import string - -import pytz - -import github - -import exports -import utils -from colorama import Fore, Style -from packaging.version import parse as parse_version - -# Tag issues created with these labels. -LABELS = ['dependencies', 'area/build', 'no stalebot'] -GITHUB_REPO_LOCATION = "envoyproxy/envoy" - -BODY_TPL = """ -Package Name: ${dep} -Current Version: ${metadata_version}@${release_date} -Available Version: ${tag_name}@${created_at} -Upstream releases: https://github.com/${package_name}/releases -""" - -CLOSING_TPL = """ -New version is available for this package -New Version: ${tag_name}@${created_at} -Upstream releases: https://github.com/${full_name}/releases -New Issue Link: https://github.com/${repo_location}/issues/${number} -""" - - -# Thrown on errors related to release date or version. -class ReleaseDateVersionError(Exception): - pass - - -# Errors that happen during issue creation. -class DependencyUpdateError(Exception): - pass - - -# Format a datetime object as UTC YYYY-MM-DD. -def format_utc_date(date): - date = date.replace(tzinfo=pytz.UTC) - return date.date().isoformat() - - -# Get the chronologically latest release from a github repo -def get_latest_release(repo, version_min): - current_version = parse_version(version_min) - latest_version = current_version - latest_release = None - for release in repo.get_releases(): - if release.prerelease: - continue - version = parse_version(release.tag_name) - if not version: - continue - if version >= latest_version: - latest_release = release - latest_version = version - return latest_release - - -# Obtain latest release version and compare against metadata version, warn on -# mismatch. -def verify_and_print_latest_release(dep, repo, metadata_version, release_date, create_issue=False): - try: - latest_release = get_latest_release(repo, metadata_version) - except github.GithubException as err: - # Repositories can not have releases or if they have releases may not publish a latest releases. Return - print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') - return - if latest_release and latest_release.created_at > release_date and latest_release.tag_name != metadata_version: - print( - f'{Fore.YELLOW}*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' - f'{latest_release.tag_name}@<{latest_release.created_at}>{Style.RESET_ALL}') - # check for --check_deps flag, To run this only on github action schedule - # and it does not bloat CI on every push - if create_issue: - create_issues(dep, repo, metadata_version, release_date, latest_release) - - -def is_sha(text): - if len(text) != 40: - return False - try: - int(text, 16) - except ValueError: - return False - return True - - -# create issue for stale dependency -def create_issues(dep, package_repo, metadata_version, release_date, latest_release): - """Create issues in GitHub. - - Args: - dep : name of the deps - package_repo: package Url - metadata_version: current version information - release_date : old release_date - latest_release : latest_release (name and date ) - """ - access_token = os.getenv('GITHUB_TOKEN') - git = github.Github(access_token) - repo = git.get_repo(GITHUB_REPO_LOCATION) - # Find GitHub label objects for LABELS. - labels = [] - for label in repo.get_labels(): - if label.name in LABELS: - labels.append(label.name) - if len(labels) != len(LABELS): - raise DependencyUpdateError('Unknown labels (expected %s, got %s)' % (LABELS, labels)) - # trunctate metadata_version to 7 char if its sha_hash - if is_sha(metadata_version): - metadata_version = metadata_version[0:7] - title = f'Newer release available `{dep}`: {latest_release.tag_name} (current: {metadata_version})' - # search for old package opened issue and close them - body = string.Template(BODY_TPL).substitute( - dep=dep, - metadata_version=metadata_version, - release_date=release_date, - tag_name=latest_release.tag_name, - created_at=latest_release.created_at, - package_name=package_repo.full_name) - if issues_exist(title, git): - print("Issue with %s already exists" % title) - print(' >> Issue already exists, not posting!') - return - print('Creating issues...') - try: - issue_created = repo.create_issue(title, body=body, labels=LABELS) - latest_release.latest_issue_number = issue_created.number - except github.GithubException as e: - print(f'Unable to create issue, received error: {e}') - raise - search_old_version_open_issue_exist(title, git, package_repo, latest_release) - - -# checks if issue exist -def issues_exist(title, git): - # search for common title - title_search = title[0:title.index("(") - 1] - query = f'repo:{GITHUB_REPO_LOCATION} {title_search} in:title' - try: - issues = git.search_issues(query) - except github.GithubException as e: - print(f'There is a problem looking for issue title: {title}, received {e}') - raise - return issues.totalCount > 0 - - -# search for issue by title and delete old issue if new package version is available -def search_old_version_open_issue_exist(title, git, package_repo, latest_release): - # search for only "Newer release available {dep}:" as will be common in dep issue - title_search = title[0:title.index(":")] - query = f'repo:{GITHUB_REPO_LOCATION} {title_search} in:title is:open' - # there might be more than one issue - # if current package version == issue package version no need to do anything, right issue is open - # if current package version != issue_title_version means a newer updated version is available - # and close old issue - issues = git.search_issues(query) - for issue in issues: - issue_version = get_package_version_from_issue(issue.title) - if issue_version != latest_release.tag_name: - close_old_issue(git, issue.number, latest_release, package_repo) - - -def get_package_version_from_issue(issue_title): - # issue title create by github action has two form - return issue_title.split(":")[1].split("(")[0].strip() - - -def close_old_issue(git, issue_number, latest_release, package_repo): - repo = git.get_repo(GITHUB_REPO_LOCATION) - closing_comment = string.Template(CLOSING_TPL) - try: - issue = repo.get_issue(number=issue_number) - print(f'Publishing closing comment... ') - issue.create_comment( - closing_comment.substitute( - tag_name=latest_release.tag_name, - created_at=latest_release.created_at, - full_name=package_repo.full_name, - repo_location=GITHUB_REPO_LOCATION, - number=latest_release.latest_issue_number)) - print(f'Closing this issue as new package is available') - issue.edit(state='closed') - except github.GithubException as e: - print(f'There was a problem in publishing comment or closing this issue {e}') - raise - return - - -# Print GitHub release date, throw ReleaseDateVersionError on mismatch with metadata release date. -def verify_and_print_release_date(dep, github_release_date, metadata_release_date): - mismatch = '' - iso_release_date = format_utc_date(github_release_date) - print(f'{Fore.GREEN}{dep} has a GitHub release date {iso_release_date}{Style.RESET_ALL}') - if iso_release_date != metadata_release_date: - raise ReleaseDateVersionError( - f'Mismatch with metadata release date of {metadata_release_date}') - - -# Extract release date from GitHub API for tagged releases. -def get_tagged_release_date(repo, metadata_version, github_release): - try: - latest = get_latest_release(repo, github_release.version) - if latest: - release = repo.get_release(github_release.version) - return release.published_at - except github.GithubException as err: - # Repositories can not have releases or if they have releases may not publish a latest releases. If this is the case we keep going - latest = '' - print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') - - tags = repo.get_tags() - current_metadata_tag_commit_date = '' - current_version = parse_version(github_release.version) - latest_version = current_version - for tag in tags.reversed: - if tag.name == github_release.version: - current_metadata_tag_commit_date = tag.commit.commit.committer.date - continue - tag_version = parse_version(tag.name) - if not tag_version.is_prerelease and tag_version > latest_version: - latest = tag - latest_version = tag_version - if latest: - print( - f'{Fore.YELLOW}*WARNING* {repo.name} has a newer release than {github_release.version}@<{current_metadata_tag_commit_date}>: ' - f'{latest.name}@<{latest.commit.commit.committer.date}>{Style.RESET_ALL}') - return current_metadata_tag_commit_date - - -# Extract release date from GitHub API for untagged releases. -def get_untagged_release_date(repo, metadata_version, github_release): - if metadata_version != github_release.version: - raise ReleaseDateVersionError( - f'Mismatch with metadata version {metadata_version} and github release version {github_release.version}' - ) - commit = repo.get_commit(github_release.version) - commits = repo.get_commits(since=commit.commit.committer.date) - if commits.totalCount > 1: - print( - f'{Fore.YELLOW}*WARNING* {repo.name} has {str(commits.totalCount - 1)} commits since {github_release.version}@<{commit.commit.committer.date}>{Style.RESET_ALL}' - ) - return commit.commit.committer.date - - -# Verify release dates in metadata against GitHub API. -def verify_and_print_release_dates(repository_locations, github_instance, create_issue=False): - for dep, metadata in sorted(repository_locations.items()): - release_date = None - # Obtain release information from GitHub API. - github_release = utils.get_github_release_from_urls(metadata['urls']) - if not github_release: - print(f'{dep} is not a GitHub repository') - continue - print('github_release: ', github_release) - repo = github_instance.get_repo(f'{github_release.organization}/{github_release.project}') - if github_release.tagged: - release_date = get_tagged_release_date(repo, metadata['version'], github_release) - else: - release_date = get_untagged_release_date(repo, metadata['version'], github_release) - if release_date: - # Check whether there is a more recent version and warn if necessary. - verify_and_print_latest_release( - dep, repo, github_release.version, release_date, create_issue) - # Verify that the release date in metadata and GitHub correspond, - # otherwise throw ReleaseDateVersionError. - verify_and_print_release_date(dep, release_date, metadata['release_date']) - else: - raise ReleaseDateVersionError( - f'{dep} is a GitHub repository with no no inferrable release date') - - -if __name__ == '__main__': - # parsing location and github_action flag with argparse - parser = argparse.ArgumentParser() - parser.add_argument('location', type=str) - parser.add_argument('--create_issues', action='store_true') - args = parser.parse_args() - access_token = os.getenv('GITHUB_TOKEN') - if not access_token: - print('Missing GITHUB_TOKEN') - sys.exit(1) - path = args.location - create_issue = args.create_issues - spec_loader = exports.repository_locations_utils.load_repository_locations_spec - path_module = exports.load_module('repository_locations', path) - try: - verify_and_print_release_dates( - spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token), - create_issue) - except ReleaseDateVersionError as e: - print( - f'{Fore.RED}An error occurred while processing {path}, please verify the correctness of the ' - f'metadata: {e}{Style.RESET_ALL}') - sys.exit(1) diff --git a/tools/dependency/release_dates.sh b/tools/dependency/release_dates.sh deleted file mode 100755 index 51b48039a3ce5..0000000000000 --- a/tools/dependency/release_dates.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -. tools/shell_utils.sh - -set -e - -# TODO(phlax): move this job to bazel and remove this -export API_PATH=api/ - -PYTHONPATH=. python_venv release_dates "$@"