diff --git a/.github/workflows/check-deps.yml b/.github/workflows/check-deps.yml new file mode 100644 index 0000000000000..48444f2578101 --- /dev/null +++ b/.github/workflows/check-deps.yml @@ -0,0 +1,34 @@ +name: Check for latest_release of deps + +on : + schedule : + - cron : '0 8 * * *' + + workflow_dispatch : + +jobs : + build : + runs-on : ubuntu-latest + steps : + - name : checkout + uses : actions/checkout/@v2 + with : + ref : ${{ github.head_ref }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install virtualenv + + - name: setting up virtualenv + run : | + export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + # --create_issues flag to create issue only in github action + # and not interfere with the CI + ./tools/dependency/release_dates.sh ./bazel/repository_locations.bzl --create_issues + ./tools/dependency/release_dates.sh ./api/bazel/repository_locations.bzl --create_issues diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index 10733549ae779..73e496897b5d6 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -13,6 +13,7 @@ import os import sys +import argparse import github @@ -22,12 +23,20 @@ from colorama import Fore, Style from packaging import version +# Tag issues created with these labels. +LABELS = ['dependencies', 'area/build', 'no stalebot'] + # Thrown on errors related to release date or version. class ReleaseDateVersionError(Exception): pass +# Errors that happen during issue creation. +class DependencyUpdateError(Exception): + pass + + # Format a datetime object as UTC YYYY-MM-DD. def format_utc_date(date): # We only handle naive datetime objects right now, which is what PyGithub @@ -40,7 +49,7 @@ def format_utc_date(date): # Obtain latest release version and compare against metadata version, warn on # mismatch. -def verify_and_print_latest_release(dep, repo, metadata_version, release_date): +def verify_and_print_latest_release(dep, repo, metadata_version, release_date, create_issue=False): try: latest_release = repo.get_latest_release() except github.GithubException as err: @@ -51,6 +60,55 @@ def verify_and_print_latest_release(dep, repo, metadata_version, release_date): print( f'{Fore.YELLOW}*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' f'{latest_release.tag_name}@<{latest_release.created_at}>{Style.RESET_ALL}') + # check for --check_deps flag, To run this only on github action schedule + # and it does not bloat CI on every push + if create_issue: + create_issues(dep, metadata_version, release_date, latest_release) + + +# create issue for stale dependency +def create_issues(dep, metadata_version, release_date, latest_release): + """Create issues in GitHub. + + Args: + dep : name of the deps + metadata_version : + release_date : old release_date + latest_release : latest_release (name and date ) + """ + access_token = os.getenv('GITHUB_TOKEN') + git = github.Github(access_token) + repo = git.get_repo('envoyproxy/envoy') + # Find GitHub label objects for LABELS. + labels = [] + for label in repo.get_labels(): + if label.name in LABELS: + labels.append(label.name) + if len(labels) != len(LABELS): + raise DependencyUpdateError('Unknown labels (expected %s, got %s)' % (LABELS, labels)) + body = f'*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>:{latest_release.tag_name}@<{latest_release.created_at}>' + title = f'Newer release available {dep}: {latest_release.tag_name}' + if issues_exist(title, git): + print("Issue with %s already exists" % title) + print(' >> Issue already exists, not posting!') + return + print('Creating issues...') + try: + repo.create_issue(title, body=body, labels=LABELS) + except github.GithubException as e: + print(f'Unable to create issue, received error: {e}') + raise + + +# checks if issue exist +def issues_exist(title, git): + query = f'repo:envoyproxy/envoy {title} in:title' + try: + issues = git.search_issues(query) + except github.GithubException as e: + print(f'There is a problem looking for issue title: {title}, received {e}') + raise + return issues.totalCount > 0 # Print GitHub release date, throw ReleaseDateVersionError on mismatch with metadata release date. @@ -106,7 +164,7 @@ def get_untagged_release_date(repo, metadata_version, github_release): # Verify release dates in metadata against GitHub API. -def verify_and_print_release_dates(repository_locations, github_instance): +def verify_and_print_release_dates(repository_locations, github_instance, create_issue=False): for dep, metadata in sorted(repository_locations.items()): release_date = None # Obtain release information from GitHub API. @@ -122,7 +180,8 @@ def verify_and_print_release_dates(repository_locations, github_instance): release_date = get_untagged_release_date(repo, metadata['version'], github_release) if release_date: # Check whether there is a more recent version and warn if necessary. - verify_and_print_latest_release(dep, repo, github_release.version, release_date) + verify_and_print_latest_release( + dep, repo, github_release.version, release_date, create_issue) # Verify that the release date in metadata and GitHub correspond, # otherwise throw ReleaseDateVersionError. verify_and_print_release_date(dep, release_date, metadata['release_date']) @@ -132,19 +191,23 @@ def verify_and_print_release_dates(repository_locations, github_instance): if __name__ == '__main__': - if len(sys.argv) != 2: - print('Usage: %s ' % sys.argv[0]) - sys.exit(1) + # parsing location and github_action flag with argparse + parser = argparse.ArgumentParser() + parser.add_argument('location', type=str) + parser.add_argument('--create_issues', action='store_true') + args = parser.parse_args() access_token = os.getenv('GITHUB_TOKEN') if not access_token: print('Missing GITHUB_TOKEN') sys.exit(1) - path = sys.argv[1] + path = args.location + create_issue = args.create_issues spec_loader = exports.repository_locations_utils.load_repository_locations_spec path_module = exports.load_module('repository_locations', path) try: verify_and_print_release_dates( - spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token)) + spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token), + create_issue) except ReleaseDateVersionError as e: print( f'{Fore.RED}An error occurred while processing {path}, please verify the correctness of the ' diff --git a/tools/dependency/release_dates.sh b/tools/dependency/release_dates.sh index de12f53e4512b..51b48039a3ce5 100755 --- a/tools/dependency/release_dates.sh +++ b/tools/dependency/release_dates.sh @@ -7,4 +7,4 @@ set -e # TODO(phlax): move this job to bazel and remove this export API_PATH=api/ -PYTHONPATH=. python_venv release_dates "$1" +PYTHONPATH=. python_venv release_dates "$@" diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index e32c3c95056b8..4a8379bb67f89 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -25,5 +25,5 @@ python_venv() { pip3 install -r "${SCRIPT_DIR}"/requirements.txt shift - python3 "${SCRIPT_DIR}/${PY_NAME}.py" "$*" + python3 "${SCRIPT_DIR}/${PY_NAME}.py" "$@" }