diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 9d8159c5fd012..71f0481fce35c 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -36,7 +36,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "0d3ca4ed434958dda241fb129f77bd5ef0ce246250feed2d5a5470c6f29a77fa", strip_prefix = "buildtools-4.0.0", urls = ["https://github.com/bazelbuild/buildtools/archive/4.0.0.tar.gz"], - release_date = "2021-02-03", + release_date = "2021-02-04", use_category = ["api"], ), com_github_cncf_udpa = dict( @@ -115,7 +115,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "39cc1fb45039c7687354ca497aff8a55c71d0f1e484f6b81124ba9d821c36441", strip_prefix = "opentelemetry-proto-{version}", urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/v{version}.tar.gz"], - release_date = "2020-12-09", + release_date = "2021-01-27", use_category = ["api"], ), ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index e37ee8ea607e4..7f1d32c439c42 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -42,7 +42,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "0.31.2", sha256 = "c84962b64d9ae4472adfb01ec2cf1aa73cb2ee8308242add55fa7cc38602d882", urls = ["https://github.com/bazelbuild/rules_apple/releases/download/{version}/rules_apple.{version}.tar.gz"], - release_date = "2021-05-04", + release_date = "2021-05-07", use_category = ["build"], ), rules_fuzzing = dict( @@ -185,7 +185,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "fmt-{version}", urls = ["https://github.com/fmtlib/fmt/releases/download/{version}/fmt-{version}.zip"], use_category = ["dataplane_core", "controlplane"], - release_date = "2020-08-06", + release_date = "2020-08-07", cpe = "cpe:2.3:a:fmt:fmt:*", ), com_github_gabime_spdlog = dict( @@ -208,7 +208,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "792f250fb546bde8590e72d64311ea00a70c175fd77df6bb5e02328fa15fe28e", strip_prefix = "libprotobuf-mutator-{version}", urls = ["https://github.com/google/libprotobuf-mutator/archive/v{version}.tar.gz"], - release_date = "2020-11-06", + release_date = "2020-11-13", use_category = ["test_only"], ), com_github_google_tcmalloc = dict( @@ -244,7 +244,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "grpc-{version}", urls = ["https://github.com/grpc/grpc/archive/v{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2020-12-01", + release_date = "2020-12-02", cpe = "cpe:2.3:a:grpc:grpc:*", ), com_github_luajit_luajit = dict( @@ -328,7 +328,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( version = "8.4.0", use_category = ["observability_ext"], extensions = ["envoy.tracers.skywalking"], - release_date = "2021-01-20", + release_date = "2021-02-01", cpe = "N/A", ), com_github_skyapm_cpp2sky = dict( @@ -341,7 +341,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/SkyAPM/cpp2sky/archive/v{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.skywalking"], - release_date = "2021-03-17", + release_date = "2021-03-24", cpe = "N/A", ), com_github_datadog_dd_opentracing_cpp = dict( @@ -354,7 +354,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v{version}.tar.gz"], use_category = ["observability_ext"], extensions = ["envoy.tracers.datadog"], - release_date = "2021-01-26", + release_date = "2021-01-27", cpe = "N/A", ), com_github_google_benchmark = dict( @@ -603,7 +603,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "protobuf-{version}", urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-all-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2021-05-06", + release_date = "2021-05-07", cpe = "cpe:2.3:a:google:protobuf:*", ), grpc_httpjson_transcoding = dict( @@ -671,11 +671,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( six = dict( project_name = "Six", project_desc = "Python 2 and 3 compatibility library", - project_url = "https://pypi.org/project/six", + project_url = "https://github.com/benjaminp/six", version = "1.12.0", - sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", - urls = ["https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-{version}.tar.gz"], - release_date = "2018-12-09", + sha256 = "0ce7aef70d066b8dda6425c670d00c25579c3daad8108b3e3d41bef26003c852", + urls = ["https://github.com/benjaminp/six/archive/{version}.tar.gz"], + release_date = "2018-12-10", use_category = ["other"], ), org_llvm_llvm = dict( @@ -686,7 +686,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "df83a44b3a9a71029049ec101fb0077ecbbdf5fe41e395215025779099a98fdf", strip_prefix = "llvm-{version}.src", urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-{version}.src.tar.xz"], - release_date = "2020-03-23", + release_date = "2020-03-24", use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wavm"], cpe = "cpe:2.3:a:llvm:*:*", @@ -871,7 +871,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # Only allow peeking at fuzzer related files for now. strip_prefix = "compiler-rt-{version}.src", urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/compiler-rt-{version}.src.tar.xz"], - release_date = "2020-12-18", + release_date = "2021-01-06", use_category = ["test_only"], ), upb = dict( @@ -918,7 +918,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "05f7c6eecb402f11fcb7e524c903f1ba1c38d3bdc9bf42bc8ec3cf7567b9f979", strip_prefix = "kafka-python-{version}", urls = ["https://github.com/dpkp/kafka-python/archive/{version}.tar.gz"], - release_date = "2020-02-20", + release_date = "2020-09-30", use_category = ["test_only"], ), proxy_wasm_cpp_sdk = dict( diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index 9d8159c5fd012..71f0481fce35c 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -36,7 +36,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "0d3ca4ed434958dda241fb129f77bd5ef0ce246250feed2d5a5470c6f29a77fa", strip_prefix = "buildtools-4.0.0", urls = ["https://github.com/bazelbuild/buildtools/archive/4.0.0.tar.gz"], - release_date = "2021-02-03", + release_date = "2021-02-04", use_category = ["api"], ), com_github_cncf_udpa = dict( @@ -115,7 +115,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "39cc1fb45039c7687354ca497aff8a55c71d0f1e484f6b81124ba9d821c36441", strip_prefix = "opentelemetry-proto-{version}", urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/v{version}.tar.gz"], - release_date = "2020-12-09", + release_date = "2021-01-27", use_category = ["api"], ), ) diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index 25071f9320df7..10733549ae779 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -19,9 +19,12 @@ import exports import utils +from colorama import Fore, Style +from packaging import version -# Thrown on errors related to release date. -class ReleaseDateError(Exception): + +# Thrown on errors related to release date or version. +class ReleaseDateVersionError(Exception): pass @@ -29,7 +32,9 @@ class ReleaseDateError(Exception): def format_utc_date(date): # We only handle naive datetime objects right now, which is what PyGithub # appears to be handing us. - assert (date.tzinfo is None) + if date.tzinfo is not None: + raise ReleaseDateVersionError( + "Expected UTC date without timezone information. Received timezone information") return date.date().isoformat() @@ -38,35 +43,66 @@ def format_utc_date(date): def verify_and_print_latest_release(dep, repo, metadata_version, release_date): try: latest_release = repo.get_latest_release() - if latest_release.created_at > release_date and latest_release.tag_name != metadata_version: - print( - f'*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' - f'{latest_release.tag_name}@<{latest_release.created_at}>') - except github.UnknownObjectException: - pass + except github.GithubException as err: + # Repositories can not have releases or if they have releases may not publish a latest releases. Return + print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') + return + if latest_release.created_at > release_date and latest_release.tag_name != metadata_version: + print( + f'{Fore.YELLOW}*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' + f'{latest_release.tag_name}@<{latest_release.created_at}>{Style.RESET_ALL}') -# Print GitHub release date, throw ReleaseDateError on mismatch with metadata release date. +# Print GitHub release date, throw ReleaseDateVersionError on mismatch with metadata release date. def verify_and_print_release_date(dep, github_release_date, metadata_release_date): mismatch = '' iso_release_date = format_utc_date(github_release_date) - print(f'{dep} has a GitHub release date {iso_release_date}') + print(f'{Fore.GREEN}{dep} has a GitHub release date {iso_release_date}{Style.RESET_ALL}') if iso_release_date != metadata_release_date: - raise ReleaseDateError(f'Mismatch with metadata release date of {metadata_release_date}') + raise ReleaseDateVersionError( + f'Mismatch with metadata release date of {metadata_release_date}') -# Extract release date from GitHub API. -def get_release_date(repo, metadata_version, github_release): - if github_release.tagged: +# Extract release date from GitHub API for tagged releases. +def get_tagged_release_date(repo, metadata_version, github_release): + + try: + latest = repo.get_latest_release() + except github.GithubException as err: + # Repositories can not have releases or if they have releases may not publish a latest releases. If this is the case we keep going + latest = '' + print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') + + if latest and github_release.version <= latest.tag_name: + release = repo.get_release(github_release.version) + return release.published_at + else: tags = repo.get_tags() - for tag in tags: + current_metadata_tag_commit_date = '' + for tag in tags.reversed: if tag.name == github_release.version: - return tag.commit.commit.committer.date - return None - else: - assert (metadata_version == github_release.version) - commit = repo.get_commit(github_release.version) - return commit.commit.committer.date + current_metadata_tag_commit_date = tag.commit.commit.committer.date + if not version.parse(tag.name).is_prerelease and version.parse( + tag.name) > version.parse(github_release.version): + print( + f'{Fore.YELLOW}*WARNING* {repo.name} has a newer release than {github_release.version}@<{current_metadata_tag_commit_date}>: ' + f'{tag.name}@<{tag.commit.commit.committer.date}>{Style.RESET_ALL}') + return current_metadata_tag_commit_date + + +# Extract release date from GitHub API for untagged releases. +def get_untagged_release_date(repo, metadata_version, github_release): + if metadata_version != github_release.version: + raise ReleaseDateVersionError( + f'Mismatch with metadata version {metadata_version} and github release version {github_release.version}' + ) + commit = repo.get_commit(github_release.version) + commits = repo.get_commits(since=commit.commit.committer.date) + if commits.totalCount > 1: + print( + f'{Fore.YELLOW}*WARNING* {repo.name} has {str(commits.totalCount - 1)} commits since {github_release.version}@<{commit.commit.committer.date}>{Style.RESET_ALL}' + ) + return commit.commit.committer.date # Verify release dates in metadata against GitHub API. @@ -75,19 +111,23 @@ def verify_and_print_release_dates(repository_locations, github_instance): release_date = None # Obtain release information from GitHub API. github_release = utils.get_github_release_from_urls(metadata['urls']) + print('github_release: ', github_release) if not github_release: print(f'{dep} is not a GitHub repository') continue repo = github_instance.get_repo(f'{github_release.organization}/{github_release.project}') - release_date = get_release_date(repo, metadata['version'], github_release) + if github_release.tagged: + release_date = get_tagged_release_date(repo, metadata['version'], github_release) + else: + release_date = get_untagged_release_date(repo, metadata['version'], github_release) if release_date: # Check whether there is a more recent version and warn if necessary. verify_and_print_latest_release(dep, repo, github_release.version, release_date) # Verify that the release date in metadata and GitHub correspond, - # otherwise throw ReleaseDateError. + # otherwise throw ReleaseDateVersionError. verify_and_print_release_date(dep, release_date, metadata['release_date']) else: - raise ReleaseDateError( + raise ReleaseDateVersionError( f'{dep} is a GitHub repository with no no inferrable release date') @@ -105,8 +145,8 @@ def verify_and_print_release_dates(repository_locations, github_instance): try: verify_and_print_release_dates( spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token)) - except ReleaseDateError as e: + except ReleaseDateVersionError as e: print( - f'An error occurred while processing {path}, please verify the correctness of the ' - f'metadata: {e}') + f'{Fore.RED}An error occurred while processing {path}, please verify the correctness of the ' + f'metadata: {e}{Style.RESET_ALL}') sys.exit(1) diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt index 68995769e3133..a159b96fe667d 100644 --- a/tools/dependency/requirements.txt +++ b/tools/dependency/requirements.txt @@ -48,13 +48,19 @@ cffi==1.14.5 \ --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c - # via pynacl + # via + # -r tools/dependency/requirements.txt + # pynacl chardet==4.0.0 \ --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 # via # -r tools/dependency/requirements.txt # requests +colorama==0.4.4 \ + --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ + --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 + # via -r tools/dependency/requirements.txt deprecated==1.2.12 \ --hash=sha256:08452d69b6b5bc66e8330adde0a4f8642e969b9e1702904d137eeb29c8ffc771 \ --hash=sha256:6d2de2de7931a968874481ef30208fd4e08da39177d61d3d4ebdf4366e7dbca1 @@ -67,10 +73,16 @@ idna==2.10 \ # via # -r tools/dependency/requirements.txt # requests +packaging==20.9 \ + --hash=sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5 \ + --hash=sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a + # via -r tools/dependency/requirements.txt pycparser==2.20 \ --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 - # via cffi + # via + # -r tools/dependency/requirements.txt + # cffi pygithub==1.55 \ --hash=sha256:1bbfff9372047ff3f21d5cd8e07720f3dbfdaf6462fcaed9d815f528f1ba7283 \ --hash=sha256:2caf0054ea079b71e539741ae56c5a95e073b81fa472ce222e81667381b9601b @@ -78,7 +90,9 @@ pygithub==1.55 \ pyjwt==2.1.0 \ --hash=sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1 \ --hash=sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130 - # via pygithub + # via + # -r tools/dependency/requirements.txt + # pygithub pynacl==1.4.0 \ --hash=sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4 \ --hash=sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4 \ @@ -98,7 +112,15 @@ pynacl==1.4.0 \ --hash=sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514 \ --hash=sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff \ --hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80 - # via pygithub + # via + # -r tools/dependency/requirements.txt + # pygithub +pyparsing==2.4.7 \ + --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ + --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b + # via + # -r tools/dependency/requirements.txt + # packaging pyyaml==5.4.1 \ --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ @@ -140,9 +162,9 @@ six==1.16.0 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 # via pynacl -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 +urllib3==1.26.5 \ + --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \ + --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098 # via # -r tools/dependency/requirements.txt # requests