Tools: update release date checker to check for release date vs. last commit of release

api/bazel/repository_locations.bzl

@@ -36,7 +36,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "0d3ca4ed434958dda241fb129f77bd5ef0ce246250feed2d5a5470c6f29a77fa",
         strip_prefix = "buildtools-4.0.0",
         urls = ["https://github.com/bazelbuild/buildtools/archive/4.0.0.tar.gz"],
-        release_date = "2021-02-03",
+        release_date = "2021-02-04",
         use_category = ["api"],
     ),
     com_github_cncf_udpa = dict(
@@ -115,7 +115,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "39cc1fb45039c7687354ca497aff8a55c71d0f1e484f6b81124ba9d821c36441",
         strip_prefix = "opentelemetry-proto-{version}",
         urls = ["https://github.com/open-telemetry/opentelemetry-proto/archive/v{version}.tar.gz"],
-        release_date = "2020-12-09",
+        release_date = "2021-01-27",
         use_category = ["api"],
     ),
 )

bazel/repository_locations.bzl

@@ -42,7 +42,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         version = "0.31.2",
         sha256 = "c84962b64d9ae4472adfb01ec2cf1aa73cb2ee8308242add55fa7cc38602d882",
         urls = ["https://github.com/bazelbuild/rules_apple/releases/download/{version}/rules_apple.{version}.tar.gz"],
-        release_date = "2021-05-04",
+        release_date = "2021-05-07",
         use_category = ["build"],
     ),
     rules_fuzzing = dict(
@@ -185,7 +185,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "fmt-{version}",
         urls = ["https://github.com/fmtlib/fmt/releases/download/{version}/fmt-{version}.zip"],
         use_category = ["dataplane_core", "controlplane"],
-        release_date = "2020-08-06",
+        release_date = "2020-08-07",
         cpe = "cpe:2.3:a:fmt:fmt:*",
     ),
     com_github_gabime_spdlog = dict(
@@ -208,7 +208,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "792f250fb546bde8590e72d64311ea00a70c175fd77df6bb5e02328fa15fe28e",
         strip_prefix = "libprotobuf-mutator-{version}",
         urls = ["https://github.com/google/libprotobuf-mutator/archive/v{version}.tar.gz"],
-        release_date = "2020-11-06",
+        release_date = "2020-11-13",
         use_category = ["test_only"],
     ),
     com_github_google_tcmalloc = dict(
@@ -244,7 +244,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "grpc-{version}",
         urls = ["https://github.com/grpc/grpc/archive/v{version}.tar.gz"],
         use_category = ["dataplane_core", "controlplane"],
-        release_date = "2020-12-01",
+        release_date = "2020-12-02",
         cpe = "cpe:2.3:a:grpc:grpc:*",
     ),
     com_github_luajit_luajit = dict(
@@ -328,7 +328,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         version = "8.4.0",
         use_category = ["observability_ext"],
         extensions = ["envoy.tracers.skywalking"],
-        release_date = "2021-01-20",
+        release_date = "2021-02-01",
         cpe = "N/A",
     ),
     com_github_skyapm_cpp2sky = dict(
@@ -341,7 +341,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         urls = ["https://github.com/SkyAPM/cpp2sky/archive/v{version}.tar.gz"],
         use_category = ["observability_ext"],
         extensions = ["envoy.tracers.skywalking"],
-        release_date = "2021-03-17",
+        release_date = "2021-03-24",
         cpe = "N/A",
     ),
     com_github_datadog_dd_opentracing_cpp = dict(
@@ -354,7 +354,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v{version}.tar.gz"],
         use_category = ["observability_ext"],
         extensions = ["envoy.tracers.datadog"],
-        release_date = "2021-01-26",
+        release_date = "2021-01-27",
         cpe = "N/A",
     ),
     com_github_google_benchmark = dict(
@@ -603,7 +603,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "protobuf-{version}",
         urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-all-{version}.tar.gz"],
         use_category = ["dataplane_core", "controlplane"],
-        release_date = "2021-05-06",
+        release_date = "2021-05-07",
         cpe = "cpe:2.3:a:google:protobuf:*",
     ),
     grpc_httpjson_transcoding = dict(
@@ -671,11 +671,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
     six = dict(
         project_name = "Six",
         project_desc = "Python 2 and 3 compatibility library",
-        project_url = "https://pypi.org/project/six",
+        project_url = "https://github.com/benjaminp/six",
         version = "1.12.0",
-        sha256 = "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73",
-        urls = ["https://files.pythonhosted.org/packages/dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca/six-{version}.tar.gz"],
-        release_date = "2018-12-09",
+        sha256 = "0ce7aef70d066b8dda6425c670d00c25579c3daad8108b3e3d41bef26003c852",
+        urls = ["https://github.com/benjaminp/six/archive/{version}.tar.gz"],
+        release_date = "2018-12-10",
         use_category = ["other"],
     ),
     org_llvm_llvm = dict(
@@ -686,7 +686,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "df83a44b3a9a71029049ec101fb0077ecbbdf5fe41e395215025779099a98fdf",
         strip_prefix = "llvm-{version}.src",
         urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/llvm-{version}.src.tar.xz"],
-        release_date = "2020-03-23",
+        release_date = "2020-03-24",
         use_category = ["dataplane_ext"],
         extensions = ["envoy.wasm.runtime.wavm"],
         cpe = "cpe:2.3:a:llvm:*:*",
@@ -871,7 +871,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         # Only allow peeking at fuzzer related files for now.
         strip_prefix = "compiler-rt-{version}.src",
         urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-{version}/compiler-rt-{version}.src.tar.xz"],
-        release_date = "2020-12-18",
+        release_date = "2021-01-06",
         use_category = ["test_only"],
     ),
     upb = dict(
@@ -918,7 +918,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         sha256 = "05f7c6eecb402f11fcb7e524c903f1ba1c38d3bdc9bf42bc8ec3cf7567b9f979",
         strip_prefix = "kafka-python-{version}",
         urls = ["https://github.com/dpkp/kafka-python/archive/{version}.tar.gz"],
-        release_date = "2020-02-20",
+        release_date = "2020-09-30",
         use_category = ["test_only"],
     ),
     proxy_wasm_cpp_sdk = dict(

tools/dependency/release_dates.py

@@ -15,21 +15,23 @@
 import sys
 
 import github
-
 import exports
 import utils
 
+from colorama import Fore, Style
+from packaging import version
+
 
-# Thrown on errors related to release date.
-class ReleaseDateError(Exception):
+# Thrown on errors related to release date or version.
+class ReleaseDateVersionError(Exception):
     pass
 
 
 # Format a datetime object as UTC YYYY-MM-DD.
 def format_utc_date(date):
     # We only handle naive datetime objects right now, which is what PyGithub
     # appears to be handing us.
-    assert (date.tzinfo is None)
+    assert date.tzinfo is None
     return date.date().isoformat()
 
 
@@ -38,35 +40,67 @@ def format_utc_date(date):
 def verify_and_print_latest_release(dep, repo, metadata_version, release_date):
     try:
         latest_release = repo.get_latest_release()
-        if latest_release.created_at > release_date and latest_release.tag_name != metadata_version:
-            print(
-                f'*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: '
-                f'{latest_release.tag_name}@<{latest_release.created_at}>')
-    except github.UnknownObjectException:
-        pass
+    except github.GithubException as err:
+        # Repositories can not have releases or if they have releases may not publish a latest releases. Return
+        print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.')
+        return
+    if latest_release.created_at > release_date and latest_release.tag_name != metadata_version:
+        print(
+            f'{Fore.YELLOW}*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: '
+            f'{latest_release.tag_name}@<{latest_release.created_at}>{Style.RESET_ALL}')
 
 
-# Print GitHub release date, throw ReleaseDateError on mismatch with metadata release date.
+# Print GitHub release date, throw ReleaseDateVersionError on mismatch with metadata release date.
 def verify_and_print_release_date(dep, github_release_date, metadata_release_date):
     mismatch = ''
     iso_release_date = format_utc_date(github_release_date)
-    print(f'{dep} has a GitHub release date {iso_release_date}')
+    print(f'{Fore.GREEN}{dep} has a GitHub release date {iso_release_date}{Style.RESET_ALL}')
     if iso_release_date != metadata_release_date:
-        raise ReleaseDateError(f'Mismatch with metadata release date of {metadata_release_date}')
+        raise ReleaseDateVersionError(
+            f'Mismatch with metadata release date of {metadata_release_date}')
+
+
+# Extract release date from GitHub API for tagged releases.
+def get_tagged_release_date(repo, metadata_version, github_release):
 
+    try:
+        latest = repo.get_latest_release()
+    except github.GithubException as err:
+        # Repositories can not have releases or if they have releases may not publish a latest releases. If this is the case we keep going
+        latest = ''
+        print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.')
+        pass
 
-# Extract release date from GitHub API.
-def get_release_date(repo, metadata_version, github_release):
-    if github_release.tagged:
+    if latest and github_release.version <= latest.tag_name:
+        release = repo.get_release(github_release.version)
+        return release.published_at
+    else:
         tags = repo.get_tags()
-        for tag in tags:
+        current_metadata_tag_commit_date = ''
+        for tag in tags.reversed:
             if tag.name == github_release.version:
-                return tag.commit.commit.committer.date
-        return None
-    else:
-        assert (metadata_version == github_release.version)
-        commit = repo.get_commit(github_release.version)
-        return commit.commit.committer.date
+                current_metadata_tag_commit_date = tag.commit.commit.committer.date
+            if not version.parse(tag.name).is_prerelease and version.parse(
+                    tag.name) > version.parse(github_release.version):
+                print(
+                    f'{Fore.YELLOW}*WARNING* {repo.name} has a newer release than {github_release.version}@<{current_metadata_tag_commit_date}>: '
+                    f'{tag.name}@<{tag.commit.commit.committer.date}>{Style.RESET_ALL}')
+        return current_metadata_tag_commit_date
+
+
+# Extract release date from GitHub API for untagged releases.
+def get_untagged_release_date(repo, metadata_version, github_release):
+    if metadata_version != github_release.version:
+        raise ReleaseDateVersionError(
+            f'Mismatch with metadata version {metadata_version} and github release version {github_release.version}'
+        )
+    commit = repo.get_commit(github_release.version)
+    commits = repo.get_commits(since=commit.commit.committer.date)
+    if commits.totalCount > 1:
+        print(
+            f'{Fore.YELLOW}*WARNING* {repo.name} has {str(commits.totalCount - 1)} commits since {github_release.version}@<{commit.commit.committer.date}>{Style.RESET_ALL}'
+        )
+    return commit.commit.committer.date
 
 
 # Verify release dates in metadata against GitHub API.
@@ -75,19 +109,23 @@ def verify_and_print_release_dates(repository_locations, github_instance):
         release_date = None
         # Obtain release information from GitHub API.
         github_release = utils.get_github_release_from_urls(metadata['urls'])
+        print('github_release: ', github_release)
         if not github_release:
             print(f'{dep} is not a GitHub repository')
             continue
         repo = github_instance.get_repo(f'{github_release.organization}/{github_release.project}')
-        release_date = get_release_date(repo, metadata['version'], github_release)
+        if github_release.tagged:
+            release_date = get_tagged_release_date(repo, metadata['version'], github_release)
+        else:
+            release_date = get_untagged_release_date(repo, metadata['version'], github_release)
         if release_date:
             # Check whether there is a more recent version and warn if necessary.
             verify_and_print_latest_release(dep, repo, github_release.version, release_date)
             # Verify that the release date in metadata and GitHub correspond,
-            # otherwise throw ReleaseDateError.
+            # otherwise throw ReleaseDateVersionError.
             verify_and_print_release_date(dep, release_date, metadata['release_date'])
         else:
-            raise ReleaseDateError(
+            raise ReleaseDateVersionError(
                 f'{dep} is a GitHub repository with no no inferrable release date')
 
 
@@ -105,8 +143,8 @@ def verify_and_print_release_dates(repository_locations, github_instance):
     try:
         verify_and_print_release_dates(
             spec_loader(path_module.REPOSITORY_LOCATIONS_SPEC), github.Github(access_token))
-    except ReleaseDateError as e:
+    except ReleaseDateVersionError as e:
         print(
-            f'An error occurred while processing {path}, please verify the correctness of the '
-            f'metadata: {e}')
+            f'{Fore.RED}An error occurred while processing {path}, please verify the correctness of the '
+            f'metadata: {e}{Style.RESET_ALL}')
         sys.exit(1)

tools/dependency/requirements.txt

@@ -48,13 +48,19 @@ cffi==1.14.5 \
     --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \
     --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \
     --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c
-    # via pynacl
+    # via
+    #   -r tools/dependency/requirements.txt
+    #   pynacl
 chardet==4.0.0 \
     --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \
     --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5
     # via
     #   -r tools/dependency/requirements.txt
     #   requests
+colorama==0.4.4 \
+    --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
+    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2
+    # via -r tools/dependency/requirements.txt
 deprecated==1.2.12 \
     --hash=sha256:08452d69b6b5bc66e8330adde0a4f8642e969b9e1702904d137eeb29c8ffc771 \
     --hash=sha256:6d2de2de7931a968874481ef30208fd4e08da39177d61d3d4ebdf4366e7dbca1
@@ -67,18 +73,26 @@ idna==2.10 \
     # via
     #   -r tools/dependency/requirements.txt
     #   requests
+packaging==20.9 \
+    --hash=sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5 \
+    --hash=sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a
+    # via -r tools/dependency/requirements.txt
 pycparser==2.20 \
     --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
     --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
-    # via cffi
+    # via
+    #   -r tools/dependency/requirements.txt
+    #   cffi
 pygithub==1.55 \
     --hash=sha256:1bbfff9372047ff3f21d5cd8e07720f3dbfdaf6462fcaed9d815f528f1ba7283 \
     --hash=sha256:2caf0054ea079b71e539741ae56c5a95e073b81fa472ce222e81667381b9601b
     # via -r tools/dependency/requirements.txt
 pyjwt==2.1.0 \
     --hash=sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1 \
     --hash=sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130
-    # via pygithub
+    # via
+    #   -r tools/dependency/requirements.txt
+    #   pygithub
 pynacl==1.4.0 \
     --hash=sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4 \
     --hash=sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4 \
@@ -98,7 +112,15 @@ pynacl==1.4.0 \
     --hash=sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514 \
     --hash=sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff \
     --hash=sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80
-    # via pygithub
+    # via
+    #   -r tools/dependency/requirements.txt
+    #   pygithub
+pyparsing==2.4.7 \
+    --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
+    --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
+    # via
+    #   -r tools/dependency/requirements.txt
+    #   packaging
 pyyaml==5.4.1 \
     --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \
     --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \
@@ -140,9 +162,9 @@ six==1.16.0 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
     # via pynacl
-urllib3==1.26.4 \
-    --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \
-    --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937
+urllib3==1.26.5 \
+    --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \
+    --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098
     # via
     #   -r tools/dependency/requirements.txt
     #   requests