|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 3 | +# or more contributor license agreements. See the NOTICE file |
| 4 | +# distributed with this work for additional information |
| 5 | +# regarding copyright ownership. The ASF licenses this file |
| 6 | +# to you under the Apache License, Version 2.0 (the |
| 7 | +# "License"); you may not use this file except in compliance |
| 8 | +# with the License. You may obtain a copy of the License at |
| 9 | +# |
| 10 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +# |
| 12 | +# Unless required by applicable law or agreed to in writing, |
| 13 | +# software distributed under the License is distributed on an |
| 14 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | +# KIND, either express or implied. See the License for the |
| 16 | +# specific language governing permissions and limitations |
| 17 | +# under the License. |
| 18 | + |
| 19 | +import argparse |
| 20 | +import re |
| 21 | +import logging |
| 22 | +import datetime |
| 23 | +import os |
| 24 | +import json |
| 25 | +from urllib import error |
| 26 | +from typing import List, Dict, Any, Optional, Callable |
| 27 | +from git_utils import git, parse_remote, GitHubRepo |
| 28 | +from cmd_utils import REPO_ROOT, init_log |
| 29 | +from should_rebuild_docker import docker_api |
| 30 | + |
| 31 | +JENKINSFILE = REPO_ROOT / "jenkins" / "Jenkinsfile.j2" |
| 32 | +GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] |
| 33 | +BRANCH = "nightly-docker-update" |
| 34 | + |
| 35 | + |
| 36 | +def _testing_docker_api(data: Dict[str, Any]) -> Callable[[str], Dict[str, Any]]: |
| 37 | + """Returns a function that can be used in place of docker_api""" |
| 38 | + |
| 39 | + def mock(url: str) -> Dict[str, Any]: |
| 40 | + if url in data: |
| 41 | + return data[url] |
| 42 | + else: |
| 43 | + raise error.HTTPError(url, 404, f"Not found: {url}", {}, None) |
| 44 | + |
| 45 | + return mock |
| 46 | + |
| 47 | + |
| 48 | +def parse_docker_date(d: str) -> datetime.datetime: |
| 49 | + """Turn a date string from the Docker API into a datetime object""" |
| 50 | + return datetime.datetime.strptime(d, "%Y-%m-%dT%H:%M:%S.%fZ") |
| 51 | + |
| 52 | + |
| 53 | +def latest_tag(user: str, repo: str) -> List[Dict[str, Any]]: |
| 54 | + """ |
| 55 | + Queries Docker Hub and finds the most recent tag for the specified image/repo pair |
| 56 | + """ |
| 57 | + r = docker_api(f"repositories/{user}/{repo}/tags") |
| 58 | + results = r["results"] |
| 59 | + |
| 60 | + for result in results: |
| 61 | + result["last_updated"] = parse_docker_date(result["last_updated"]) |
| 62 | + |
| 63 | + results = list(sorted(results, key=lambda d: d["last_updated"])) |
| 64 | + return results[-1] |
| 65 | + |
| 66 | + |
| 67 | +def latest_tlcpackstaging_image(source: str) -> Optional[str]: |
| 68 | + """ |
| 69 | + Finds the latest full tag to use in the Jenkinsfile or returns None if no |
| 70 | + update is needed |
| 71 | + """ |
| 72 | + name, current_tag = source.split(":") |
| 73 | + user, repo = name.split("/") |
| 74 | + logging.info( |
| 75 | + f"Running with name: {name}, current_tag: {current_tag}, user: {user}, repo: {repo}" |
| 76 | + ) |
| 77 | + |
| 78 | + staging_repo = repo.replace("-", "_") |
| 79 | + latest_tlcpackstaging_tag = latest_tag(user="tlcpackstaging", repo=staging_repo) |
| 80 | + logging.info(f"Found latest tlcpackstaging tag:\n{latest_tlcpackstaging_tag}") |
| 81 | + |
| 82 | + if latest_tlcpackstaging_tag["name"] == current_tag: |
| 83 | + logging.info(f"tlcpackstaging tag is the same as the one in the Jenkinsfile") |
| 84 | + |
| 85 | + latest_tlcpack_tag = latest_tag(user="tlcpack", repo=repo) |
| 86 | + logging.info(f"Found latest tlcpack tag:\n{latest_tlcpack_tag}") |
| 87 | + |
| 88 | + if latest_tlcpack_tag["name"] == latest_tlcpackstaging_tag["name"]: |
| 89 | + logging.info("Tag names were the same, no update needed") |
| 90 | + return None |
| 91 | + |
| 92 | + if latest_tlcpack_tag["last_updated"] > latest_tlcpackstaging_tag["last_updated"]: |
| 93 | + new_spec = f"tlcpack/{repo}:{latest_tlcpack_tag['name']}" |
| 94 | + else: |
| 95 | + # Even if the image doesn't exist in tlcpack, it will fall back to tlcpackstaging |
| 96 | + # so hardcode the username here |
| 97 | + new_spec = f"tlcpack/{repo}:{latest_tlcpackstaging_tag['name']}" |
| 98 | + logging.info("Using tlcpackstaging tag on tlcpack") |
| 99 | + |
| 100 | + logging.info(f"Found newer image, using: {new_spec}") |
| 101 | + return new_spec |
| 102 | + |
| 103 | + |
| 104 | +if __name__ == "__main__": |
| 105 | + init_log() |
| 106 | + help = "Open a PR to update the Docker images to use the latest available in tlcpackstaging" |
| 107 | + parser = argparse.ArgumentParser(description=help) |
| 108 | + parser.add_argument("--remote", default="origin", help="ssh remote to parse") |
| 109 | + parser.add_argument("--dry-run", action="store_true", help="don't send PR to GitHub") |
| 110 | + parser.add_argument("--testing-docker-data", help="JSON data to mock Docker Hub API response") |
| 111 | + args = parser.parse_args() |
| 112 | + |
| 113 | + # Install test mock if necessary |
| 114 | + if args.testing_docker_data is not None: |
| 115 | + docker_api = _testing_docker_api(data=json.loads(args.testing_docker_data)) |
| 116 | + |
| 117 | + remote = git(["config", "--get", f"remote.{args.remote}.url"]) |
| 118 | + user, repo = parse_remote(remote) |
| 119 | + |
| 120 | + # Read the existing images from the Jenkinsfile |
| 121 | + logging.info(f"Reading {JENKINSFILE}") |
| 122 | + with open(JENKINSFILE) as f: |
| 123 | + content = f.readlines() |
| 124 | + |
| 125 | + # Build a new Jenkinsfile with the latest images from tlcpack or tlcpackstaging |
| 126 | + new_content = [] |
| 127 | + for line in content: |
| 128 | + m = re.match(r"^(ci_[a-zA-Z0-9]+) = \'(.*)\'", line.strip()) |
| 129 | + if m is not None: |
| 130 | + logging.info(f"Found match on line {line.strip()}") |
| 131 | + groups = m.groups() |
| 132 | + new_image = latest_tlcpackstaging_image(groups[1]) |
| 133 | + if new_image is None: |
| 134 | + logging.info(f"No new image found") |
| 135 | + new_content.append(line) |
| 136 | + else: |
| 137 | + logging.info(f"Using new image {new_image}") |
| 138 | + new_content.append(f"{groups[0]} = '{new_image}'\n") |
| 139 | + else: |
| 140 | + new_content.append(line) |
| 141 | + |
| 142 | + # Write out the new content |
| 143 | + if args.dry_run: |
| 144 | + logging.info(f"Dry run, would have written new content to {JENKINSFILE}") |
| 145 | + else: |
| 146 | + logging.info(f"Writing new content to {JENKINSFILE}") |
| 147 | + with open(JENKINSFILE, "w") as f: |
| 148 | + f.write("".join(new_content)) |
| 149 | + |
| 150 | + # Publish the PR |
| 151 | + title = "[ci][docker] Nightly Docker image update" |
| 152 | + body = "This bumps the Docker images to the latest versions from Docker Hub." |
| 153 | + message = f"{title}\n\n\n{body}" |
| 154 | + |
| 155 | + if args.dry_run: |
| 156 | + logging.info("Dry run, would have committed Jenkinsfile") |
| 157 | + else: |
| 158 | + logging.info(f"Creating git commit") |
| 159 | + git(["checkout", "-B", BRANCH]) |
| 160 | + git(["add", str(JENKINSFILE.relative_to(REPO_ROOT))]) |
| 161 | + git(["config", "user.name", "tvm-bot"]) |
| 162 | + git([ "config", "user.email", "[email protected]"]) |
| 163 | + git(["commit", "-m", message]) |
| 164 | + git(["push", "--set-upstream", args.remote, BRANCH, "--force"]) |
| 165 | + |
| 166 | + logging.info(f"Sending PR to GitHub") |
| 167 | + github = GitHubRepo(user=user, repo=repo, token=GITHUB_TOKEN) |
| 168 | + data = { |
| 169 | + "title": title, |
| 170 | + "body": body, |
| 171 | + "head": BRANCH, |
| 172 | + "base": "main", |
| 173 | + "maintainer_can_modify": True, |
| 174 | + } |
| 175 | + url = "pulls" |
| 176 | + if args.dry_run: |
| 177 | + logging.info(f"Dry run, would have sent {data} to {url}") |
| 178 | + else: |
| 179 | + try: |
| 180 | + github.post(url, data=data) |
| 181 | + except error.HTTPError as e: |
| 182 | + # Ignore the exception if the PR already exists (which gives a 422). The |
| 183 | + # existing PR will have been updated in place |
| 184 | + if e.code == 422: |
| 185 | + logging.info("PR already exists, ignoring error") |
| 186 | + logging.exception(e) |
| 187 | + else: |
| 188 | + raise e |
0 commit comments