Skip to content

Commit 478a517

Browse files
skeet70skeet70
authored
Merge pull request #5 from IronCoreLabs/calculate-tag-state-script
Actually push everywhere, build manifests
2 parents 915b6b3 + fe00735 commit 478a517

File tree

2 files changed

+183
-13
lines changed

2 files changed

+183
-13
lines changed

‎tag-state/README.md

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Buildlog Tag State
2+
3+
This script is used to repair IronCore's Google Container Registry in case tags ever get out of sync with the buildlog. GCR is deprecated but this same script could be modified to move from GCR to GAR (or to clone IronCore's public images with tags into a local repository based on the public buildlog).
4+
5+
## Usage
6+
7+
Requires the `docker` CLI tool to be installed and working. If pushing to IronCore's GCR, you must be correctly authenticated (use `icl-auth`).
8+
9+
```console
10+
./fix-tag-state.py ../tenant-security-proxy.json
11+
```
12+
13+
The script will use the buildlog to print out the correct state of images and their tags. It'll then pull the image digests referenced by that state and push that state out into GCR.
14+
15+
> WARNING: there's currently no dry run functionality, if you run it and don't kill it when it first starts running docker commands, things will be changed.

‎tag-state/calculate-tag-state.py

+168-13
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,23 @@
88
import re
99
import signal
1010
import sys
11+
import subprocess
1112

12-
split_version_re = re.compile("[.+-]")
13+
# requires docker be available on the CLI
14+
15+
# constants
16+
SPLIT_VERSION_REGEX = re.compile("[.+-]")
17+
NEEDS_MANIFEST = "NEEDS_MANIFEST"
18+
PENDING = "PENDING"
19+
SUCCESS = "SUCCESS"
20+
FAILED = "FAILED"
1321

1422

1523
# distutils.version.LooseVersion is deprecated. packaging.version is the recommended replacement but I'd like to
1624
# keep this script vanilla. This is an attempt at re-implementation good enough for our purposes.
1725
class LooseVersion:
1826
def __init__(self, s):
19-
self.vs = split_version_re.split(s)
27+
self.vs = SPLIT_VERSION_REGEX.split(s)
2028
self.n = max(len(x) for x in self.vs)
2129

2230
def cmp(self, other, op):
@@ -81,6 +89,10 @@ def read_buildlog_file(buildlog_file_path: Path) -> dict:
8189
print_error(f"Buildlog not found at {parsed_args.buildlog_file_path}")
8290

8391

92+
def semver_sorted_dict(d: dict) -> dict:
93+
return dict(sorted(d.items(), key=lambda i: LooseVersion(i[0])))
94+
95+
8496
def build_tag_state(buildlog_data: list) -> dict:
8597
tag_state = {}
8698
for buildlog_entry in buildlog_data:
@@ -90,26 +102,51 @@ def build_tag_state(buildlog_data: list) -> dict:
90102
major, minor, patch, arch = pad_none(
91103
version.replace(".", "-").split("-"), 4
92104
)
105+
manifest_semver = f"{major}.{minor}.{patch}"
106+
medium_tag = f"{major}.{minor}"
93107
if arch:
94-
tag_state[version] = digest
95-
# TODO: need to create a manifest that gets all the else tags of both digests
108+
tag_state[version] = {"digest": digest, "status": PENDING}
109+
tag_state[major] = {"digest": NEEDS_MANIFEST, "status": PENDING}
110+
tag_state[medium_tag] = {
111+
"digest": NEEDS_MANIFEST,
112+
"status": PENDING,
113+
}
114+
tag_state[manifest_semver] = {
115+
"digest": NEEDS_MANIFEST,
116+
"status": PENDING,
117+
}
96118
else:
97-
if f"{major}.{minor}.{patch}" in tag_state:
98-
tag_state[f"{major}.{minor}.{patch}"] = digest
119+
if manifest_semver in tag_state:
120+
# if the current digest for our major or minor tag is the previous version of this container, update it
121+
if (
122+
tag_state[major]["digest"]
123+
is tag_state[manifest_semver]["digest"]
124+
):
125+
tag_state[major]["digest"] = digest
126+
if (
127+
tag_state[medium_tag]["digest"]
128+
is tag_state[manifest_semver]["digest"]
129+
):
130+
tag_state[medium_tag]["digest"] = digest
131+
# update the full tag to the new version of the container always
132+
tag_state[manifest_semver] = {"digest": digest, "status": PENDING}
99133
else:
100-
tag_state[major] = digest
101-
tag_state[f"{major}.{minor}"] = digest
102-
tag_state[f"{major}.{minor}.{patch}"] = digest
134+
tag_state[major] = {"digest": digest, "status": PENDING}
135+
tag_state[medium_tag] = {
136+
"digest": digest,
137+
"status": PENDING,
138+
}
139+
tag_state[manifest_semver] = {"digest": digest, "status": PENDING}
103140
else:
104141
print_error(
105142
f"Buildlog entry found without associated container hash: {buildlog_entry}"
106143
)
107-
return tag_state
144+
return semver_sorted_dict(tag_state)
108145

109146

110-
def semver_sorted_dict(d: dict) -> str:
147+
def pretty_printable_dict(d: dict) -> str:
111148
return json.dumps(
112-
dict(sorted(tag_state.items(), key=lambda i: LooseVersion(i[0]))),
149+
d,
113150
indent=4,
114151
sort_keys=False,
115152
)
@@ -121,4 +158,122 @@ def semver_sorted_dict(d: dict) -> str:
121158
parsed_args = arg_parser.parse_args(sys.argv[1:])
122159
buildlog_data = read_buildlog_file(parsed_args.buildlog_file_path)
123160
tag_state = build_tag_state(buildlog_data)
124-
print(semver_sorted_dict(tag_state))
161+
print(pretty_printable_dict(tag_state))
162+
# tag and push everything that doesn't need a manifest
163+
image_name = parsed_args.buildlog_file_path.stem
164+
for key, value in tag_state.items():
165+
if value["digest"] is not NEEDS_MANIFEST:
166+
# Errors will show up here where containers/versions have been removed either because of our retention
167+
# policy or ones that were yanked.
168+
image_name_by_digest = (
169+
f"gcr.io/ironcore-images/{image_name}@sha256:{value['digest']}"
170+
)
171+
pull = subprocess.run(["docker", "image", "pull", image_name_by_digest])
172+
# tag with the new value
173+
if pull.returncode is 0:
174+
tag = subprocess.run(
175+
[
176+
"docker",
177+
"image",
178+
"tag",
179+
image_name_by_digest,
180+
f"gcr.io/ironcore-images/{image_name}:{key}",
181+
]
182+
)
183+
if tag.returncode is 0:
184+
push = subprocess.run(
185+
["docker", "push", f"gcr.io/ironcore-images/{image_name}:{key}"]
186+
)
187+
if push.returncode is 0:
188+
value["status"] = SUCCESS
189+
continue
190+
value["status"] = FAILED
191+
# print what was tagged
192+
successfully_tagged = {
193+
k: v["digest"] for k, v in tag_state.items() if v["status"] is SUCCESS
194+
}
195+
failed_tags = {
196+
k: v["digest"] for k, v in tag_state.items() if v["status"] is FAILED
197+
}
198+
needs_manifest = {
199+
k: v["digest"] for k, v in tag_state.items() if v["digest"] is NEEDS_MANIFEST
200+
}
201+
print(pretty_printable_dict(successfully_tagged))
202+
print_error(pretty_printable_dict(failed_tags))
203+
print(pretty_printable_dict(needs_manifest))
204+
# use the tags just pushed to create manifests
205+
link_manifest = {}
206+
for key in needs_manifest:
207+
# we expect no arch tags at this point
208+
major, minor, patch = pad_none(key.split("."), 3)
209+
# if there's a patch we need to create a manifest for it
210+
if patch:
211+
gcr_image = f"gcr.io/ironcore-images/{image_name}:{key}"
212+
# all our images that need manifests have arm64/amd64 versions
213+
create_manifest = subprocess.run(
214+
[
215+
"docker",
216+
"manifest",
217+
"create",
218+
gcr_image,
219+
f"{gcr_image}-arm64",
220+
f"{gcr_image}-amd64",
221+
]
222+
)
223+
if create_manifest.returncode is 0:
224+
annotate_manifest_arm = subprocess.run(
225+
[
226+
"docker",
227+
"manifest",
228+
"annotate",
229+
gcr_image,
230+
f"{gcr_image}-arm64",
231+
"--arch",
232+
"arm64",
233+
]
234+
)
235+
annotate_manifest_amd = subprocess.run(
236+
[
237+
"docker",
238+
"manifest",
239+
"annotate",
240+
gcr_image,
241+
f"{gcr_image}-amd64",
242+
"--arch",
243+
"amd64",
244+
]
245+
)
246+
# if creation and annotation worked, push the new manifest
247+
if (
248+
annotate_manifest_arm.returncode is 0
249+
and annotate_manifest_amd.returncode is 0
250+
):
251+
# rebuilds aren't a factor now and the tags are semver sorted, so the last writer is the right one
252+
major_manifest_name = f"gcr.io/ironcore-images/{image_name}:{major}"
253+
minor_manifest_name = (
254+
f"gcr.io/ironcore-images/{image_name}:{major}.{minor}"
255+
)
256+
subprocess.run(
257+
[
258+
"docker",
259+
"manifest",
260+
"create",
261+
major_manifest_name,
262+
f"{gcr_image}-arm64",
263+
f"{gcr_image}-amd64",
264+
]
265+
)
266+
subprocess.run(
267+
[
268+
"docker",
269+
"manifest",
270+
"create",
271+
minor_manifest_name,
272+
f"{gcr_image}-arm64",
273+
f"{gcr_image}-amd64",
274+
]
275+
)
276+
# the more specific annotation of the first manifest seems to carry through to the others
277+
subprocess.run(["docker", "manifest", "push", gcr_image])
278+
subprocess.run(["docker", "manifest", "push", major_manifest_name])
279+
subprocess.run(["docker", "manifest", "push", minor_manifest_name])

0 commit comments

Comments
 (0)