container pushing updates

build.sh

@@ -133,6 +133,13 @@ configure_user(){
     echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel-nopasswd
     # Contents of /etc/sudoers.d need not to be world writable
     chmod 600 /etc/sudoers.d/wheel-nopasswd
+
+    # Allow the builder user to run rootless podman
+    # Referenced at: https://github.com/containers/podman/issues/4056#issuecomment-1245715492
+    # Lifted from: https://github.com/containers/podman/blob/6e382d9ec2e6eb79a72537544341e496368b6c63/contrib/podmanimage/stable/Containerfile#L25-L26
+    echo -e "builder:1:999\nbuilder:1001:64535" > /etc/subuid
+    echo -e "builder:1:999\nbuilder:1001:64535" > /etc/subgid
+
 }
 
 write_archive_info() {

src/cmd-push-container-manifest

@@ -1,15 +1,16 @@
 #!/usr/bin/python3
 
 # Push a container manifest (i.e. multi-arch) to a container registry based on
-# arguments provided by ther user.
-
-# The inverse of cmd-buildfetch (i.e. we upload a build which later can be
-# partially re-downloaded with cmd-buildfetch).
+# arguments provided by the user.
 
 import argparse
 import os
 import sys
+import tempfile
 from cosalib.container_manifest import create_and_push_container_manifest
+from cosalib.builds import Builds
+from cosalib.meta import GenericBuildMeta
+from cosalib.cmdlib import sha256sum_file
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -18,8 +19,57 @@ def main():
     args = parse_args()
     if args.authfile:
         os.environ["REGISTRY_AUTH_FILE"] = args.authfile
-    create_and_push_container_manifest(
-        args.repo, args.tag, args.images, args.v2s2)
+    if args.images:
+        # User provided images directly
+        create_and_push_container_manifest(
+            args.repo, args.tag, args.images, args.v2s2, None)
+    else:
+        # Picking up images from artifacts in meta.json
+        builds = Builds()
+        if args.build == 'latest':
+            args.build = builds.get_latest()
+        print(f"Targeting build: {args.build}")
+        build_arches = builds.get_build_arches(args.build)
+        if not args.arches:
+            args.arches = build_arches
+        # Iterate over the requested architectures and:
+        #   - Make sure the container images exist and are on disk
+        #   - Store the buildmeta for the build/arch in the buildmetas dict
+        #   - Store the path to the container image in the container_images list
+        images = []
+        buildmetas = dict()
+        for arch in args.arches:
+            if arch not in build_arches:
+                print(f"Requested architecture {arch} is not in {args.build}")
+                raise Exception
+            builddir = builds.get_build_dir(build_id=args.build, basearch=arch)
+            buildmeta = GenericBuildMeta(build=args.build, basearch=arch,
+                                         workdir=os.path.abspath(os.getcwd()))
+            buildmetas[arch] = buildmeta
+            if not buildmeta['images'][args.artifact]:
+                print(f"No artifact {args.artifact} in {args.build}/{arch}")
+                raise Exception
+            ociarchive = os.path.join(builddir, buildmeta['images'][args.artifact]['path'])
+            ocisha256sum = buildmeta['images'][args.artifact]['sha256']
+            if not os.path.exists(ociarchive):
+                print(f"The file does not exist on disk: {ociarchive}")
+                raise Exception
+            if sha256sum_file(ociarchive) != ocisha256sum:
+                print(f"The file on disk {ociarchive} has an incorrect checksum")
+                raise Exception
+            images.append(f"oci-archive:{ociarchive}")
+
+        # Create/Upload the manifest list to the container registry
+        with tempfile.NamedTemporaryFile() as digestfile:
+            create_and_push_container_manifest(
+                args.repo, args.tag, images, args.v2s2, digestfile.name)
+            digestfile.seek(0)
+            digest = digestfile.read().decode('utf-8').strip()
+
+        # Update the meta.json in each build/arch metadata
+        for _, buildmeta in buildmetas.items():
+            buildmeta[args.metajsonname] = {'image': f"{args.repo}@{digest}"}
+        buildmeta.write(artifact_name=args.metajsonname)
 
 
 def parse_args():
@@ -39,17 +89,32 @@ Examples:
         --repo quay.io/dustymabe/fedora-coreos --tag stable \\
         --image oci-archive://builds/36.20220716.3.1/x86_64/fedora-coreos-37.20220725.91.0-ostree.x86_64.ociarchive \\
         --image oci-archive://builds/36.20220716.3.1/aarch64/fedora-coreos-37.20220725.91.0-ostree.aarch64.ociarchive \\
-        --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive""")
+        --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive
+
+    cosa push-container-manifest \\
+        --repo quay.io/dustymabe/fedora-coreos --tag stable --artifact=ostree \\
+        --metajsonname=base-oscontainer --build=latest --arch=x86_64 --arch=aarch64""")
     parser.add_argument("--repo", required=True, help="The registry repo to target for the manifest")
     parser.add_argument("--tag", required=True, help="The tag of the manifest to use")
     parser.add_argument("--authfile", help="A file to use for registry auth")
     parser.add_argument('--v2s2', action='store_true',
                         help='Use old image manifest version2 schema 2 format')
-    parser.add_argument("--images", required=True, action='append', default=[],
-                        help="""The images to add to the manifest. Can be specified multiple times like
-                        --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456
-                        --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""")
 
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--image", dest='images', action='append', default=[],
+                       help="""The images to add to the manifest. Can be specified multiple times like
+                       --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456
+                       --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""")
+    group.add_argument("--artifact", help="""The artifact""")
+
+    # A few more arguments that are used for `--artifact`
+    parser.add_argument("--build", default="latest", help="Build ID")
+    parser.add_argument("--arch", dest='arches', action='append', default=[],
+                        help="""Limit the architectures to upload to the specificed set
+                        (otherwise it defaults to all available for that build). Can be
+                        specificed multiple times like: --arch x86_64 --arch aarch64""")
+    parser.add_argument("--metajsonname",
+                        help="The name under which to store the container information in meta.json")
     return parser.parse_args()
 
 

src/cmd-remote-build-container

@@ -8,25 +8,27 @@ import tempfile
 import tenacity
 
 from cosalib.cmdlib import runcmd
-from os import environ
 
 # Set up logging
 logging.basicConfig(level=logging.INFO,
                     format="%(asctime)s %(levelname)s - %(message)s")
 
 
-def build_container_image(labels, buildDir, cacheTTL, repo, tag):
+def build_container_image(labels, buildDir, fromimage, cacheTTL, repo, tag):
     '''
     Build the image  using podman remote and push to the registry
     @param labels list labels to add to image
     @param buildDir str the location of the directory to build from
+    @param fromimage str value to pass to `podman build --from=`
     @param cacheTTL str value to pass to `podman build --cache-ttl=`
     @param repo str registry repository
     @param tag  str image tag
     '''
     cmd = ["podman", "build", f"--cache-ttl={cacheTTL}", f"--tag={repo}:{tag}", buildDir]
     for label in labels:
         cmd.extend([f"--label={label}"])
+    if fromimage:
+        cmd.extend([f"--from={fromimage}"])
     # Long running command. Send output to stdout for logging
     runcmd(cmd)
 
@@ -58,6 +60,19 @@ def push_container_image(repo, tag):
         raise Exception(f"Image pushed but not viewable in registry: tag: {tag}")
 
 
+def pull_oci_archive_from_remote(repo, tag, file):
+    '''
+    Retrieve the oci archive of the image and write it to a file
+    @param repo str registry repository (used to deduce image name)
+    @param tag  str image tag (used to deduce image name)
+    @param file str The name of the file to write the image to
+    '''
+    cmd = ["podman", "image", "save",
+           "--format=oci-archive", f"--output={file}", f"{repo}:{tag}"]
+    # Long running command. Send output to stdout for logging
+    runcmd(cmd)
+
+
 def is_tag_in_podman_storage(repo, tag):
     '''
     Search for a tag in the local podman storage
@@ -89,7 +104,7 @@ def main():
     if args.authfile:
         os.environ["REGISTRY_AUTH_FILE"] = args.authfile
     # Check for requisite env vars
-    if environ.get('CONTAINER_HOST') is None or environ.get('CONTAINER_SSHKEY') is None:
+    if os.environ.get('CONTAINER_HOST') is None or os.environ.get('CONTAINER_SSHKEY') is None:
         sys.exit('You must have CONTAINER_HOST and CONTAINER_SSHKEY environment variables setup')
 
     # Podman supports building from a specific commit
@@ -111,6 +126,9 @@ def main():
         commit = runcmd(cmd, quiet=True, capture_output=True).stdout.strip().decode()
         shortcommit = commit[0:7]
         logging.info(f"Translated {args.git_url}#{args.git_ref} into {shortcommit}")
+        # Add some information about the commit to labels for the container
+        args.labels.append(f"org.opencontainers.image.revision={commit}")
+        args.labels.append(f"org.opencontainers.image.source={args.git_url}")
         # If a tag wasn't passed then use the arch + shortcommit
         if not args.tag:
             args.tag = f"{args.arch}-{shortcommit}"
@@ -137,13 +155,16 @@ def main():
         if needbuild:
             logging.info("Building container via podman")
             builddir = os.path.join(gitdir, args.git_sub_dir)
-            build_container_image(args.label, builddir,
+            build_container_image(args.labels, builddir, args.fromimage,
                                   args.cache_ttl, args.repo, args.tag)
 
-    # Push to the registry if needed
+    # Push to the registry if needed, else save the image to a file
     if args.push_to_registry:
         logging.info("Pushing to remote registry")
         push_container_image(args.repo, args.tag)
+    else:
+        logging.info("Archiving build container image from remote")
+        pull_oci_archive_from_remote(args.repo, args.tag, args.write_to_file)
 
 
 def parse_args():
@@ -175,11 +196,14 @@ Examples:
         help="""Pass along --cache-ttl=<value> to `podman build`.
                 Defaults to 0.1s, which is effectively `--no-cache`""")
     parser.add_argument(
-        '--label', default=[], action='append',
+        '--label', dest="labels", default=[], action='append',
         required=False, help='Add image label(s)')
     parser.add_argument(
         '--force', required=False, action='store_true',
         help='Force image overwrite')
+    parser.add_argument(
+        '--from', dest="fromimage", required=False,
+        help='Pass along --from=<value> to `podman build`.')
     parser.add_argument(
         '--git-ref', required=True,
         help='Git branch or tag or commit')
@@ -190,14 +214,18 @@ Examples:
         '--git-sub-dir', default='', required=False,
         help='Git sub directory to use for container build')
     parser.add_argument(
-        '--push-to-registry', required=False, action='store_true',
-        help='Push image to registry. You must be logged in before pushing images')
-    parser.add_argument(
-        '--repo', required=True,
+        '--repo', default='localhost', required=False,
         help='Registry repository')
     parser.add_argument(
         '--tag', required=False,
         help='Force image tag. The default is arch-commit')
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        '--push-to-registry', required=False, action='store_true',
+        help='Push image to registry. You must be logged in before pushing images')
+    group.add_argument(
+        '--write-to-file', required=False,
+        help='Write container oci archive to named file')
 
     return parser.parse_args()
 

src/cmd-upload-oscontainer

@@ -0,0 +1 @@
+cmd-upload-oscontainer-deprecated-legacy-format