[rhcos-4.11] support being initiated from a cosa remote-session

src/cmd-build

@@ -139,11 +139,15 @@ prepare_build
 ostree --version
 rpm-ostree --version
 
-previous_build=$(get_latest_build)
+previous_build=$(get_latest_build_for_arch "$basearch")
+echo "Previous build: ${previous_build:-none}"
 if [ -n "${previous_build}" ]; then
     previous_builddir=$(get_build_dir "${previous_build}")
+    if [ ! -d "${previous_builddir}" ]; then
+        echo "Previous build directory doesn't exist locally. Ignoring..."
+        previous_build=""
+    fi
 fi
-echo "Previous build: ${previous_build:-none}"
 
 previous_commit=
 previous_ostree_tarfile_path=

src/cmd-buildfetch

@@ -22,7 +22,7 @@ from cosalib.cmdlib import (
     retry_stop,
     rm_allow_noent,
     sha256sum_file)
-from cosalib.s3 import download_file, head_bucket, head_object
+from cosalib.s3 import S3
 
 retry_requests_exception = (retry_if_exception_type(requests.Timeout) |
                             retry_if_exception_type(requests.ReadTimeout) |
@@ -34,6 +34,8 @@ FCOS_STREAMS_URL = "https://builds.coreos.fedoraproject.org/prod/streams"
 
 def main():
     args = parse_args()
+    if args.aws_config_file:
+        os.environ["AWS_CONFIG_FILE"] = args.aws_config_file
 
     url = args.url or f'{FCOS_STREAMS_URL}/{args.stream}/builds'
     if url.startswith("s3://"):
@@ -157,6 +159,8 @@ def parse_args():
                         help="the target architecture(s)")
     parser.add_argument("--artifact", default=[], action='append',
                         help="Fetch given image artifact(s)", metavar="ARTIFACT")
+    parser.add_argument("--aws-config-file", metavar='CONFIG', default="",
+                        help="Path to AWS config file")
     return parser.parse_args()
 
 
@@ -221,20 +225,23 @@ class HTTPFetcher(Fetcher):
 
 
 class S3Fetcher(Fetcher):
+    def __init__(self, url_base):
+        super().__init__(url_base)
+        self.s3_client = S3()
 
     def fetch_impl(self, url, dest):
         assert url.startswith("s3://")
         bucket, key = url[len("s3://"):].split('/', 1)
         # this function does not need to be retried with the decorator as download_file would
         # retry automatically based on s3config settings
-        download_file(bucket, key, dest)
+        self.s3_client.download_file(bucket, key, dest)
 
     def exists_impl(self, url):
         assert url.startswith("s3://")
         bucket, key = url[len("s3://"):].split('/', 1)
         # sanity check that the bucket exists and we have access to it
-        head_bucket(bucket=bucket)
-        return head_object(bucket=bucket, key=key)
+        self.s3_client.head_bucket(bucket=bucket)
+        return self.s3_client.head_object(bucket=bucket, key=key)
 
 
 class LocalFetcher(Fetcher):

src/cmd-buildupload

@@ -57,6 +57,8 @@ def parse_args():
                     "and set Content-Disposition names", action='store_true')
     s3.add_argument("--endpoint-url", help="URL of S3-compatible server",
                     action="store", metavar="URL")
+    s3.add_argument("--aws-config-file", metavar='CONFIG', default="",
+                    help="Path to AWS config file")
     s3.set_defaults(func=cmd_upload_s3)
 
     return parser.parse_args()
@@ -65,6 +67,8 @@ def parse_args():
 def cmd_upload_s3(args):
     bucket, prefix = args.url.split('/', 1)
     builds = Builds()
+    if args.aws_config_file:
+        os.environ["AWS_CONFIG_FILE"] = args.aws_config_file
     s3_client = boto3.client('s3', endpoint_url=args.endpoint_url)
     # This can't be an error for backcompat reasons, but let's print something
     if not os.path.isfile(BUILDFILES['sourceurl']):

src/cmd-generate-hashlist

@@ -82,8 +82,7 @@ class HashListV1(dict):
         import_ostree_commit(
             os.getcwd(),
             self._metadata.build_dir,
-            self._metadata,
-            force=True)
+            self._metadata)
         subprocess.check_call([
             'ostree', 'checkout',
             '--repo=tmp/repo', '-U',

src/cmd-meta

@@ -65,7 +65,7 @@ def new_cli():
 
     def pather(val):
         path = val.split('.')
-        if val.startswith("coreos-assembler."):
+        if val.startswith("coreos-assembler.") or val.startswith("fedora-coreos."):
             new_path = [f"{path[0]}.{path[1]}"]
             new_path.extend(path[2:])
             return ".".join(new_path)

src/cmd-sign

@@ -53,6 +53,8 @@ def parse_args():
                                     'RoboSignatory via fedora-messaging')
     robosig.add_argument("--s3", metavar='<BUCKET>[/PREFIX]', required=True,
                          help="bucket and prefix to S3 builds/ dir")
+    robosig.add_argument("--aws-config-file", metavar='CONFIG', default="",
+                         help="Path to AWS config file")
     group = robosig.add_mutually_exclusive_group(required=True)
     group.add_argument("--ostree", help="sign commit", action='store_true')
     group.add_argument("--images", help="sign images", action='store_true')
@@ -72,6 +74,8 @@ def parse_args():
 
 
 def cmd_robosignatory(args):
+    if args.aws_config_file:
+        os.environ["AWS_CONFIG_FILE"] = args.aws_config_file
     s3 = boto3.client('s3')
     args.bucket, args.prefix = get_bucket_and_prefix(args.s3)
 
@@ -134,7 +138,7 @@ def robosign_ostree(args, s3, build, gpgkey):
     # Ensure we have an unpacked repo with the ostree content
     if not os.path.exists('tmp/repo'):
         subprocess.check_call(['ostree', '--repo=tmp/repo', 'init', '--mode=archive'])
-    import_ostree_commit(os.getcwd(), builddir, build, force=True)
+    import_ostree_commit(os.getcwd(), builddir, build)
     repo = OSTree.Repo.new(Gio.File.new_for_path('tmp/repo'))
     repo.open(None)
 

src/cmdlib.sh

@@ -884,6 +884,18 @@ get_latest_build() {
     fi
 }
 
+get_latest_build_for_arch() {
+    local arch=$1; shift
+    # yup, this is happening
+    (python3 -c "
+import sys
+sys.path.insert(0, '${DIR}')
+from cosalib.builds import Builds
+buildid = Builds('${workdir:-$(pwd)}').get_latest_for_arch('${arch}')
+if buildid:
+    print(buildid)")
+}
+
 get_build_dir() {
     local buildid=$1; shift
     # yup, this is happening

src/cosalib/aws.py

@@ -1,5 +1,6 @@
 import boto3
 import json
+import os
 import subprocess
 import sys
 
@@ -113,6 +114,9 @@ def aws_run_ore(build, args):
     if args.force:
         ore_args.extend(['--force'])
 
+    if args.credentials_file:
+        ore_args.extend(['--credentials-file', args.credentials_file])
+
     region = "us-east-1"
     if args.region is not None and len(args.region) > 0:
         region = args.region[0]
@@ -159,6 +163,8 @@ def aws_run_ore(build, args):
 
 def aws_cli(parser):
     parser.add_argument("--bucket", help="S3 Bucket")
+    parser.add_argument("--credentials-file", help="AWS config file",
+                        default=os.environ.get("AWS_CONFIG_FILE"))
     parser.add_argument("--name-suffix", help="Suffix for name")
     parser.add_argument("--grant-user", help="Grant user launch permission",
                         nargs="*", default=[])

src/cosalib/builds.py

@@ -69,6 +69,12 @@ def get_latest(self):
         # just let throw if there are none
         return self._data['builds'][0]['id']
 
+    def get_latest_for_arch(self, basearch):
+        for build in self._data['builds']:
+            if basearch in build['arches']:
+                return build['id']
+        return None
+
     def get_build_arches(self, build_id):
         for build in self._data['builds']:
             if build['id'] == build_id:
@@ -130,14 +136,15 @@ def init_build_meta_json(self, ostree_commit, parent_build, destdir):
         genver_key = 'coreos-assembler.image-genver'
         if not self.is_empty():
             previous_buildid = parent_build or self.get_latest()
-            metapath = self.get_build_dir(previous_buildid) + '/meta.json'
-            with open(metapath) as f:
-                previous_buildmeta = json.load(f)
-            previous_commit = previous_buildmeta['ostree-commit']
-            previous_image_genver = int(previous_buildmeta[genver_key])
-            if previous_commit == ostree_commit:
-                image_genver = previous_image_genver + 1
-                buildid = f"{version}-{image_genver}"
+            if get_basearch() in self.get_build_arches(previous_buildid):
+                metapath = self.get_build_dir(previous_buildid) + '/meta.json'
+                with open(metapath) as f:
+                    previous_buildmeta = json.load(f)
+                previous_commit = previous_buildmeta['ostree-commit']
+                previous_image_genver = int(previous_buildmeta[genver_key])
+                if previous_commit == ostree_commit:
+                    image_genver = previous_image_genver + 1
+                    buildid = f"{version}-{image_genver}"
         meta = {
             'buildid': buildid,
             genver_key: image_genver