Add cosa buildextend-extensions

mantle/cosa/cosa_v1.go

@@ -1,6 +1,6 @@
 package cosa
 
-// generated by "schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT
+// generated by "../tools/bin/schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT
 
 type AliyunImage struct {
 	ImageID string `json:"id"`
@@ -39,6 +39,7 @@ type Build struct {
 	CosaDelayedMetaMerge      bool                  `json:"coreos-assembler.delayed-meta-merge,omitempty"`
 	CosaImageChecksum         string                `json:"coreos-assembler.image-config-checksum,omitempty"`
 	CosaImageVersion          int                   `json:"coreos-assembler.image-genver,omitempty"`
+	Extensions                *Extensions           `json:"extensions,omitempty"`
 	FedoraCoreOsParentCommit  string                `json:"fedora-coreos.parent-commit,omitempty"`
 	FedoraCoreOsParentVersion string                `json:"fedora-coreos.parent-version,omitempty"`
 	Gcp                       *Gcp                  `json:"gcp,omitempty"`
@@ -95,6 +96,13 @@ type Cloudartifact struct {
 	URL   string `json:"url"`
 }
 
+type Extensions struct {
+	Manifest       map[string]interface{} `json:"manifest"`
+	Path           string                 `json:"path"`
+	RpmOstreeState string                 `json:"rpm-ostree-state"`
+	Sha256         string                 `json:"sha256"`
+}
+
 type Gcp struct {
 	ImageFamily  string `json:"family,omitempty"`
 	ImageName    string `json:"image"`

mantle/cosa/schema_doc.go

@@ -186,6 +186,7 @@ var generatedSchemaJSON = `{
    "ibmcloud",
    "images",
    "oscontainer",
+   "extensions",
    "parent-pkgdiff",
    "pkgdiff",
    "release-payload",
@@ -524,6 +525,41 @@ var generatedSchemaJSON = `{
      "title":"Oscontainer",
      "$ref": "#/definitions/image"
     },
+   "extensions": {
+     "$id":"#/properties/extensions",
+     "type":"object",
+     "title":"Extensions",
+     "required": [
+         "path",
+         "sha256",
+         "rpm-ostree-state",
+         "manifest"
+     ],
+     "properties": {
+       "path": {
+         "$id": "#/artifact/Path",
+         "type":"string",
+         "title":"Path"
+        },
+       "sha256": {
+         "$id": "#/artifact/sha256",
+         "type":"string",
+         "title":"SHA256"
+        },
+       "rpm-ostree-state": {
+         "$id":"#/properties/extensions/items/properties/rpm-ostree-state",
+         "type":"string",
+         "title":"RpmOstreeState",
+         "default":"",
+         "minLength": 64
+        },
+       "manifest": {
+         "$id":"#/properties/extensions/items/properties/manifest",
+         "type":"object",
+         "title":"Manifest"
+       }
+      }
+    },
    "ostree-commit": {
      "$id":"#/properties/ostree-commit",
      "type":"string",

src/cmd-buildextend-extensions

@@ -0,0 +1,125 @@
+#!/usr/bin/python3 -u
+
+import argparse
+import os
+import shutil
+import sys
+
+import createrepo_c as cr
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from cosalib import cmdlib
+from cosalib.builds import Builds
+from cosalib.meta import GenericBuildMeta
+
+
+def main():
+    args = parse_args()
+
+    workdir = os.path.abspath(os.getcwd())
+
+    builds = Builds()
+    builddir = builds.get_build_dir(args.build)
+    buildmeta = GenericBuildMeta(workdir=workdir, build=args.build)
+
+    if 'extensions' in buildmeta and not args.force:
+        print(f"Extensions already exist: {buildmeta['extensions']['path']}")
+        print("Use --force to force a rebuild")
+        return
+
+    treefile_src = 'src/config/manifest.yaml'
+    extensions_src = 'src/config/extensions.yaml'
+    if not os.path.exists(extensions_src):
+        raise Exception(f"Missing {extensions_src}")
+
+    commit = buildmeta['ostree-commit']
+    ostree_tarball = os.path.join(builddir, buildmeta['images']['ostree']['path'])
+    cmdlib.import_ostree_commit('tmp/repo', commit, ostree_tarball)
+
+    tmpworkdir = prepare_tmpworkdir()
+    changed = run_rpmostree(tmpworkdir, commit, treefile_src, extensions_src)
+    if not changed:
+        # For now, rpm-ostree will always detect a change because we don't seed
+        # state from the previous build, so we won't hit this. Need to rework
+        # how change detection is wired in `cmd-build` to do this properly.
+        return
+
+    outputdir = f"{tmpworkdir}/output"
+    with open(f'{outputdir}/.rpm-ostree-state-chksum', encoding='utf-8') as f:
+        rpm_ostree_state_chksum = f.read()
+
+    pkglist = create_yumrepo(outputdir)
+    extensions_tarball = create_tarball(buildmeta, outputdir, tmpworkdir)
+    extensions_tarball_base = os.path.basename(extensions_tarball)
+
+    buildmeta['extensions'] = {
+        "path": extensions_tarball_base,
+        "sha256": cmdlib.sha256sum_file(extensions_tarball),
+        "rpm-ostree-state": rpm_ostree_state_chksum,
+        "manifest": pkglist,
+    }
+
+    cmdlib.rm_allow_noent(f'{builddir}/{extensions_tarball_base}')
+    shutil.move(extensions_tarball, builddir)
+    buildmeta.write(artifact_name='extensions')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--build", help="Build ID", default='latest')
+    parser.add_argument("--force", help="Force rebuild", action='store_true')
+    return parser.parse_args()
+
+
+def prepare_tmpworkdir():
+    tmpworkdir = 'tmp/extensions'
+    if os.path.exists(tmpworkdir):
+        shutil.rmtree(tmpworkdir)
+    os.mkdir(tmpworkdir)
+    return tmpworkdir
+
+
+def run_rpmostree(workdir, commit, treefile, extensions):
+    cmdlib.cmdlib_sh(f'''
+        changed_stamp={workdir}/changed
+        runcompose_extensions {workdir}/output {treefile} {extensions} \
+            --base-rev {commit}''')
+    return os.path.exists(f'{workdir}/changed')
+
+
+def create_yumrepo(repodir):
+    cmdlib.run_verbose(['createrepo_c', repodir])
+    # we could also have rpm-ostree output the pkglist for us, but meh... we
+    # need to run createrepo_c anyway and it's nice that we're using it as the
+    # source of truth, since that's what rpm-ostree clients will also use
+    repomd = cr.Repomd(os.path.join(repodir, "repodata/repomd.xml"))
+    pkglist = {}
+
+    def cb(pkg):
+        epoch = ''
+        if pkg.epoch and int(pkg.epoch) > 0:
+            epoch = f'{pkg.epoch}:'
+        pkglist[pkg.name] = f'{epoch}{pkg.version}-{pkg.release}.{pkg.arch}'
+
+    for record in repomd.records:
+        if record.type == 'primary':
+            primary_xml = os.path.join(repodir, record.location_href)
+            cr.xml_parse_primary(primary_xml, do_files=False, pkgcb=cb)
+            break
+
+    if len(pkglist) == 0:
+        raise Exception("No RPMs found in output dir")
+    return pkglist
+
+
+def create_tarball(buildmeta, srcdir, destdir):
+    destdir = os.path.abspath(destdir)
+    basearch = buildmeta['coreos-assembler.basearch']
+    tarfile = f'{destdir}/{buildmeta["name"]}-{buildmeta["buildid"]}-extensions.{basearch}.tar'
+    cmdlib.run_verbose(['tar', '-cf', tarfile, '.'], cwd=srcdir)
+    return tarfile
+
+
+if __name__ == '__main__':
+    sys.exit(main())

src/cmd-upload-oscontainer

@@ -40,23 +40,24 @@ metapath = f"{latest_build_path}/meta.json"
 with open(metapath) as f:
     meta = json.load(f)
 
+# for backcompat, we auto-build extensions if they're missing
+if os.path.exists('src/config/extensions.yaml'):
+    if 'extensions' not in meta:
+        cmdlib.run_verbose(['coreos-assembler', 'buildextend-extensions'])
+        with open(metapath) as f:
+            meta = json.load(f)
+    assert 'extensions' in meta
+
 configdir = os.path.abspath('src/config')
 oscconfigpath = f'{configdir}/oscontainer.yaml'
+# XXX: fold oscontainer.yaml handling into oscontainer.py
+configyaml = {}
 if os.path.exists(oscconfigpath):
     with open(oscconfigpath) as f:
-        c = yaml.safe_load(f)
-        base = c.get('base')
-        if base is not None:
-            args.from_image = base
-
-extensions_src = 'src/config/extensions.yaml'
-extensions_destdir = None
-if os.path.exists(extensions_src):
-    extensions_destdir = 'tmp/extensions'
-    if os.path.exists(extensions_destdir):
-        shutil.rmtree(extensions_destdir)
-    os.mkdir(extensions_destdir)
-    cmdlib.run_verbose(['/usr/lib/coreos-assembler/download-extensions', extensions_destdir])
+        configyaml = yaml.safe_load(f)
+
+if 'base' in configyaml:
+    args.from_image = configyaml['base']
 
 print("Preparing to upload oscontainer for build: {}".format(latest_build))
 ostree_commit = meta['ostree-commit']
@@ -106,9 +107,10 @@ os.environ['REGISTRY_AUTH_FILE'] = authfile
 cosa_argv.extend(['/usr/lib/coreos-assembler/oscontainer.py', '--workdir=./tmp', 'build',  f"--from={args.from_image}"])
 for d in args.add_directory:
     cosa_argv.append(f"--add-directory={d}")
-if extensions_destdir is not None:
-    cosa_argv.append(f"--add-directory={extensions_destdir}")
 cosa_argv.append(f"--display-name={display_name}")
+if 'labeled-packages' in configyaml:
+    pkgs = ' '.join(configyaml['labeled-packages'])
+    cosa_argv.append(f"--labeled-packages={pkgs}")
 subprocess.check_call(cosa_argv +
     [f'--digestfile={digestfile}',
         '--push', tmprepo,

src/cmdlib.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 # Shared shell script library
 
-DIR=$(dirname "$0")
+DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 RFC3339="%Y-%m-%dT%H:%M:%SZ"
 
 # Detect what platform we are on
@@ -381,6 +381,14 @@ runcompose_tree() {
     fi
 }
 
+runcompose_extensions() {
+    local outputdir=$1; shift
+    impl_rpmostree_compose extensions "$@" --output-dir "$outputdir"
+    if has_privileges; then
+        sudo chown -R -h "${USER}":"${USER}" "${outputdir}"
+    fi
+}
+
 impl_rpmostree_compose() {
     local cmd=$1; shift
     local workdir=${workdir:-$(pwd)}

src/cosalib/cmdlib.py

@@ -33,6 +33,8 @@
                       retry_if_exception_type(IncompleteReadError) |
                       retry_if_exception_type(ReadTimeoutError))
 
+THISDIR = os.path.dirname(os.path.abspath(__file__))
+
 
 def retry_callback(retry_state):
     print(f"Retrying after {retry_state.outcome.exception()}")
@@ -309,3 +311,12 @@ def image_info(image):
         return out
     except Exception as e:
         raise Exception(f"failed to inspect {image} with qemu", e)
+
+
+# Hackily run some bash code from cmdlib.sh helpers.
+def cmdlib_sh(script):
+    subprocess.check_call(['bash', '-c', f'''
+        set -euo pipefail
+        source {THISDIR}/../cmdlib.sh
+        {script}
+    '''])

src/deps.txt

@@ -13,7 +13,7 @@ sudo
 dumb-init
 
 # For composes
-rpm-ostree createrepo_c openssh-clients
+rpm-ostree createrepo_c openssh-clients python3-createrepo_c
 dnf-utils
 
 # For generating ISO images