From f9f32ddcc65909e7042b749d603e51a0a0ba47f4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 31 Jan 2024 20:36:43 -0500 Subject: [PATCH 1/7] qemu: Detect raw format too Raw format is fine to use on systems that have reflinks for example. --- mantle/platform/qemu.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index a9841597c9..f61dfeb32a 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -1071,6 +1071,8 @@ func (disk *Disk) prepare(builder *QemuBuilder) error { // on our own. if strings.HasSuffix(backingFile, "qcow2") { format = "qcow2" + } else if strings.HasSuffix(backingFile, "raw") { + format = "raw" } } if format != "" { From 4cd82893ad9c9a0b2c8e55d07cb503748d6640f5 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 13 Feb 2024 11:38:18 -0500 Subject: [PATCH 2/7] supermin: don't cd $workdir in supermin-init-prelude I've been investigating why a seemingly innocuous change (changing compression on OSBuild generated qemu qcow2) would cause disk images to not boot [1]. I think I have found the issue. I was first trying to make sure 100% that the files got written out over the virtiofs mount before the VM got shutdown so I decided to add a `umount $workdir` to the process. But this ended up with a `umount: /srv/: target is busy.` error. When the supermin VM gets run we `cd "${workdir}"` at the end of supermin-init-prelude.sh. This has the effect of causing all spawned processes (including PID1/init) to have a cwd of /srv/. ``` bash-5.2# lsof /srv COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME init 1 root cwd DIR 0,26 4096 10485829 /srv kthreadd 2 root cwd DIR 0,26 4096 10485829 /srv pool_work 3 root cwd DIR 0,26 4096 10485829 /srv kworker/R 4 root cwd DIR 0,26 4096 10485829 /srv ... ... ``` Which means it's unlikely that the virtiofs mount ever gets cleanly unmounted. Let's rework things here so that actual work gets spawned in a subshell to prevent `init` from having a cwd on the virtiofs mount. We also add in an `umount` of the cache qcow2 (if exists) and the virtiofs mount to strengthen our chances of a clean unmount. [1] https://github.com/coreos/coreos-assembler/issues/3728 --- src/cmdlib.sh | 15 +++++++++++---- src/supermin-init-prelude.sh | 3 --- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index be616a912d..986e3f7483 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -753,17 +753,24 @@ export USER=$(id -u) export RUNVM_NONET=${RUNVM_NONET:-} $(cat "${DIR}"/supermin-init-prelude.sh) rc=0 -# tee to the virtio port so its output is also part of the supermin output in -# case e.g. a key msg happens in dmesg when the command does a specific operation +# - tee to the virtio port so its output is also part of the supermin output in +# case e.g. a key msg happens in dmesg when the command does a specific operation. +# - Use a subshell because otherwise init will use workdir as its cwd and we won't +# be able to unmount the virtiofs mount cleanly. This leads to consistency issues. if [ -z "${RUNVM_SHELL:-}" ]; then - bash ${tmp_builddir}/cmd.sh |& tee /dev/virtio-ports/cosa-cmdout || rc=\$? + (cd ${workdir}; bash ${tmp_builddir}/cmd.sh |& tee /dev/virtio-ports/cosa-cmdout) || rc=\$? else - bash; poweroff -f -f; sleep infinity + (cd ${workdir}; bash) fi echo \$rc > ${rc_file} if [ -n "\${cachedev}" ]; then /sbin/fstrim -v ${workdir}/cache + mount -o remount,ro ${workdir}/cache + fsfreeze -f ${workdir}/cache + fsfreeze -u ${workdir}/cache + umount ${workdir}/cache fi +umount ${workdir} /sbin/reboot -f EOF chmod a+x "${vmpreparedir}"/init diff --git a/src/supermin-init-prelude.sh b/src/supermin-init-prelude.sh index 8cf8150d50..24eed7ea76 100644 --- a/src/supermin-init-prelude.sh +++ b/src/supermin-init-prelude.sh @@ -69,6 +69,3 @@ touch /etc/cosa-supermin # the missing link. Hehe. update-alternatives --install /etc/alternatives/iptables iptables /usr/sbin/iptables-legacy 1 update-alternatives --install /etc/alternatives/ip6tables ip6tables /usr/sbin/ip6tables-legacy 1 - -# https://github.com/koalaman/shellcheck/wiki/SC2164 -cd "${workdir}" || exit From 0d8f2beb1dfa9ac64b2e698b40f1e7661cb7f935 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 13 Feb 2024 21:49:41 -0500 Subject: [PATCH 3/7] osbuild: don't use supermin VM rootfs for outdir Just in case this is what is causing issues with file consistency when copying out of the supermin VM. --- src/runvm-osbuild | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/runvm-osbuild b/src/runvm-osbuild index 348f5925cb..1a651d3e18 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -65,10 +65,11 @@ fi # Since it doesn't exist create loop-control mknod /dev/loop-control c 10 237 -# Tell osbuild to write out artifacts into a file in the root -# filesystem of the supermin VM, which is ephemeral. -mkdir /var/osbuild -outdir=/var/osbuild/out +# Put the store and the output dir on the cache. At the end we'll mv +# out the created artifact from the output dir to the place it's supposed +# to go. +outdir=cache/osbuild/out +storedir=cache/osbuild/store # Run through the preprocessor # Note: don't quote the size arguements since they are numbers, not strings @@ -82,17 +83,18 @@ osbuild-mpp \ -D metal_image_size_mb="${metal_image_size_mb}" \ -D cloud_image_size_mb="${cloud_image_size_mb}" \ "${mppyaml}" \ - /var/osbuild/processed.json + /processed.json # Build the image osbuild \ - --out "$outdir" \ - --store cache/osbuild/store/ \ + --out "$outdir" \ + --store "$storedir" \ --cache-max-size 9GiB \ --checkpoint tree \ --checkpoint raw-image \ - --export "$platform" /var/osbuild/processed.json + --export "$platform" /processed.json -# Copy it out to the specified location -cp "${outdir}/${platform}/${filename}" "${filepath}" +# Copy it out to the specified location. Use mv here so we remove it +# from the cache qcow2 so we don't cache it. +mv "${outdir}/${platform}/${filename}" "${filepath}" From ab75c46cfad7f273ccb08aa04d306840ded530eb Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Wed, 14 Feb 2024 15:33:03 -0500 Subject: [PATCH 4/7] osbuild: updated to v108 And drop all patches that have now been upstreamed. The only remaining patches are one to enable s390x builds to work while we figure out [1] and another that adds a log statement when cache eviction happens, which I plan to upstream at some point. [1] https://github.com/coreos/fedora-coreos-tracker/issues/1667 --- build.sh | 20 +- ...d-new-FsCache._last_used_objs-helper.patch | 69 --- ...eployment-Fix-ostree-deployment-call.patch | 35 -- ...-ostree.deployment-rework-unmounting.patch | 135 ------ ...nience-function-for-using-default-OS.patch | 437 ------------------ ...ges-copy-allow-copying-from-the-tree.patch | 72 --- ...ld.qemu-make-qcow2-compression-optio.patch | 73 --- ...nfig-support-setting-sysroot.bootpre.patch | 41 -- ...ined-partition-numbers-for-GPT-disks.patch | 250 ---------- ...sCache._remove_lru-to-remove-entries.patch | 89 ---- ...ployment-rename-var-root-deploy_root.patch | 49 -- ...2-stages-sgdisk-support-label-option.patch | 32 -- ...nst-support-appending-kernel-options.patch | 52 --- ...ve_lru-to-reclaim-space-when-the-cac.patch | 79 ---- ...eployment-use-target-instead-of-tree.patch | 71 --- ...ployment-support-deployments-on-moun.patch | 149 ------ ...eate-stages-org.osbuild.chattr-stage.patch | 123 ----- src/cmdlib.sh | 7 - 18 files changed, 2 insertions(+), 1781 deletions(-) delete mode 100644 src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch delete mode 100644 src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch delete mode 100644 src/0001-mounts-ostree.deployment-rework-unmounting.patch delete mode 100644 src/0001-ostree-add-convenience-function-for-using-default-OS.patch delete mode 100644 src/0001-stages-copy-allow-copying-from-the-tree.patch delete mode 100644 src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch delete mode 100644 src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch delete mode 100644 src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch delete mode 100644 src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch delete mode 100644 src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch delete mode 100644 src/0002-stages-sgdisk-support-label-option.patch delete mode 100644 src/0002-stages-zipl.inst-support-appending-kernel-options.patch delete mode 100644 src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch delete mode 100644 src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch delete mode 100644 src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch delete mode 100644 src/0005-Create-stages-org.osbuild.chattr-stage.patch diff --git a/build.sh b/build.sh index e1d808988a..f8e50bdd31 100755 --- a/build.sh +++ b/build.sh @@ -45,7 +45,7 @@ install_rpms() { frozendeps="" # We freeze the version for now since we're carrying patches. - frozendeps+=" $(echo osbuild{,-ostree,-selinux,-tools}-106-1.fc39.noarch)" + frozendeps+=" $(echo osbuild{,-ostree,-selinux,-tools}-108-1.fc39.noarch)" # First, a general update; this is best practice. We also hit an issue recently # where qemu implicitly depended on an updated libusbx but didn't have a versioned @@ -176,24 +176,8 @@ patch_osbuild() { mv /usr/bin/osbuild-mpp /usr/lib/osbuild/tools/ # Now all the software is under the /usr/lib/osbuild dir and we can patch - cat /usr/lib/coreos-assembler/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch \ - /usr/lib/coreos-assembler/0001-mounts-ostree.deployment-rework-unmounting.patch \ - /usr/lib/coreos-assembler/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch \ - /usr/lib/coreos-assembler/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch \ - /usr/lib/coreos-assembler/0004-mounts-ostree.deployment-support-deployments-on-moun.patch \ - /usr/lib/coreos-assembler/0005-Create-stages-org.osbuild.chattr-stage.patch \ - /usr/lib/coreos-assembler/0001-fscache-add-new-FsCache._last_used_objs-helper.patch \ - /usr/lib/coreos-assembler/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch \ - /usr/lib/coreos-assembler/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch \ - /usr/lib/coreos-assembler/0004-fscache-add-eviction-log-statement.patch \ - /usr/lib/coreos-assembler/0001-support-user-defined-partition-numbers-for-GPT-disks.patch \ - /usr/lib/coreos-assembler/0002-stages-sgdisk-support-label-option.patch \ + cat /usr/lib/coreos-assembler/0004-fscache-add-eviction-log-statement.patch \ /usr/lib/coreos-assembler/0001-stages-zipl.inst-improve-kernel-initrd-path-resoluti.patch \ - /usr/lib/coreos-assembler/0002-stages-zipl.inst-support-appending-kernel-options.patch \ - /usr/lib/coreos-assembler/0001-stages-copy-allow-copying-from-the-tree.patch \ - /usr/lib/coreos-assembler/0001-ostree-add-convenience-function-for-using-default-OS.patch \ - /usr/lib/coreos-assembler/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch \ - /usr/lib/coreos-assembler/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch \ | patch -d /usr/lib/osbuild -p1 # And then move the files back; supermin appliance creation will need it back diff --git a/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch b/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch deleted file mode 100644 index ecfcd36ac9..0000000000 --- a/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 4a9831fa889b073ccb14568c4571f57d28dfe84f Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Tue, 12 Dec 2023 21:20:42 +0100 -Subject: [PATCH 1/4] fscache: add new `FsCache._last_used_objs()' helper - -This commit adds a helper that can be used to get a sorted list -of cache entries. The list includes the name and the last_used -information. ---- - osbuild/util/fscache.py | 29 ++++++++++++++++++++++++++++- - 1 file changed, 28 insertions(+), 1 deletion(-) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index c8fc99aa..58c9a310 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -15,7 +15,7 @@ import json - import os - import subprocess - import uuid --from typing import Any, Dict, NamedTuple, Optional, Tuple, Union -+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union - - from osbuild.util import ctx, linux, rmrf - -@@ -101,6 +101,17 @@ class FsCacheInfo(NamedTuple): - return data - - -+class FsCacheObjectInfo(NamedTuple): -+ """ File System Cache object information -+ -+ This type represents information about a single cache object. The -+ last_used information is only guaranteed to be valid while the cache -+ is locked. -+ """ -+ name: str -+ last_used: float -+ -+ - class FsCache(contextlib.AbstractContextManager, os.PathLike): - """File System Cache - -@@ -1059,6 +1070,22 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - raise self.MissError() from None - raise e - -+ def _last_used_objs(self) -> List[FsCacheObjectInfo]: -+ """Return a list of FsCacheObjectInfo with name, last_used -+ information sorted by last_used time. -+ -+ Note that this function will be racy when used without a lock and -+ the caller needs to handle this. -+ """ -+ objs = [] -+ for name in os.listdir(self._path(self._dirname_objects)): -+ try: -+ last_used = self._last_used(name) -+ except (OSError, FsCache.MissError): -+ continue -+ objs.append(FsCacheObjectInfo(name=name, last_used=last_used)) -+ return sorted(objs, key=lambda obj: obj.last_used) -+ - @property - def info(self) -> FsCacheInfo: - """Query Cache Information --- -2.43.0 - diff --git a/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch b/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch deleted file mode 100644 index ba7075fae6..0000000000 --- a/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch +++ /dev/null @@ -1,35 +0,0 @@ -From c623ca19f418c7f6d1d9dbf7ac6a80973c3ac7ab Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Fri, 9 Feb 2024 22:19:35 -0500 -Subject: [PATCH] mount/ostree.deployment: Fix ostree deployment call - -We need to pass in the root of the ostree deployment which can -be the tree or the mount. Fixes e1cbf92 ---- - mounts/org.osbuild.ostree.deployment | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 9637b5b2..c72dec6e 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -125,7 +125,6 @@ class OSTreeDeploymentMount(mounts.MountService): - options = args["options"] - source = options.get("source", "tree") - deployment = options["deployment"] -- osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - # The user could specify either the tree or mountroot as the - # place where we want the deployment to be mounted. -@@ -134,6 +133,8 @@ class OSTreeDeploymentMount(mounts.MountService): - else: - target = tree - -+ osname, ref, serial = ostree.parse_deployment_option(target, deployment) -+ - # create a private mountpoint for the target path, which is - # needed in order to be able to move the deployment `root` - # mountpoint here, which is contained inside tree, since --- -2.43.0 - diff --git a/src/0001-mounts-ostree.deployment-rework-unmounting.patch b/src/0001-mounts-ostree.deployment-rework-unmounting.patch deleted file mode 100644 index 3bd7d13de6..0000000000 --- a/src/0001-mounts-ostree.deployment-rework-unmounting.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 0da68e9af5a7b148e9841ff2a48bffd96be79b72 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Tue, 9 Jan 2024 23:23:25 -0500 -Subject: [PATCH 1/5] mounts/ostree.deployment: rework unmounting -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This unwinds part of a25ae2b. The way the code ended up both -self.tree and self.mountpoint ended up pointing to the exactly -same path and so we'd end up doing two `umount -R` operations -on the same path. This ended up being a duplicate unmount. - -On Fedora 39 this yields an error like: - -``` -mount/ostree.deployment (org.osbuild.ostree.deployment): umount: /var/osbuild/store/stage/uuid-efaac9370d25455d9e8df6d847ecb5b3/data/tree: not mounted -mount/ostree.deployment (org.osbuild.ostree.deployment): Traceback (most recent call last): -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 136, in -mount/ostree.deployment (org.osbuild.ostree.deployment): main() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 132, in main -mount/ostree.deployment (org.osbuild.ostree.deployment): service.main() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/osbuild/host.py", line 252, in main -mount/ostree.deployment (org.osbuild.ostree.deployment): self.stop() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/osbuild/mounts.py", line 126, in stop -mount/ostree.deployment (org.osbuild.ostree.deployment): self.umount() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 125, in umount -mount/ostree.deployment (org.osbuild.ostree.deployment): subprocess.run(["umount", "-R", self.tree], -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/usr/lib64/python3.12/subprocess.py", line 571, in run -mount/ostree.deployment (org.osbuild.ostree.deployment): raise CalledProcessError(retcode, process.args, -mount/ostree.deployment (org.osbuild.ostree.deployment): subprocess.CalledProcessError: Command '['umount', '-R', '/var/osbuild/store/stage/uuid-efaac9370d25455d9e8df6d847ecb5b3/data/tree'] -' returned non-zero exit status 1. - -⏱ Duration: 103s -``` - -I think this was necessary because of a bug in util-linux that -mean some of the accounting information got out of date when -doing a `mount --move` operation, which we use here. I think this -bug (or bugs) is now fixed [1][2] in util-linux v2.39 (in Fedora 39), -which is now causing the above pasted error on F39. - -Let's just add code here that mentions the problem and workaround -it with a loop to keep unmounting (essentially what the umount -R -should have done to overmounted filesystems if the mountinfo/utab -was correct) and also mention when we should be able to drop this -workaround. - -[1] https://github.com/karelzak/util-linux/commit/a04149fbb7c1952da1194d1514e298ff07dbc7ca -[2] https://github.com/karelzak/util-linux/commit/8cf6c5075780598fe3b30e7a7753d8323d093e22 ---- - mounts/org.osbuild.ostree.deployment | 44 +++++++++++++++++++++------- - 1 file changed, 34 insertions(+), 10 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 5571cc1f..9046fb05 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -63,7 +63,6 @@ class OSTreeDeploymentMount(mounts.MountService): - def __init__(self, args): - super().__init__(args) - -- self.tree = None - self.mountpoint = None - self.check = False - -@@ -72,7 +71,22 @@ class OSTreeDeploymentMount(mounts.MountService): - subprocess.run([ - "mount", "--bind", "--make-private", source, target, - ], check=True) -- return target -+ -+ def is_mounted(self): -+ # Use `mountpoint` command here to determine if the mountpoint is mounted. -+ # We would use os.path.ismount() here but that only works if a device is -+ # mounted (i.e. it doesn't use the mountinfo file in the heuristic and -+ # thus things like `mount --move` wouldn't show up). The exit codes from -+ # `mountpoint` are: -+ # -+ # 0 success; the directory is a mountpoint, or device is block device on --devno -+ # 1 failure; incorrect invocation, permissions or system error -+ # 32 failure; the directory is not a mountpoint, or device is not a block device on --devno -+ # -+ cp = subprocess.run(["mountpoint", "-q", self.mountpoint], check=False) -+ if cp.returncode not in [0, 32]: -+ cp.check_returncode() # will raise error -+ return cp.returncode == 0 - - def mount(self, args: Dict): - -@@ -89,7 +103,7 @@ class OSTreeDeploymentMount(mounts.MountService): - # is contained inside tree, since "moving a mount residing - # under a shared mount is invalid and unsupported." - # - `mount(8)` -- self.tree = self.bind_mount(tree, tree) -+ self.bind_mount(tree, tree) - - root = ostree.deployment_path(tree, osname, ref, serial) - -@@ -116,15 +130,25 @@ class OSTreeDeploymentMount(mounts.MountService): - if self.mountpoint: - subprocess.run(["sync", "-f", self.mountpoint], - check=self.check) -- -- subprocess.run(["umount", "-R", self.mountpoint], -+ subprocess.run(["umount", "-v", "-R", self.mountpoint], - check=self.check) -- self.mountpoint = None - -- if self.tree: -- subprocess.run(["umount", "-R", self.tree], -- check=self.check) -- self.tree = None -+ # Handle bug in older util-linux mount where the -+ # mountinfo/utab wouldn't have updated information -+ # when mount --move is performed, which means that -+ # umount -R wouldn't unmount all overmounted mounts -+ # on the target because it was operating on outdated -+ # information. The umount -R behavior is fixed in v2.39 -+ # of util-linux most likely by [1] or [2] or both. This -+ # loop can be removed when all hosts we care about have -+ # moved to v2.39+. -+ # [1] https://github.com/karelzak/util-linux/commit/a04149fbb7c1952da1194d1514e298ff07dbc7ca -+ # [2] https://github.com/karelzak/util-linux/commit/8cf6c5075780598fe3b30e7a7753d8323d093e22 -+ while self.is_mounted(): -+ print(f"extra unmount {self.mountpoint}") -+ subprocess.run(["umount", "-v", self.mountpoint], -+ check=self.check) -+ self.mountpoint = None - - - def main(): --- -2.43.0 - diff --git a/src/0001-ostree-add-convenience-function-for-using-default-OS.patch b/src/0001-ostree-add-convenience-function-for-using-default-OS.patch deleted file mode 100644 index 40cf03af51..0000000000 --- a/src/0001-ostree-add-convenience-function-for-using-default-OS.patch +++ /dev/null @@ -1,437 +0,0 @@ -From e1cbf92673fa6aa52209d091150097a5961a0b21 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Thu, 25 Jan 2024 13:57:51 -0500 -Subject: [PATCH] ostree: add convenience function for using default OSTree - deployment - -This adds a `default: true` option for all cases where OSTree -information is specified in schemas and allows for the information -to be picked up from the filesystem. - -This is a safe operation because when building disk images there is -no known case where having two deployments makes sense. In the case -there ever were a case then the osname, ref, and serial options still -exist and can be used. - -Co-authored-by: Luke Yang -Co-authored-by: Michael Vogt ---- - mounts/org.osbuild.ostree.deployment | 31 ++++++++++++++++---- - osbuild/util/ostree.py | 42 ++++++++++++++++++++++++++-- - stages/org.osbuild.bootupd | 30 +++++++++++++++++--- - stages/org.osbuild.fstab | 30 +++++++++++++++++--- - stages/org.osbuild.ostree.aleph | 31 +++++++++++++++++--- - stages/org.osbuild.ostree.fillvar | 30 +++++++++++++++++--- - stages/org.osbuild.ostree.selinux | 30 +++++++++++++++++--- - 8 files changed, 217 insertions(+), 29 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 24df7731..9637b5b2 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -41,7 +41,26 @@ SCHEMA_2 = """ - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -55,6 +74,11 @@ SCHEMA_2 = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -99,12 +123,9 @@ class OSTreeDeploymentMount(mounts.MountService): - tree = args["tree"] - mountroot = args["root"] - options = args["options"] -- - source = options.get("source", "tree") - deployment = options["deployment"] -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - # The user could specify either the tree or mountroot as the - # place where we want the deployment to be mounted. -diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py -index 173a9676..4c0344d1 100644 ---- a/osbuild/util/ostree.py -+++ b/osbuild/util/ostree.py -@@ -1,13 +1,15 @@ - import collections - import contextlib -+import glob - import json - import os -+import re - import subprocess - import sys - import tempfile - import typing - # pylint doesn't understand the string-annotation below --from typing import Any, List # pylint: disable=unused-import -+from typing import Any, Dict, List, Tuple # pylint: disable=unused-import - - from osbuild.util.rhsm import Subscriptions - -@@ -214,7 +216,43 @@ def parse_input_commits(commits): - return commits["path"], data["refs"] - - --def deployment_path(root: PathLike, osname: str, ref: str, serial: int): -+def parse_deployment_option(root: PathLike, deployment: Dict) -> Tuple[str, str, str]: -+ """Parse the deployment option and return the osname, ref, and serial -+ -+ The `deployment` arg contains the following sub fields: -+ - osname: Name of the stateroot used in the deployment (ie. fedora-coreos) -+ - ref: OStree ref to used for the deployment (ie. fedora/aarch64/coreos/next) -+ - serial: The deployment serial (ie. 0) -+ - default: Boolean to determine whether the default ostree deployment should be used -+ """ -+ -+ default_deployment = deployment.get("default") -+ if default_deployment: -+ filenames = glob.glob(os.path.join(root, 'ostree/deploy/*/deploy/*.0')) -+ if len(filenames) < 1: -+ raise ValueError("Could not find deployment") -+ if len(filenames) > 1: -+ raise ValueError(f"More than one deployment found: {filenames}") -+ -+ # We pick up the osname, commit, and serial from the filesystem -+ # here. We'll return the detected commit as the ref in this -+ # since it's a valid substitute for all subsequent uses in -+ # the code base. -+ f = re.search("/ostree/deploy/(.*)/deploy/(.*)\\.([0-9])", filenames[0]) -+ if not f: -+ raise ValueError("cannot find ostree deployment in {filenames[0]}") -+ osname = f.group(1) -+ commit = f.group(2) -+ serial = f.group(3) -+ return osname, commit, serial -+ -+ osname = deployment["osname"] -+ ref = deployment["ref"] -+ serial = deployment.get("serial", 0) -+ return osname, ref, serial -+ -+ -+def deployment_path(root: PathLike, osname: str = "", ref: str = "", serial: int = 0): - """Return the path to a deployment given the parameters""" - - base = os.path.join(root, "ostree") -diff --git a/stages/org.osbuild.bootupd b/stages/org.osbuild.bootupd -index 224bd6d4..5663ea3e 100755 ---- a/stages/org.osbuild.bootupd -+++ b/stages/org.osbuild.bootupd -@@ -32,7 +32,26 @@ SCHEMA_2 = r""" - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -46,6 +65,11 @@ SCHEMA_2 = r""" - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - }, -@@ -101,9 +125,7 @@ def main(args, options): - # we'll call ostree.deployment_path() helper to find it for us. - root = mounts - if deployment: -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(mounts, deployment) - root = ostree.deployment_path(mounts, osname, ref, serial) - - bootupd_args = [] -diff --git a/stages/org.osbuild.fstab b/stages/org.osbuild.fstab -index eb5ac2b0..a9af5ffe 100755 ---- a/stages/org.osbuild.fstab -+++ b/stages/org.osbuild.fstab -@@ -29,7 +29,26 @@ SCHEMA = """ - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname","ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -43,6 +62,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -118,9 +142,7 @@ def main(tree, options): - - if ostree_options: - deployment = ostree_options["deployment"] -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - root = ostree.deployment_path(tree, osname, ref, serial) - -diff --git a/stages/org.osbuild.ostree.aleph b/stages/org.osbuild.ostree.aleph -index 6baedeb0..bcef1fe3 100755 ---- a/stages/org.osbuild.ostree.aleph -+++ b/stages/org.osbuild.ostree.aleph -@@ -19,6 +19,7 @@ COREOS_ALEPH_FILENAME = ".coreos-aleph-version.json" - SCHEMA_2 = """ - "options": { - "additionalProperties": false, -+ "required": ["deployment"], - "properties": { - "coreos_compat": { - "description": "boolean to allow for CoreOS aleph version backwards compatibility", -@@ -26,7 +27,26 @@ SCHEMA_2 = """ - }, - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -40,6 +60,11 @@ SCHEMA_2 = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -131,9 +156,7 @@ def construct_aleph_json(tree, origin): - def main(tree, options): - coreos_compat = options.get("coreos_compat", False) - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - origin = ostree.deployment_path(tree, osname, ref, serial) + ".origin" - data = construct_aleph_json(tree, origin) -diff --git a/stages/org.osbuild.ostree.fillvar b/stages/org.osbuild.ostree.fillvar -index e5b5eacd..dfb706e8 100755 ---- a/stages/org.osbuild.ostree.fillvar -+++ b/stages/org.osbuild.ostree.fillvar -@@ -18,7 +18,26 @@ SCHEMA = """ - "properties": { - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -32,6 +51,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -71,9 +95,7 @@ def populate_var(sysroot): - - def main(tree, options): - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - deployment = ostree.deployment_path(tree, osname, ref, serial) - var = os.path.join(tree, "ostree", "deploy", osname, "var") -diff --git a/stages/org.osbuild.ostree.selinux b/stages/org.osbuild.ostree.selinux -index d990a5e5..58663e34 100755 ---- a/stages/org.osbuild.ostree.selinux -+++ b/stages/org.osbuild.ostree.selinux -@@ -21,7 +21,26 @@ SCHEMA = """ - "properties": { - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -35,6 +54,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -44,9 +68,7 @@ SCHEMA = """ - - def main(tree, options): - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - # this created a state root at `osname` - stateroot = f"{tree}/ostree/deploy/{osname}" --- -2.43.0 - diff --git a/src/0001-stages-copy-allow-copying-from-the-tree.patch b/src/0001-stages-copy-allow-copying-from-the-tree.patch deleted file mode 100644 index 8cc0ad195a..0000000000 --- a/src/0001-stages-copy-allow-copying-from-the-tree.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 3ecec2e2848a51803e4a9953b132a6eaf4e5043c Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Thu, 1 Feb 2024 23:14:06 -0500 -Subject: [PATCH] stages(copy): allow copying from the tree - -It seems like an artifical limitation to prevent copying from one -location in the tree to another. It just so happens we need this -functionality when building CoreOS images because we want to take -a file embedded in the OSTree at a location and copy it to another -location in the tree. The particular example here is we want to copy -/usr/share/coreos-assembler/platforms.json -> /boot/coreos/platforms.json -See https://github.com/coreos/coreos-assembler/pull/3709 - -Allowing to copy from/to the tree we can now do something like: - -``` -- type: org.osbuild.copy - options: - paths: - - from: tree:///usr/share/coreos-assembler/platforms.json - to: tree:///boot/coreos/platforms.json - mounts: - - name: ostree.deployment - type: org.osbuild.ostree.deployment - options: - deployment: - ref: ostree/1/1/0 - osname: - fedora-coreos -``` ---- - stages/org.osbuild.copy | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/stages/org.osbuild.copy b/stages/org.osbuild.copy -index 3c137af5..d476efdb 100755 ---- a/stages/org.osbuild.copy -+++ b/stages/org.osbuild.copy -@@ -45,9 +45,18 @@ SCHEMA_2 = r""" - "required": ["from", "to"], - "properties": { - "from": { -- "type": "string", -- "description": "The source", -- "pattern": "^input:\/\/[^\/]+\/" -+ "oneOf": [ -+ { -+ "type": "string", -+ "description": "The source, if an input", -+ "pattern": "^input:\/\/[^\/]+\/" -+ }, -+ { -+ "type": "string", -+ "description": "The source, if the tree", -+ "pattern": "^tree:\/\/\/" -+ } -+ ] - }, - "to": { - "oneOf": [ -@@ -58,7 +67,7 @@ SCHEMA_2 = r""" - }, - { - "type": "string", -- "description": "The destination, if a tree", -+ "description": "The destination, if the tree", - "pattern": "^tree:\/\/\/" - } - ] --- -2.43.0 - diff --git a/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch b/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch deleted file mode 100644 index 108c3cfe16..0000000000 --- a/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 619a64f0bd7322628ae5fedf8b3a640e42f9edcd Mon Sep 17 00:00:00 2001 -From: Luke Yang -Date: Tue, 6 Feb 2024 13:54:15 -0500 -Subject: [PATCH] stages/org.osbuild.qemu: make qcow2 compression optional - -Modify the stages/org.osbuild.qemu stage such that compression is -optional. This resolves the image size differences between an image -built with coreos assember vs osbuild, as discussed in: -https://github.com/coreos/fedora-coreos-tracker/issues/1653#issuecomment-1928342241 ---- - stages/org.osbuild.qemu | 23 +++++++++++++++++++++-- - 1 file changed, 21 insertions(+), 2 deletions(-) - -diff --git a/stages/org.osbuild.qemu b/stages/org.osbuild.qemu -index 98a04f00..642b5146 100755 ---- a/stages/org.osbuild.qemu -+++ b/stages/org.osbuild.qemu -@@ -31,6 +31,11 @@ SCHEMA_2 = r""" - "type": "string", - "enum": ["qcow2"] - }, -+ "compression": { -+ "description": "Enable/disable compression of the qcow2 image", -+ "type": "boolean", -+ "default": true -+ }, - "compat": { - "description": "The qcow2-compatibility-version to use", - "type": "string" -@@ -61,6 +66,11 @@ SCHEMA_2 = r""" - "type": "string", - "enum": ["vmdk"] - }, -+ "compression": { -+ "description": "Enable/disable compression of the vmdk image", -+ "type": "boolean", -+ "default": true -+ }, - "subformat": { - "description": "VMDK flat extent format", - "type": "string", -@@ -140,17 +150,26 @@ SCHEMA_2 = r""" - - - def qcow2_arguments(options): -- argv = ["-c"] -+ argv = [] -+ compression = options.get("compression", True) - compat = options.get("compat") - -+ if compression: -+ argv += ["-c"] -+ - if compat: - argv += ["-o", f"compat={compat}"] - return argv - - - def vmdk_arguments(options): -- argv = ["-c"] -+ argv = [] -+ compression = options.get("compression", True) - subformat = options.get("subformat") -+ -+ if compression: -+ argv += ["-c"] -+ - if subformat: - argv += ["-o", f"subformat={subformat}"] - return argv --- -2.43.0 - diff --git a/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch b/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch deleted file mode 100644 index e4e6483154..0000000000 --- a/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 612d3abd2ba05072a27142d5197645b4049561ae Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Mon, 5 Feb 2024 23:27:59 -0500 -Subject: [PATCH] stages(ostree.config): support setting sysroot.bootprefix - -See https://github.com/ostreedev/ostree/pull/2705 and also -https://github.com/osbuild/osbuild/issues/1566. ---- - stages/org.osbuild.ostree.config | 9 +++++++++ - 1 file changed, 9 insertions(+) - -diff --git a/stages/org.osbuild.ostree.config b/stages/org.osbuild.ostree.config -index 7d5bb71d..5a91a8ae 100755 ---- a/stages/org.osbuild.ostree.config -+++ b/stages/org.osbuild.ostree.config -@@ -39,6 +39,10 @@ SCHEMA = """ - "type": "string", - "enum": ["none", "auto", "grub2", "syslinux", "uboot", "zipl", "aboot"] - }, -+ "bootprefix": { -+ "description": "If set to true, the bootloader entries generated will include /boot as a prefix.", -+ "type": "boolean" -+ }, - "readonly": { - "description": "Read only sysroot and boot", - "type": "boolean" -@@ -63,6 +67,11 @@ def main(tree, options): - if bootloader: - ostree.cli("config", "set", "sysroot.bootloader", bootloader, repo=repo) - -+ bootprefix = sysroot_options.get("bootprefix") -+ if bootprefix is not None: # can be False, which we would want to set -+ bp = "true" if bootprefix else "false" -+ ostree.cli("config", "set", "sysroot.bootprefix", bp, repo=repo) -+ - readonly = sysroot_options.get("readonly") - if readonly is not None: # can be False, which we would want to set - ro = "true" if readonly else "false" --- -2.43.0 - diff --git a/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch b/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch deleted file mode 100644 index 066d903d6c..0000000000 --- a/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch +++ /dev/null @@ -1,250 +0,0 @@ -From d84df08668b1e7350f9f151eff9d79817a1847ab Mon Sep 17 00:00:00 2001 -From: Nikita Dubrovskii -Date: Fri, 26 Jan 2024 14:18:21 +0100 -Subject: [PATCH 1/2] support user-defined partition numbers for GPT disks - -Partitions by default are indexed starting at 1, but in -some cases, such as CoreOS for IBM Z, it may be usefull -to set the 'partnum' for GPT disks explicitly, without -creating dummy partitions. - -Now user can define an image: - -``` - mpp-define-images: - - id: image - size: 10737418240 - table: - uuid: 00000000-0000-4000-a000-000000000001 - label: gpt - partitions: - - name: boot - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 3 - size: 786432 - - name: root - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 4 - size: 4194304 -``` - -So target disk would look like: - -``` - Disklabel type: gpt - Disk identifier: 00000000-0000-4000-A000-000000000001 - Device Start End Sectors Size Type - /dev/loop0p3 2048 788479 786432 384M Linux filesystem - /dev/loop0p4 788480 4982783 4194304 2G Linux filesystem -``` - -This patch updates the osbuild-mpp tool and the sgdisk and sfdisk -stages to support this. - -Co-authored-by: Dusty Mabe ---- - stages/org.osbuild.sfdisk | 19 ++++++++++++++++--- - stages/org.osbuild.sgdisk | 9 ++++++++- - tools/osbuild-mpp | 18 +++++++++++++----- - 3 files changed, 37 insertions(+), 9 deletions(-) - -diff --git a/stages/org.osbuild.sfdisk b/stages/org.osbuild.sfdisk -index 9887a02e..108866c4 100755 ---- a/stages/org.osbuild.sfdisk -+++ b/stages/org.osbuild.sfdisk -@@ -4,6 +4,7 @@ Partition a target using sfdisk(8) - """ - - import json -+import re - import subprocess - import sys - from typing import Optional -@@ -50,6 +51,10 @@ SCHEMA_2 = r""" - "description": "The partition name (GPT)", - "type": "string" - }, -+ "partnum": { -+ "description": "The partition number", -+ "type": "integer" -+ }, - "size": { - "description": "The size of this partition", - "type": "integer" -@@ -88,6 +93,7 @@ SCHEMA_2 = r""" - class Partition: - def __init__(self, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -95,13 +101,15 @@ class Partition: - uuid: str = None, - attrs: int = None): - self.type = pttype -+ self.partnum = partnum - self.start = start - self.size = size - self.bootable = bootable - self.name = name - self.uuid = uuid -- self.index = None - self.attrs = attrs -+ self.index = partnum - 1 if partnum else None -+ self.partnum = partnum if partnum else None - - @property - def start_in_bytes(self): -@@ -167,7 +175,10 @@ class PartitionTable: - fields += [f'{field}="{value}"'] - if partition.bootable: - fields += ["bootable"] -- command += "\n" + ", ".join(fields) -+ if partition.partnum: -+ command += "\n" + f'{target}p{partition.partnum}: ' + ", ".join(fields) -+ else: -+ command += "\n" + ", ".join(fields) - - print(command) - -@@ -190,7 +201,8 @@ class PartitionTable: - - assert len(disk_parts) == len(self.partitions) - for i, part in enumerate(self.partitions): -- part.index = i -+ part.partnum = int(re.findall(r'\d+$', disk_parts[i]["node"])[0]) -+ part.index = part.partnum - 1 - part.start = disk_parts[i]["start"] - part.size = disk_parts[i]["size"] - part.type = disk_parts[i].get("type") -@@ -200,6 +212,7 @@ class PartitionTable: - - def partition_from_json(js) -> Partition: - p = Partition(pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -diff --git a/stages/org.osbuild.sgdisk b/stages/org.osbuild.sgdisk -index e11e0119..1627a90f 100755 ---- a/stages/org.osbuild.sgdisk -+++ b/stages/org.osbuild.sgdisk -@@ -50,6 +50,10 @@ SCHEMA_2 = r""" - "description": "The partition name", - "type": "string" - }, -+ "partnum": { -+ "description": "The partition number", -+ "type": "integer" -+ }, - "size": { - "description": "The size of this partition", - "type": "integer" -@@ -88,6 +92,7 @@ SCHEMA_2 = r""" - class Partition: - def __init__(self, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -95,6 +100,7 @@ class Partition: - uuid: str = None, - attrs: int = None): - self.type = pttype -+ self.partnum = partnum - self.start = start - self.size = size - self.name = name -@@ -129,7 +135,7 @@ class PartitionTable: - command += ["-U", self.uuid] - - for i, part in enumerate(self.partitions): -- idx = i + 1 # partitions are 1-indexed -+ idx = part.partnum if part.partnum else i + 1 # partitions are 1-indexed - - # format is 'partnum:start:end' - size = "0" -@@ -173,6 +179,7 @@ class PartitionTable: - - def partition_from_json(js) -> Partition: - p = Partition(pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp -index cf74488e..7bf93a18 100755 ---- a/tools/osbuild-mpp -+++ b/tools/osbuild-mpp -@@ -357,6 +357,7 @@ import hashlib - import json - import os - import pathlib -+import re - import string - import subprocess - import sys -@@ -815,6 +816,7 @@ class Partition: - def __init__(self, - uid: str = None, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -830,8 +832,8 @@ class Partition: - self.name = name - self.uuid = uuid - self.attrs = attrs -- self.index = None -- self.partnum = None -+ self.index = partnum - 1 if partnum else None -+ self.partnum = partnum if partnum else None - - @property - def start_in_bytes(self): -@@ -845,6 +847,7 @@ class Partition: - def from_dict(cls, js): - p = cls(uid=js.get("id"), - pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -@@ -858,6 +861,8 @@ class Partition: - - if self.start: - data["start"] = self.start -+ if self.partnum: -+ data["partnum"] = self.partnum - if self.size: - data["size"] = self.size - if self.type: -@@ -915,7 +920,10 @@ class PartitionTable: - fields += [f'{field}="{value}"'] - if partition.bootable: - fields += ["bootable"] -- command += "\n" + ", ".join(fields) -+ if partition.partnum: -+ command += "\n" + f'{target}p{partition.partnum}: ' + ", ".join(fields) -+ else: -+ command += "\n" + ", ".join(fields) - - subprocess.run(["sfdisk", "-q", "--no-tell-kernel", target], - input=command, -@@ -936,8 +944,8 @@ class PartitionTable: - - assert len(disk_parts) == len(self.partitions) - for i, part in enumerate(self.partitions): -- part.index = i -- part.partnum = i + 1 -+ part.partnum = int(re.findall(r'\d+$', disk_parts[i]["node"])[0]) -+ part.index = part.partnum - 1 - part.start = disk_parts[i]["start"] - part.size = disk_parts[i]["size"] - part.type = disk_parts[i].get("type") --- -2.43.0 - diff --git a/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch b/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch deleted file mode 100644 index 3bd536a4b8..0000000000 --- a/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch +++ /dev/null @@ -1,89 +0,0 @@ -From fff7dd5e1ab967165c16edec66b989adc70e39b7 Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Wed, 13 Dec 2023 10:07:32 +0100 -Subject: [PATCH 2/4] fscache: add FsCache._remove_lru() to remove entries - -The FsCache._remove_lru() removes the least recently used entry -from the cache. ---- - osbuild/util/fscache.py | 64 +++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 64 insertions(+) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index 58c9a310..32056a35 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -1086,6 +1086,70 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - objs.append(FsCacheObjectInfo(name=name, last_used=last_used)) - return sorted(objs, key=lambda obj: obj.last_used) - -+ def _remove_lru(self, required_size): -+ """" -+ Make room in the cache for "required_size" by remove the least -+ recently used entry from the cache. Note that the cache may -+ clear more than required_size. -+ """ -+ # To avoid having to take a global cache lock the strategy is: -+ # 1. Get list of (object, last_used) sorted from oldest to newest. -+ # This is racy so we need to take care of that in step(2). -+ # 2. Start with the oldest entry, try to take a write_lock -+ # (with O_NOATIME to be extra sure that atime information is -+ # correct). Get the "last_used" (atime) time and compare to what -+ # we expect in the list. If it diverges the object got load()ed -+ # while we iterated. Skip it and go to (2) again. -+ # 3. Remove entry, update cache size after the entry is removed. -+ # -+ # Note that there is a risk to get out-of-sync in (3). If the -+ # process dies while removing and before updating the cache -+ # size the cache will be over reported. -+ -+ # Try to clean at least twice the requested size to avoid having -+ # to do this all over again -+ try_to_free = required_size * 2 -+ freed_so_far = 0 -+ for name, last_used in self._last_used_objs(): -+ # take write lock for the indivdual object -+ rpath = os.path.join(self._dirname_objects, name) -+ rpath_lock = os.path.join(rpath, self._filename_object_lock) -+ # Ideally there would some lock helper instead of the low-level -+ # file manipulation to abstract this a bit more. -+ try: -+ with self._atomic_open( -+ rpath_lock, -+ wait=False, -+ write=True, -+ # atime carries the "last-used" data so don't alter it -+ oflags=os.O_EXCL | os.O_NOATIME, -+ ): -+ if last_used != self._last_used(name): -+ continue -+ # This is racy right now if the process is killed -+ # during "_rm_r_object(rpath)" because then the -+ # cache size is never reduced by the amount that -+ # was about to be deleted. -+ # -+ # To fix it we need to (atomic) rename the -+ # "object.info" file in _rm_r_object() to -+ # something like "object.removing". Then when -+ # opening the cache scan for leftover -+ # "object.removing" files and finish the cleanup -+ # and update the cache size based on the size -+ # recorded inside "object.removing". -+ size = self._calculate_space(self._path(rpath)) -+ self._rm_r_object(rpath) -+ self._update_cache_size(-size) -+ freed_so_far += size -+ if freed_so_far >= try_to_free: -+ break -+ except BlockingIOError: -+ continue -+ -+ # return True if at least the required size got freed -+ return freed_so_far > required_size -+ - @property - def info(self) -> FsCacheInfo: - """Query Cache Information --- -2.43.0 - diff --git a/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch b/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch deleted file mode 100644 index 7cde89da71..0000000000 --- a/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch +++ /dev/null @@ -1,49 +0,0 @@ -From e43abe1a9cbff3508eebc31d33efdc2fa44306aa Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 00:02:38 -0500 -Subject: [PATCH 2/5] mounts/ostree.deployment: rename var root -> deploy_root - -It makes things a little more clear to know the variable is pointing -to the path of the deployment. ---- - mounts/org.osbuild.ostree.deployment | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 9046fb05..f23a44da 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -105,22 +105,22 @@ class OSTreeDeploymentMount(mounts.MountService): - # - `mount(8)` - self.bind_mount(tree, tree) - -- root = ostree.deployment_path(tree, osname, ref, serial) -+ deploy_root = ostree.deployment_path(tree, osname, ref, serial) - -- print(f"Deployment root at '{os.path.relpath(root, tree)}'") -+ print(f"Deployment root at '{os.path.relpath(deploy_root, tree)}'") - - var = os.path.join(tree, "ostree", "deploy", osname, "var") - boot = os.path.join(tree, "boot") - -- self.mountpoint = root -- self.bind_mount(root, root) # prepare to move it later -+ self.mountpoint = deploy_root -+ self.bind_mount(deploy_root, deploy_root) # prepare to move it later - -- self.bind_mount(tree, os.path.join(root, "sysroot")) -- self.bind_mount(var, os.path.join(root, "var")) -- self.bind_mount(boot, os.path.join(root, "boot")) -+ self.bind_mount(tree, os.path.join(deploy_root, "sysroot")) -+ self.bind_mount(var, os.path.join(deploy_root, "var")) -+ self.bind_mount(boot, os.path.join(deploy_root, "boot")) - - subprocess.run([ -- "mount", "--move", root, tree, -+ "mount", "--move", deploy_root, tree, - ], check=True) - - self.mountpoint = tree --- -2.43.0 - diff --git a/src/0002-stages-sgdisk-support-label-option.patch b/src/0002-stages-sgdisk-support-label-option.patch deleted file mode 100644 index f66b4aa118..0000000000 --- a/src/0002-stages-sgdisk-support-label-option.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 92b7f81e1dde7c760327d0cc2170d6ce225e093b Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Tue, 30 Jan 2024 13:39:46 -0500 -Subject: [PATCH 2/2] stages(sgdisk): support label option - -We only support `gpt` here so it would seem this option doesn't -make much sense to add, but it will make it so that the mpp-define-images -from osbuild-mpp can be passed in to `org.osbuild.sgdisk` too as well -as `org.osbuild.sfdisk`. ---- - stages/org.osbuild.sgdisk | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/stages/org.osbuild.sgdisk b/stages/org.osbuild.sgdisk -index 1627a90f..f85a762a 100755 ---- a/stages/org.osbuild.sgdisk -+++ b/stages/org.osbuild.sgdisk -@@ -35,6 +35,11 @@ SCHEMA_2 = r""" - "type": "boolean", - "default": true - }, -+ "label": { -+ "description": "The type of the partition table. Only 'gpt' supported.", -+ "type": "string", -+ "enum": ["gpt"] -+ }, - "partitions": { - "description": "Partition layout ", - "type": "array", --- -2.43.0 - diff --git a/src/0002-stages-zipl.inst-support-appending-kernel-options.patch b/src/0002-stages-zipl.inst-support-appending-kernel-options.patch deleted file mode 100644 index 211f7f98f6..0000000000 --- a/src/0002-stages-zipl.inst-support-appending-kernel-options.patch +++ /dev/null @@ -1,52 +0,0 @@ -From 564e1e4841c914767ab2890bfe293cc397c4fca6 Mon Sep 17 00:00:00 2001 -From: Nikita Dubrovskii -Date: Tue, 30 Jan 2024 09:09:58 +0100 -Subject: [PATCH 2/2] stages(zipl.inst): support appending kernel options - -In some cases it could useful to add additional kernel options without -modifying BLS config, so any subsequent call to `zipl` would ignore them. -We are going to use this to append `ignition.firstboot` to kernel cmdline -for CoreOS on s390x. ---- - stages/org.osbuild.zipl.inst | 11 ++++++++++- - 1 file changed, 10 insertions(+), 1 deletion(-) - -diff --git a/stages/org.osbuild.zipl.inst b/stages/org.osbuild.zipl.inst -index 92dfd33e..96cf8cff 100755 ---- a/stages/org.osbuild.zipl.inst -+++ b/stages/org.osbuild.zipl.inst -@@ -28,6 +28,14 @@ SCHEMA_2 = r""" - "location": { - "type": "number" - }, -+ "kernel_opts_append": { -+ "description": "Additional kernel options to add to the discovered options", -+ "type": "array", -+ "items": { -+ "description": "A single kernel command line option", -+ "type": "string" -+ } -+ }, - "blocksize": { - "type": "number", - "default": 512 -@@ -93,6 +101,7 @@ def find_kernel(root, kernel: str): - def main(paths, devices, options): - kver = options["kernel"] - location = options["location"] -+ append_kopts = options.get("kernel_opts_append", []) - blocksize = options.get("blocksize", 512) - - root = paths["mounts"] -@@ -108,7 +117,7 @@ def main(paths, devices, options): - "--target", f"{root}/boot", - "--image", kernel, - "--ramdisk", initrd, -- "--parameters", kopts, -+ "--parameters", f"{kopts} {' '.join(append_kopts)}", - "--targetbase", device, - "--targettype", "SCSI", - "--targetblocksize", str(blocksize), --- -2.43.0 - diff --git a/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch b/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch deleted file mode 100644 index abb23f2088..0000000000 --- a/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch +++ /dev/null @@ -1,79 +0,0 @@ -From 6ac1d20c8c52f3cbc6d890ed8a008f8f2d4c147b Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Wed, 13 Dec 2023 17:39:55 +0100 -Subject: [PATCH 3/4] fscache: use remove_lru() to reclaim space when the cache - is full - -This commit adds code that will remove the least recently used -entries when a store() operation does not succeeds because the -cache is full. To be more efficient it will try to free -twice the requested size (this can be configured in the code). ---- - osbuild/util/fscache.py | 32 +++++++++++++++++++++----------- - 1 file changed, 21 insertions(+), 11 deletions(-) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index 32056a35..59039522 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -683,8 +683,11 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - self._info_maximum_size = -1 - elif isinstance(info.maximum_size, int): - self._info_maximum_size = info.maximum_size -- else: -+ elif info.maximum_size is None: - self._info_maximum_size = 0 -+ else: -+ raise ValueError( -+ f"maximum-size can only be set to 'unlimited' or an integer value, got {type(info.maximum_size)}") - - def _is_active(self): - # Internal helper to verify we are in an active context-manager. -@@ -942,19 +945,27 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - info["creation-boot-id"] = self._bootid - info["size"] = self._calculate_space(path_data) - -- # Update the total cache-size. If it exceeds the limits, bail out -- # but do not trigger an error. It behaves as if the entry was -- # committed and immediately deleted by racing cache management. No -- # need to tell the caller about it (if that is ever needed, we can -- # provide for it). -+ # Exit early if it never is going to fit -+ if self._info_maximum_size > -1 and info["size"] > self._info_maximum_size: -+ return -+ -+ # Update the total cache-size. If it exceeds the limits, remove -+ # least recently used objects until there is enough space. - # - # Note that if we crash after updating the total cache size, but - # before committing the object information, the total cache size -- # will be out of sync. However, it is never overcommitted, so we -- # will never violate any cache invariants. The cache-size will be -- # re-synchronized by any full cache-management operation. -+ # will be out of sync. -+ # -+ # However, it is never overcommitted, so we will never -+ # violate any cache invariants. Future code needs to resync -+ # the cache (e.g. on open with some simple journal strategy). - if not self._update_cache_size(info["size"]): -- return -+ # try to free space -+ self._remove_lru(info["size"]) -+ # and see if the update can happen now -+ if not self._update_cache_size(info["size"]): -+ # stil could not free enough space -+ return - - try: - # Commit the object-information, thus marking it as fully -@@ -1146,7 +1157,6 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - break - except BlockingIOError: - continue -- - # return True if at least the required size got freed - return freed_so_far > required_size - --- -2.43.0 - diff --git a/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch b/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch deleted file mode 100644 index f1dd6f7b42..0000000000 --- a/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch +++ /dev/null @@ -1,71 +0,0 @@ -From be90d8c36c0f0a6846a475f394843e898ad35a24 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 00:12:01 -0500 -Subject: [PATCH 3/5] mounts/ostree.deployment: use target instead of tree - -We still target the tree here, but we open ourselves up to be able -to target something other than the tree in the future. This mostly -exchanges the `tree` variable for `target`. - -We also update the comment to try to enhance clarity. ---- - mounts/org.osbuild.ostree.deployment | 29 ++++++++++++++++------------ - 1 file changed, 17 insertions(+), 12 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index f23a44da..b200eca3 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -98,32 +98,37 @@ class OSTreeDeploymentMount(mounts.MountService): - ref = deployment["ref"] - serial = deployment.get("serial", 0) - -- # create a private mountpoint for the tree, which is needed -- # in order to be able to move the `root` mountpoint, which -- # is contained inside tree, since "moving a mount residing -- # under a shared mount is invalid and unsupported." -+ # The target path where we want the deployment to be mounted -+ # is the root of the tree. -+ target = tree -+ -+ # create a private mountpoint for the target path, which is -+ # needed in order to be able to move the deployment `root` -+ # mountpoint here, which is contained inside tree, since -+ # "moving a mount residing under a shared mount is invalid -+ # and unsupported." - # - `mount(8)` -- self.bind_mount(tree, tree) -+ self.bind_mount(target, target) - -- deploy_root = ostree.deployment_path(tree, osname, ref, serial) -+ deploy_root = ostree.deployment_path(target, osname, ref, serial) - -- print(f"Deployment root at '{os.path.relpath(deploy_root, tree)}'") -+ print(f"Deployment root at '{os.path.relpath(deploy_root, target)}'") - -- var = os.path.join(tree, "ostree", "deploy", osname, "var") -- boot = os.path.join(tree, "boot") -+ var = os.path.join(target, "ostree", "deploy", osname, "var") -+ boot = os.path.join(target, "boot") - - self.mountpoint = deploy_root - self.bind_mount(deploy_root, deploy_root) # prepare to move it later - -- self.bind_mount(tree, os.path.join(deploy_root, "sysroot")) -+ self.bind_mount(target, os.path.join(deploy_root, "sysroot")) - self.bind_mount(var, os.path.join(deploy_root, "var")) - self.bind_mount(boot, os.path.join(deploy_root, "boot")) - - subprocess.run([ -- "mount", "--move", deploy_root, tree, -+ "mount", "--move", deploy_root, target, - ], check=True) - -- self.mountpoint = tree -+ self.mountpoint = target - self.check = True - - def umount(self): --- -2.43.0 - diff --git a/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch b/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch deleted file mode 100644 index c9a67630c7..0000000000 --- a/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch +++ /dev/null @@ -1,149 +0,0 @@ -From bd6b8ffb83384e7f9e78dc42a9cee626830b990f Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 11:07:14 -0500 -Subject: [PATCH 4/5] mounts/ostree.deployment: support deployments on mount - -Instead of operating directly on the tree for a stage we can operate -on a mount too. This is useful in the case where operating on the -directory tree of files isn't sufficient and the modifications need -to be made directly to the filesystems on the disk image that we are -creating. - -One such example of this is we are having a problem right now where -the immutable bit being set on an OSTree deployment root doesn't -survive the `cp -a --reflink=auto` in the org.osbuild.copy stage when -being copied from the directory tree into the mounted XFS filesystem -we created on the disk image. Thus we have to workaround this loss -of attribute by applying the attribute directly on the mounted -filesystem from the disk. - -In this change here we also add a check in osbuild/mounts.py to not -attempt a umount of the root of the mounts directory if that path -is no longer a mountpoint, which can happen when the umount -R -from the mounts/org.osbuild.ostree.deployment also removes the -overmount. - -Here is an example of how this would be used: - -``` - - type: org.osbuild.chattr - options: - immutable: true - path: mount://root/ - devices: - disk: - type: org.osbuild.loopback - options: - filename: disk.img - partscan: true - mounts: - - name: root - type: org.osbuild.xfs - source: disk - partition: - mpp-format-int: '{image.layout[''root''].partnum}' - target: / - - name: ostree.deployment - type: org.osbuild.ostree.deployment - options: - source: mount - deployment: - osname: fedora-coreos - ref: ostree/1/1/0 -``` - -The initial mount on `/` is the filesystem from the root partition -on the disk. The second mount (of type org.osbuild.ostree.deployment) -then reconfigures things similar to how an OSTree system is set up. ---- - mounts/org.osbuild.ostree.deployment | 18 +++++++++++++++--- - osbuild/mounts.py | 13 ++++++++++--- - 2 files changed, 25 insertions(+), 6 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index b200eca3..24df7731 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -32,6 +32,12 @@ SCHEMA_2 = """ - "type": "object", - "required": ["deployment"], - "properties": { -+ "source": { -+ "type": "string", -+ "pattern": "^(mount|tree)$", -+ "default": "tree", -+ "description": "The source of the OSTree filesystem tree. If 'mount', there should be a preceding mount defined that's mounted at /." -+ }, - "deployment": { - "type": "object", - "additionalProperties": false, -@@ -91,16 +97,21 @@ class OSTreeDeploymentMount(mounts.MountService): - def mount(self, args: Dict): - - tree = args["tree"] -+ mountroot = args["root"] - options = args["options"] - -+ source = options.get("source", "tree") - deployment = options["deployment"] - osname = deployment["osname"] - ref = deployment["ref"] - serial = deployment.get("serial", 0) - -- # The target path where we want the deployment to be mounted -- # is the root of the tree. -- target = tree -+ # The user could specify either the tree or mountroot as the -+ # place where we want the deployment to be mounted. -+ if source == "mount": -+ target = mountroot -+ else: -+ target = tree - - # create a private mountpoint for the target path, which is - # needed in order to be able to move the deployment `root` -@@ -113,6 +124,7 @@ class OSTreeDeploymentMount(mounts.MountService): - deploy_root = ostree.deployment_path(target, osname, ref, serial) - - print(f"Deployment root at '{os.path.relpath(deploy_root, target)}'") -+ print(f"mounting {deploy_root} -> {target}") - - var = os.path.join(target, "ostree", "deploy", osname, "var") - boot = os.path.join(target, "boot") -diff --git a/osbuild/mounts.py b/osbuild/mounts.py -index b938d21d..42b556ba 100644 ---- a/osbuild/mounts.py -+++ b/osbuild/mounts.py -@@ -181,6 +181,8 @@ class FileSystemMountService(MountService): - os.makedirs(mountpoint, exist_ok=True) - self.mountpoint = mountpoint - -+ print(f"mounting {source} -> {mountpoint}") -+ - try: - subprocess.run( - ["mount"] + -@@ -203,12 +205,17 @@ class FileSystemMountService(MountService): - if not self.mountpoint: - return - -- self.sync() -+ # It's possible this mountpoint has already been unmounted -+ # if a umount -R was run by another process, as is done in -+ # mounts/org.osbuild.ostree.deployment. -+ if not os.path.ismount(self.mountpoint): -+ print(f"already unmounted: {self.mountpoint}") -+ return - -- print("umounting") -+ self.sync() - - # We ignore errors here on purpose -- subprocess.run(["umount", self.mountpoint], -+ subprocess.run(["umount", "-v", self.mountpoint], - check=self.check) - self.mountpoint = None - --- -2.43.0 - diff --git a/src/0005-Create-stages-org.osbuild.chattr-stage.patch b/src/0005-Create-stages-org.osbuild.chattr-stage.patch deleted file mode 100644 index ac2a3a4c81..0000000000 --- a/src/0005-Create-stages-org.osbuild.chattr-stage.patch +++ /dev/null @@ -1,123 +0,0 @@ -From 477a21043eeb55468b1f75a9574e46e77cb5fdef Mon Sep 17 00:00:00 2001 -From: Luke Yang -Date: Mon, 22 Jan 2024 10:28:01 -0500 -Subject: [PATCH 5/5] Create stages/org.osbuild.chattr stage - -Add or remove the immutable bit to the specified mount directory. - -The need we have for this right now is for the CoreOS builds where -the immutable bit being set on an OSTree deployment root doesn't -survive the `cp -a --reflink=auto` in the org.osbuild.copy stage when -being copied from the directory tree into the mounted XFS filesystem -we created on the disk image. Thus we have to workaround this loss -of attribute by applying the attribute directly on the mounted -filesystem from the disk. ---- - stages/org.osbuild.chattr | 95 +++++++++++++++++++++++++++++++++++++++ - 1 file changed, 95 insertions(+) - create mode 100755 stages/org.osbuild.chattr - -diff --git a/stages/org.osbuild.chattr b/stages/org.osbuild.chattr -new file mode 100755 -index 00000000..0bc8aa0e ---- /dev/null -+++ b/stages/org.osbuild.chattr -@@ -0,0 +1,95 @@ -+#!/usr/bin/python3 -+""" -+Runs `chattr` to set file/directory attributes. -+""" -+ -+import os -+import subprocess -+import sys -+from typing import Dict -+from urllib.parse import ParseResult, urlparse -+ -+import osbuild.api -+ -+SCHEMA_2 = r""" -+"options": { -+ "additionalProperties": false, -+ "properties": { -+ "items": { -+ "type": "object", -+ "additionalProperties": false, -+ "patternProperties": { -+ "^mount:\/\/[^\/]+\/|^tree:\/\/\/": { -+ "type": "object", -+ "required": ["immutable"], -+ "properties": { -+ "immutable": { -+ "type": "boolean", -+ "description": "Make the file/directory immutable", -+ "default": true -+ } -+ } -+ } -+ } -+ } -+ } -+}, -+"devices": { -+ "type": "object", -+ "additionalProperties": true -+}, -+"mounts": { -+ "type": "array" -+} -+""" -+ -+ -+def parse_mount(url: ParseResult, args: Dict): -+ name = url.netloc -+ if name: -+ root = args["mounts"].get(name, {}).get("path") -+ if not root: -+ raise ValueError(f"Unknown mount '{name}'") -+ else: -+ root = args["paths"]["mounts"] -+ -+ return root -+ -+ -+def parse_location(location, args): -+ url = urlparse(location) -+ -+ scheme = url.scheme -+ if scheme == "tree": -+ root = args["tree"] -+ elif scheme == "mount": -+ root = parse_mount(url, args) -+ else: -+ raise ValueError(f"Unsupported scheme '{scheme}'") -+ -+ assert url.path.startswith("/") -+ -+ path = os.path.relpath(url.path, "/") -+ path = os.path.join(root, path) -+ path = os.path.normpath(path) -+ -+ if url.path.endswith("/"): -+ path = os.path.join(path, ".") -+ -+ return path -+ -+ -+def main(args, options): -+ for path, cmdargs in options["items"].items(): -+ immutable = cmdargs["immutable"] -+ dst = parse_location(path, args) -+ op = '+' if immutable else '-' -+ subprocess.run(["chattr", f"{op}i", dst], check=True) -+ -+ return 0 -+ -+ -+if __name__ == '__main__': -+ _args = osbuild.api.arguments() -+ r = main(_args, _args["options"]) -+ sys.exit(r) --- -2.43.0 - diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 986e3f7483..31d0626376 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -729,13 +729,6 @@ runvm() { # include COSA in the image find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles" - # include new patched in osbuild stage in the image. - # can drop this once the upstream PRs are merged: - # https://github.com/osbuild/osbuild/pull/1519 - # https://github.com/osbuild/osbuild/pull/1535 - # shellcheck disable=SC2129 - echo /usr/lib/osbuild/stages/org.osbuild.chattr >> "${vmpreparedir}/hostfiles" - # and include all GPG keys find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles" From d7dc943f585e0ff6375a662d745e0311399c8c50 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Wed, 14 Feb 2024 15:41:21 -0500 Subject: [PATCH 5/7] osbuild: add sanity check in qemu OSbuild stage This will compare the image that was just created to see if it has any problems. --- build.sh | 1 + ...ages-qemu-sanity-check-created-image.patch | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/0001-stages-qemu-sanity-check-created-image.patch diff --git a/build.sh b/build.sh index f8e50bdd31..0395f8f83d 100755 --- a/build.sh +++ b/build.sh @@ -178,6 +178,7 @@ patch_osbuild() { # Now all the software is under the /usr/lib/osbuild dir and we can patch cat /usr/lib/coreos-assembler/0004-fscache-add-eviction-log-statement.patch \ /usr/lib/coreos-assembler/0001-stages-zipl.inst-improve-kernel-initrd-path-resoluti.patch \ + /usr/lib/coreos-assembler/0001-stages-qemu-sanity-check-created-image.patch \ | patch -d /usr/lib/osbuild -p1 # And then move the files back; supermin appliance creation will need it back diff --git a/src/0001-stages-qemu-sanity-check-created-image.patch b/src/0001-stages-qemu-sanity-check-created-image.patch new file mode 100644 index 0000000000..a57fd21229 --- /dev/null +++ b/src/0001-stages-qemu-sanity-check-created-image.patch @@ -0,0 +1,39 @@ +From 4c73f0b53b23e694c9f031951cc0509e3687966f Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Wed, 14 Feb 2024 15:24:36 -0500 +Subject: [PATCH] stages/qemu: sanity check created image + +I'm having a lot of trouble investigating [1]. Let's add this here +as a small sanity check for now. + +[1] https://github.com/coreos/coreos-assembler/issues/3728 +--- + stages/org.osbuild.qemu | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/stages/org.osbuild.qemu b/stages/org.osbuild.qemu +index 642b5146..54e707d4 100755 +--- a/stages/org.osbuild.qemu ++++ b/stages/org.osbuild.qemu +@@ -231,6 +231,18 @@ def main(inputs, output, options): + cmd, check=True + ) + ++ # Sanity check that the image is 100% ++ cmd = [ ++ "qemu-img", "compare", ++ "-f", "raw", ++ "-F", fmt["type"], ++ source, target ++ ] ++ subprocess.run( ++ cmd, check=True ++ ) ++ ++ + return 0 + + +-- +2.43.0 + From e48e736d9991406673231deb5bbeac5413a152b2 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Wed, 14 Feb 2024 18:04:44 -0500 Subject: [PATCH 6/7] cmdlib: use ext4 for cache qcow2 We think there might be some XFS reflink issues when we run the OSBuild org.osbuild.qemu stage compression: false. See https://github.com/coreos/coreos-assembler/issues/3728#issuecomment-1944956047 --- src/cmdlib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 31d0626376..95b20a0ece 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -607,7 +607,7 @@ runvm_with_cache_snapshot() { ( # shellcheck source=src/libguestfish.sh source /usr/lib/coreos-assembler/libguestfish.sh - virt-format --filesystem=xfs --label=cosa-cache -a cache2.qcow2.tmp) + virt-format --filesystem=ext4 --label=cosa-cache -a cache2.qcow2.tmp) mv -T cache2.qcow2.tmp "${workdir}"/cache/cache2.qcow2 fi # And remove the old one From 030e281ec482bdb2c017aa6df5ee4e9ad87894a5 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 13 Feb 2024 11:56:01 -0500 Subject: [PATCH 7/7] osbuild: disable internal compression for qemu qcow2 We previously did this in a different way (2a8d1e6) but then had to revert it (39fdd61) because it caused images to not boot [1]. The root cause appears to have been the virtiofs mount not being unmounted cleanly from the supermin VM and that is now fixed so let's switch back to not compressing since we rely on our outer compression [2]. [1] https://github.com/coreos/coreos-assembler/issues/3728 [2] https://github.com/coreos/fedora-coreos-tracker/issues/1653#issuecomment-1928342241 --- src/coreos.osbuild.aarch64.mpp.yaml | 2 +- src/coreos.osbuild.ppc64le.mpp.yaml | 2 +- src/coreos.osbuild.s390x.mpp.yaml | 2 +- src/coreos.osbuild.x86_64.mpp.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreos.osbuild.aarch64.mpp.yaml b/src/coreos.osbuild.aarch64.mpp.yaml index 2eb460e2db..8d652580df 100644 --- a/src/coreos.osbuild.aarch64.mpp.yaml +++ b/src/coreos.osbuild.aarch64.mpp.yaml @@ -674,5 +674,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.ppc64le.mpp.yaml b/src/coreos.osbuild.ppc64le.mpp.yaml index 669eabd7a0..f74cd68aa5 100644 --- a/src/coreos.osbuild.ppc64le.mpp.yaml +++ b/src/coreos.osbuild.ppc64le.mpp.yaml @@ -641,5 +641,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.s390x.mpp.yaml b/src/coreos.osbuild.s390x.mpp.yaml index 42f7e1f1a6..99a307c6b3 100644 --- a/src/coreos.osbuild.s390x.mpp.yaml +++ b/src/coreos.osbuild.s390x.mpp.yaml @@ -640,5 +640,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.x86_64.mpp.yaml b/src/coreos.osbuild.x86_64.mpp.yaml index 1d2569f353..ff34d0953d 100644 --- a/src/coreos.osbuild.x86_64.mpp.yaml +++ b/src/coreos.osbuild.x86_64.mpp.yaml @@ -680,5 +680,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1'