diff --git a/build.sh b/build.sh index e1d808988a..0395f8f83d 100755 --- a/build.sh +++ b/build.sh @@ -45,7 +45,7 @@ install_rpms() { frozendeps="" # We freeze the version for now since we're carrying patches. - frozendeps+=" $(echo osbuild{,-ostree,-selinux,-tools}-106-1.fc39.noarch)" + frozendeps+=" $(echo osbuild{,-ostree,-selinux,-tools}-108-1.fc39.noarch)" # First, a general update; this is best practice. We also hit an issue recently # where qemu implicitly depended on an updated libusbx but didn't have a versioned @@ -176,24 +176,9 @@ patch_osbuild() { mv /usr/bin/osbuild-mpp /usr/lib/osbuild/tools/ # Now all the software is under the /usr/lib/osbuild dir and we can patch - cat /usr/lib/coreos-assembler/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch \ - /usr/lib/coreos-assembler/0001-mounts-ostree.deployment-rework-unmounting.patch \ - /usr/lib/coreos-assembler/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch \ - /usr/lib/coreos-assembler/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch \ - /usr/lib/coreos-assembler/0004-mounts-ostree.deployment-support-deployments-on-moun.patch \ - /usr/lib/coreos-assembler/0005-Create-stages-org.osbuild.chattr-stage.patch \ - /usr/lib/coreos-assembler/0001-fscache-add-new-FsCache._last_used_objs-helper.patch \ - /usr/lib/coreos-assembler/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch \ - /usr/lib/coreos-assembler/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch \ - /usr/lib/coreos-assembler/0004-fscache-add-eviction-log-statement.patch \ - /usr/lib/coreos-assembler/0001-support-user-defined-partition-numbers-for-GPT-disks.patch \ - /usr/lib/coreos-assembler/0002-stages-sgdisk-support-label-option.patch \ + cat /usr/lib/coreos-assembler/0004-fscache-add-eviction-log-statement.patch \ /usr/lib/coreos-assembler/0001-stages-zipl.inst-improve-kernel-initrd-path-resoluti.patch \ - /usr/lib/coreos-assembler/0002-stages-zipl.inst-support-appending-kernel-options.patch \ - /usr/lib/coreos-assembler/0001-stages-copy-allow-copying-from-the-tree.patch \ - /usr/lib/coreos-assembler/0001-ostree-add-convenience-function-for-using-default-OS.patch \ - /usr/lib/coreos-assembler/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch \ - /usr/lib/coreos-assembler/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch \ + /usr/lib/coreos-assembler/0001-stages-qemu-sanity-check-created-image.patch \ | patch -d /usr/lib/osbuild -p1 # And then move the files back; supermin appliance creation will need it back diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index a9841597c9..f61dfeb32a 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -1071,6 +1071,8 @@ func (disk *Disk) prepare(builder *QemuBuilder) error { // on our own. if strings.HasSuffix(backingFile, "qcow2") { format = "qcow2" + } else if strings.HasSuffix(backingFile, "raw") { + format = "raw" } } if format != "" { diff --git a/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch b/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch deleted file mode 100644 index ecfcd36ac9..0000000000 --- a/src/0001-fscache-add-new-FsCache._last_used_objs-helper.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 4a9831fa889b073ccb14568c4571f57d28dfe84f Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Tue, 12 Dec 2023 21:20:42 +0100 -Subject: [PATCH 1/4] fscache: add new `FsCache._last_used_objs()' helper - -This commit adds a helper that can be used to get a sorted list -of cache entries. The list includes the name and the last_used -information. ---- - osbuild/util/fscache.py | 29 ++++++++++++++++++++++++++++- - 1 file changed, 28 insertions(+), 1 deletion(-) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index c8fc99aa..58c9a310 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -15,7 +15,7 @@ import json - import os - import subprocess - import uuid --from typing import Any, Dict, NamedTuple, Optional, Tuple, Union -+from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Union - - from osbuild.util import ctx, linux, rmrf - -@@ -101,6 +101,17 @@ class FsCacheInfo(NamedTuple): - return data - - -+class FsCacheObjectInfo(NamedTuple): -+ """ File System Cache object information -+ -+ This type represents information about a single cache object. The -+ last_used information is only guaranteed to be valid while the cache -+ is locked. -+ """ -+ name: str -+ last_used: float -+ -+ - class FsCache(contextlib.AbstractContextManager, os.PathLike): - """File System Cache - -@@ -1059,6 +1070,22 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - raise self.MissError() from None - raise e - -+ def _last_used_objs(self) -> List[FsCacheObjectInfo]: -+ """Return a list of FsCacheObjectInfo with name, last_used -+ information sorted by last_used time. -+ -+ Note that this function will be racy when used without a lock and -+ the caller needs to handle this. -+ """ -+ objs = [] -+ for name in os.listdir(self._path(self._dirname_objects)): -+ try: -+ last_used = self._last_used(name) -+ except (OSError, FsCache.MissError): -+ continue -+ objs.append(FsCacheObjectInfo(name=name, last_used=last_used)) -+ return sorted(objs, key=lambda obj: obj.last_used) -+ - @property - def info(self) -> FsCacheInfo: - """Query Cache Information --- -2.43.0 - diff --git a/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch b/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch deleted file mode 100644 index ba7075fae6..0000000000 --- a/src/0001-mount-ostree.deployment-Fix-ostree-deployment-call.patch +++ /dev/null @@ -1,35 +0,0 @@ -From c623ca19f418c7f6d1d9dbf7ac6a80973c3ac7ab Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Fri, 9 Feb 2024 22:19:35 -0500 -Subject: [PATCH] mount/ostree.deployment: Fix ostree deployment call - -We need to pass in the root of the ostree deployment which can -be the tree or the mount. Fixes e1cbf92 ---- - mounts/org.osbuild.ostree.deployment | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 9637b5b2..c72dec6e 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -125,7 +125,6 @@ class OSTreeDeploymentMount(mounts.MountService): - options = args["options"] - source = options.get("source", "tree") - deployment = options["deployment"] -- osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - # The user could specify either the tree or mountroot as the - # place where we want the deployment to be mounted. -@@ -134,6 +133,8 @@ class OSTreeDeploymentMount(mounts.MountService): - else: - target = tree - -+ osname, ref, serial = ostree.parse_deployment_option(target, deployment) -+ - # create a private mountpoint for the target path, which is - # needed in order to be able to move the deployment `root` - # mountpoint here, which is contained inside tree, since --- -2.43.0 - diff --git a/src/0001-mounts-ostree.deployment-rework-unmounting.patch b/src/0001-mounts-ostree.deployment-rework-unmounting.patch deleted file mode 100644 index 3bd7d13de6..0000000000 --- a/src/0001-mounts-ostree.deployment-rework-unmounting.patch +++ /dev/null @@ -1,135 +0,0 @@ -From 0da68e9af5a7b148e9841ff2a48bffd96be79b72 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Tue, 9 Jan 2024 23:23:25 -0500 -Subject: [PATCH 1/5] mounts/ostree.deployment: rework unmounting -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This unwinds part of a25ae2b. The way the code ended up both -self.tree and self.mountpoint ended up pointing to the exactly -same path and so we'd end up doing two `umount -R` operations -on the same path. This ended up being a duplicate unmount. - -On Fedora 39 this yields an error like: - -``` -mount/ostree.deployment (org.osbuild.ostree.deployment): umount: /var/osbuild/store/stage/uuid-efaac9370d25455d9e8df6d847ecb5b3/data/tree: not mounted -mount/ostree.deployment (org.osbuild.ostree.deployment): Traceback (most recent call last): -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 136, in -mount/ostree.deployment (org.osbuild.ostree.deployment): main() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 132, in main -mount/ostree.deployment (org.osbuild.ostree.deployment): service.main() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/osbuild/host.py", line 252, in main -mount/ostree.deployment (org.osbuild.ostree.deployment): self.stop() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/osbuild/mounts.py", line 126, in stop -mount/ostree.deployment (org.osbuild.ostree.deployment): self.umount() -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/var/b/shared/code/github.com/osbuild/osbuild/mounts/org.osbuild.ostree.deployment", line 125, in umount -mount/ostree.deployment (org.osbuild.ostree.deployment): subprocess.run(["umount", "-R", self.tree], -mount/ostree.deployment (org.osbuild.ostree.deployment): File "/usr/lib64/python3.12/subprocess.py", line 571, in run -mount/ostree.deployment (org.osbuild.ostree.deployment): raise CalledProcessError(retcode, process.args, -mount/ostree.deployment (org.osbuild.ostree.deployment): subprocess.CalledProcessError: Command '['umount', '-R', '/var/osbuild/store/stage/uuid-efaac9370d25455d9e8df6d847ecb5b3/data/tree'] -' returned non-zero exit status 1. - -⏱ Duration: 103s -``` - -I think this was necessary because of a bug in util-linux that -mean some of the accounting information got out of date when -doing a `mount --move` operation, which we use here. I think this -bug (or bugs) is now fixed [1][2] in util-linux v2.39 (in Fedora 39), -which is now causing the above pasted error on F39. - -Let's just add code here that mentions the problem and workaround -it with a loop to keep unmounting (essentially what the umount -R -should have done to overmounted filesystems if the mountinfo/utab -was correct) and also mention when we should be able to drop this -workaround. - -[1] https://github.com/karelzak/util-linux/commit/a04149fbb7c1952da1194d1514e298ff07dbc7ca -[2] https://github.com/karelzak/util-linux/commit/8cf6c5075780598fe3b30e7a7753d8323d093e22 ---- - mounts/org.osbuild.ostree.deployment | 44 +++++++++++++++++++++------- - 1 file changed, 34 insertions(+), 10 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 5571cc1f..9046fb05 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -63,7 +63,6 @@ class OSTreeDeploymentMount(mounts.MountService): - def __init__(self, args): - super().__init__(args) - -- self.tree = None - self.mountpoint = None - self.check = False - -@@ -72,7 +71,22 @@ class OSTreeDeploymentMount(mounts.MountService): - subprocess.run([ - "mount", "--bind", "--make-private", source, target, - ], check=True) -- return target -+ -+ def is_mounted(self): -+ # Use `mountpoint` command here to determine if the mountpoint is mounted. -+ # We would use os.path.ismount() here but that only works if a device is -+ # mounted (i.e. it doesn't use the mountinfo file in the heuristic and -+ # thus things like `mount --move` wouldn't show up). The exit codes from -+ # `mountpoint` are: -+ # -+ # 0 success; the directory is a mountpoint, or device is block device on --devno -+ # 1 failure; incorrect invocation, permissions or system error -+ # 32 failure; the directory is not a mountpoint, or device is not a block device on --devno -+ # -+ cp = subprocess.run(["mountpoint", "-q", self.mountpoint], check=False) -+ if cp.returncode not in [0, 32]: -+ cp.check_returncode() # will raise error -+ return cp.returncode == 0 - - def mount(self, args: Dict): - -@@ -89,7 +103,7 @@ class OSTreeDeploymentMount(mounts.MountService): - # is contained inside tree, since "moving a mount residing - # under a shared mount is invalid and unsupported." - # - `mount(8)` -- self.tree = self.bind_mount(tree, tree) -+ self.bind_mount(tree, tree) - - root = ostree.deployment_path(tree, osname, ref, serial) - -@@ -116,15 +130,25 @@ class OSTreeDeploymentMount(mounts.MountService): - if self.mountpoint: - subprocess.run(["sync", "-f", self.mountpoint], - check=self.check) -- -- subprocess.run(["umount", "-R", self.mountpoint], -+ subprocess.run(["umount", "-v", "-R", self.mountpoint], - check=self.check) -- self.mountpoint = None - -- if self.tree: -- subprocess.run(["umount", "-R", self.tree], -- check=self.check) -- self.tree = None -+ # Handle bug in older util-linux mount where the -+ # mountinfo/utab wouldn't have updated information -+ # when mount --move is performed, which means that -+ # umount -R wouldn't unmount all overmounted mounts -+ # on the target because it was operating on outdated -+ # information. The umount -R behavior is fixed in v2.39 -+ # of util-linux most likely by [1] or [2] or both. This -+ # loop can be removed when all hosts we care about have -+ # moved to v2.39+. -+ # [1] https://github.com/karelzak/util-linux/commit/a04149fbb7c1952da1194d1514e298ff07dbc7ca -+ # [2] https://github.com/karelzak/util-linux/commit/8cf6c5075780598fe3b30e7a7753d8323d093e22 -+ while self.is_mounted(): -+ print(f"extra unmount {self.mountpoint}") -+ subprocess.run(["umount", "-v", self.mountpoint], -+ check=self.check) -+ self.mountpoint = None - - - def main(): --- -2.43.0 - diff --git a/src/0001-ostree-add-convenience-function-for-using-default-OS.patch b/src/0001-ostree-add-convenience-function-for-using-default-OS.patch deleted file mode 100644 index 40cf03af51..0000000000 --- a/src/0001-ostree-add-convenience-function-for-using-default-OS.patch +++ /dev/null @@ -1,437 +0,0 @@ -From e1cbf92673fa6aa52209d091150097a5961a0b21 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Thu, 25 Jan 2024 13:57:51 -0500 -Subject: [PATCH] ostree: add convenience function for using default OSTree - deployment - -This adds a `default: true` option for all cases where OSTree -information is specified in schemas and allows for the information -to be picked up from the filesystem. - -This is a safe operation because when building disk images there is -no known case where having two deployments makes sense. In the case -there ever were a case then the osname, ref, and serial options still -exist and can be used. - -Co-authored-by: Luke Yang -Co-authored-by: Michael Vogt ---- - mounts/org.osbuild.ostree.deployment | 31 ++++++++++++++++---- - osbuild/util/ostree.py | 42 ++++++++++++++++++++++++++-- - stages/org.osbuild.bootupd | 30 +++++++++++++++++--- - stages/org.osbuild.fstab | 30 +++++++++++++++++--- - stages/org.osbuild.ostree.aleph | 31 +++++++++++++++++--- - stages/org.osbuild.ostree.fillvar | 30 +++++++++++++++++--- - stages/org.osbuild.ostree.selinux | 30 +++++++++++++++++--- - 8 files changed, 217 insertions(+), 29 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 24df7731..9637b5b2 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -41,7 +41,26 @@ SCHEMA_2 = """ - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -55,6 +74,11 @@ SCHEMA_2 = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -99,12 +123,9 @@ class OSTreeDeploymentMount(mounts.MountService): - tree = args["tree"] - mountroot = args["root"] - options = args["options"] -- - source = options.get("source", "tree") - deployment = options["deployment"] -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - # The user could specify either the tree or mountroot as the - # place where we want the deployment to be mounted. -diff --git a/osbuild/util/ostree.py b/osbuild/util/ostree.py -index 173a9676..4c0344d1 100644 ---- a/osbuild/util/ostree.py -+++ b/osbuild/util/ostree.py -@@ -1,13 +1,15 @@ - import collections - import contextlib -+import glob - import json - import os -+import re - import subprocess - import sys - import tempfile - import typing - # pylint doesn't understand the string-annotation below --from typing import Any, List # pylint: disable=unused-import -+from typing import Any, Dict, List, Tuple # pylint: disable=unused-import - - from osbuild.util.rhsm import Subscriptions - -@@ -214,7 +216,43 @@ def parse_input_commits(commits): - return commits["path"], data["refs"] - - --def deployment_path(root: PathLike, osname: str, ref: str, serial: int): -+def parse_deployment_option(root: PathLike, deployment: Dict) -> Tuple[str, str, str]: -+ """Parse the deployment option and return the osname, ref, and serial -+ -+ The `deployment` arg contains the following sub fields: -+ - osname: Name of the stateroot used in the deployment (ie. fedora-coreos) -+ - ref: OStree ref to used for the deployment (ie. fedora/aarch64/coreos/next) -+ - serial: The deployment serial (ie. 0) -+ - default: Boolean to determine whether the default ostree deployment should be used -+ """ -+ -+ default_deployment = deployment.get("default") -+ if default_deployment: -+ filenames = glob.glob(os.path.join(root, 'ostree/deploy/*/deploy/*.0')) -+ if len(filenames) < 1: -+ raise ValueError("Could not find deployment") -+ if len(filenames) > 1: -+ raise ValueError(f"More than one deployment found: {filenames}") -+ -+ # We pick up the osname, commit, and serial from the filesystem -+ # here. We'll return the detected commit as the ref in this -+ # since it's a valid substitute for all subsequent uses in -+ # the code base. -+ f = re.search("/ostree/deploy/(.*)/deploy/(.*)\\.([0-9])", filenames[0]) -+ if not f: -+ raise ValueError("cannot find ostree deployment in {filenames[0]}") -+ osname = f.group(1) -+ commit = f.group(2) -+ serial = f.group(3) -+ return osname, commit, serial -+ -+ osname = deployment["osname"] -+ ref = deployment["ref"] -+ serial = deployment.get("serial", 0) -+ return osname, ref, serial -+ -+ -+def deployment_path(root: PathLike, osname: str = "", ref: str = "", serial: int = 0): - """Return the path to a deployment given the parameters""" - - base = os.path.join(root, "ostree") -diff --git a/stages/org.osbuild.bootupd b/stages/org.osbuild.bootupd -index 224bd6d4..5663ea3e 100755 ---- a/stages/org.osbuild.bootupd -+++ b/stages/org.osbuild.bootupd -@@ -32,7 +32,26 @@ SCHEMA_2 = r""" - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -46,6 +65,11 @@ SCHEMA_2 = r""" - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - }, -@@ -101,9 +125,7 @@ def main(args, options): - # we'll call ostree.deployment_path() helper to find it for us. - root = mounts - if deployment: -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(mounts, deployment) - root = ostree.deployment_path(mounts, osname, ref, serial) - - bootupd_args = [] -diff --git a/stages/org.osbuild.fstab b/stages/org.osbuild.fstab -index eb5ac2b0..a9af5ffe 100755 ---- a/stages/org.osbuild.fstab -+++ b/stages/org.osbuild.fstab -@@ -29,7 +29,26 @@ SCHEMA = """ - "deployment": { - "type": "object", - "additionalProperties": false, -- "required": ["osname","ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -43,6 +62,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -118,9 +142,7 @@ def main(tree, options): - - if ostree_options: - deployment = ostree_options["deployment"] -- osname = deployment["osname"] -- ref = deployment["ref"] -- serial = deployment.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, deployment) - - root = ostree.deployment_path(tree, osname, ref, serial) - -diff --git a/stages/org.osbuild.ostree.aleph b/stages/org.osbuild.ostree.aleph -index 6baedeb0..bcef1fe3 100755 ---- a/stages/org.osbuild.ostree.aleph -+++ b/stages/org.osbuild.ostree.aleph -@@ -19,6 +19,7 @@ COREOS_ALEPH_FILENAME = ".coreos-aleph-version.json" - SCHEMA_2 = """ - "options": { - "additionalProperties": false, -+ "required": ["deployment"], - "properties": { - "coreos_compat": { - "description": "boolean to allow for CoreOS aleph version backwards compatibility", -@@ -26,7 +27,26 @@ SCHEMA_2 = """ - }, - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -40,6 +60,11 @@ SCHEMA_2 = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -131,9 +156,7 @@ def construct_aleph_json(tree, origin): - def main(tree, options): - coreos_compat = options.get("coreos_compat", False) - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - origin = ostree.deployment_path(tree, osname, ref, serial) + ".origin" - data = construct_aleph_json(tree, origin) -diff --git a/stages/org.osbuild.ostree.fillvar b/stages/org.osbuild.ostree.fillvar -index e5b5eacd..dfb706e8 100755 ---- a/stages/org.osbuild.ostree.fillvar -+++ b/stages/org.osbuild.ostree.fillvar -@@ -18,7 +18,26 @@ SCHEMA = """ - "properties": { - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -32,6 +51,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -71,9 +95,7 @@ def populate_var(sysroot): - - def main(tree, options): - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - deployment = ostree.deployment_path(tree, osname, ref, serial) - var = os.path.join(tree, "ostree", "deploy", osname, "var") -diff --git a/stages/org.osbuild.ostree.selinux b/stages/org.osbuild.ostree.selinux -index d990a5e5..58663e34 100755 ---- a/stages/org.osbuild.ostree.selinux -+++ b/stages/org.osbuild.ostree.selinux -@@ -21,7 +21,26 @@ SCHEMA = """ - "properties": { - "deployment": { - "additionalProperties": false, -- "required": ["osname", "ref"], -+ "oneOf": [ -+ { -+ "properties": { -+ "default": {"enum": [false]} -+ }, -+ "required": ["osname", "ref"] -+ }, -+ { -+ "properties": { -+ "default": {"enum": [true]} -+ }, -+ "not": { -+ "anyOf": [ -+ {"required": ["osname"]}, -+ {"required": ["ref"]}, -+ {"required": ["serial"]} -+ ] -+ } -+ } -+ ], - "properties": { - "osname": { - "description": "Name of the stateroot to be used in the deployment", -@@ -35,6 +54,11 @@ SCHEMA = """ - "description": "The deployment serial (usually '0')", - "type": "number", - "default": 0 -+ }, -+ "default": { -+ "description": "Find and use the default ostree deployment", -+ "type": "boolean", -+ "default": false - } - } - } -@@ -44,9 +68,7 @@ SCHEMA = """ - - def main(tree, options): - dep = options["deployment"] -- osname = dep["osname"] -- ref = dep["ref"] -- serial = dep.get("serial", 0) -+ osname, ref, serial = ostree.parse_deployment_option(tree, dep) - - # this created a state root at `osname` - stateroot = f"{tree}/ostree/deploy/{osname}" --- -2.43.0 - diff --git a/src/0001-stages-copy-allow-copying-from-the-tree.patch b/src/0001-stages-copy-allow-copying-from-the-tree.patch deleted file mode 100644 index 8cc0ad195a..0000000000 --- a/src/0001-stages-copy-allow-copying-from-the-tree.patch +++ /dev/null @@ -1,72 +0,0 @@ -From 3ecec2e2848a51803e4a9953b132a6eaf4e5043c Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Thu, 1 Feb 2024 23:14:06 -0500 -Subject: [PATCH] stages(copy): allow copying from the tree - -It seems like an artifical limitation to prevent copying from one -location in the tree to another. It just so happens we need this -functionality when building CoreOS images because we want to take -a file embedded in the OSTree at a location and copy it to another -location in the tree. The particular example here is we want to copy -/usr/share/coreos-assembler/platforms.json -> /boot/coreos/platforms.json -See https://github.com/coreos/coreos-assembler/pull/3709 - -Allowing to copy from/to the tree we can now do something like: - -``` -- type: org.osbuild.copy - options: - paths: - - from: tree:///usr/share/coreos-assembler/platforms.json - to: tree:///boot/coreos/platforms.json - mounts: - - name: ostree.deployment - type: org.osbuild.ostree.deployment - options: - deployment: - ref: ostree/1/1/0 - osname: - fedora-coreos -``` ---- - stages/org.osbuild.copy | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/stages/org.osbuild.copy b/stages/org.osbuild.copy -index 3c137af5..d476efdb 100755 ---- a/stages/org.osbuild.copy -+++ b/stages/org.osbuild.copy -@@ -45,9 +45,18 @@ SCHEMA_2 = r""" - "required": ["from", "to"], - "properties": { - "from": { -- "type": "string", -- "description": "The source", -- "pattern": "^input:\/\/[^\/]+\/" -+ "oneOf": [ -+ { -+ "type": "string", -+ "description": "The source, if an input", -+ "pattern": "^input:\/\/[^\/]+\/" -+ }, -+ { -+ "type": "string", -+ "description": "The source, if the tree", -+ "pattern": "^tree:\/\/\/" -+ } -+ ] - }, - "to": { - "oneOf": [ -@@ -58,7 +67,7 @@ SCHEMA_2 = r""" - }, - { - "type": "string", -- "description": "The destination, if a tree", -+ "description": "The destination, if the tree", - "pattern": "^tree:\/\/\/" - } - ] --- -2.43.0 - diff --git a/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch b/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch deleted file mode 100644 index 108c3cfe16..0000000000 --- a/src/0001-stages-org.osbuild.qemu-make-qcow2-compression-optio.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 619a64f0bd7322628ae5fedf8b3a640e42f9edcd Mon Sep 17 00:00:00 2001 -From: Luke Yang -Date: Tue, 6 Feb 2024 13:54:15 -0500 -Subject: [PATCH] stages/org.osbuild.qemu: make qcow2 compression optional - -Modify the stages/org.osbuild.qemu stage such that compression is -optional. This resolves the image size differences between an image -built with coreos assember vs osbuild, as discussed in: -https://github.com/coreos/fedora-coreos-tracker/issues/1653#issuecomment-1928342241 ---- - stages/org.osbuild.qemu | 23 +++++++++++++++++++++-- - 1 file changed, 21 insertions(+), 2 deletions(-) - -diff --git a/stages/org.osbuild.qemu b/stages/org.osbuild.qemu -index 98a04f00..642b5146 100755 ---- a/stages/org.osbuild.qemu -+++ b/stages/org.osbuild.qemu -@@ -31,6 +31,11 @@ SCHEMA_2 = r""" - "type": "string", - "enum": ["qcow2"] - }, -+ "compression": { -+ "description": "Enable/disable compression of the qcow2 image", -+ "type": "boolean", -+ "default": true -+ }, - "compat": { - "description": "The qcow2-compatibility-version to use", - "type": "string" -@@ -61,6 +66,11 @@ SCHEMA_2 = r""" - "type": "string", - "enum": ["vmdk"] - }, -+ "compression": { -+ "description": "Enable/disable compression of the vmdk image", -+ "type": "boolean", -+ "default": true -+ }, - "subformat": { - "description": "VMDK flat extent format", - "type": "string", -@@ -140,17 +150,26 @@ SCHEMA_2 = r""" - - - def qcow2_arguments(options): -- argv = ["-c"] -+ argv = [] -+ compression = options.get("compression", True) - compat = options.get("compat") - -+ if compression: -+ argv += ["-c"] -+ - if compat: - argv += ["-o", f"compat={compat}"] - return argv - - - def vmdk_arguments(options): -- argv = ["-c"] -+ argv = [] -+ compression = options.get("compression", True) - subformat = options.get("subformat") -+ -+ if compression: -+ argv += ["-c"] -+ - if subformat: - argv += ["-o", f"subformat={subformat}"] - return argv --- -2.43.0 - diff --git a/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch b/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch deleted file mode 100644 index e4e6483154..0000000000 --- a/src/0001-stages-ostree.config-support-setting-sysroot.bootpre.patch +++ /dev/null @@ -1,41 +0,0 @@ -From 612d3abd2ba05072a27142d5197645b4049561ae Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Mon, 5 Feb 2024 23:27:59 -0500 -Subject: [PATCH] stages(ostree.config): support setting sysroot.bootprefix - -See https://github.com/ostreedev/ostree/pull/2705 and also -https://github.com/osbuild/osbuild/issues/1566. ---- - stages/org.osbuild.ostree.config | 9 +++++++++ - 1 file changed, 9 insertions(+) - -diff --git a/stages/org.osbuild.ostree.config b/stages/org.osbuild.ostree.config -index 7d5bb71d..5a91a8ae 100755 ---- a/stages/org.osbuild.ostree.config -+++ b/stages/org.osbuild.ostree.config -@@ -39,6 +39,10 @@ SCHEMA = """ - "type": "string", - "enum": ["none", "auto", "grub2", "syslinux", "uboot", "zipl", "aboot"] - }, -+ "bootprefix": { -+ "description": "If set to true, the bootloader entries generated will include /boot as a prefix.", -+ "type": "boolean" -+ }, - "readonly": { - "description": "Read only sysroot and boot", - "type": "boolean" -@@ -63,6 +67,11 @@ def main(tree, options): - if bootloader: - ostree.cli("config", "set", "sysroot.bootloader", bootloader, repo=repo) - -+ bootprefix = sysroot_options.get("bootprefix") -+ if bootprefix is not None: # can be False, which we would want to set -+ bp = "true" if bootprefix else "false" -+ ostree.cli("config", "set", "sysroot.bootprefix", bp, repo=repo) -+ - readonly = sysroot_options.get("readonly") - if readonly is not None: # can be False, which we would want to set - ro = "true" if readonly else "false" --- -2.43.0 - diff --git a/src/0001-stages-qemu-sanity-check-created-image.patch b/src/0001-stages-qemu-sanity-check-created-image.patch new file mode 100644 index 0000000000..a57fd21229 --- /dev/null +++ b/src/0001-stages-qemu-sanity-check-created-image.patch @@ -0,0 +1,39 @@ +From 4c73f0b53b23e694c9f031951cc0509e3687966f Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Wed, 14 Feb 2024 15:24:36 -0500 +Subject: [PATCH] stages/qemu: sanity check created image + +I'm having a lot of trouble investigating [1]. Let's add this here +as a small sanity check for now. + +[1] https://github.com/coreos/coreos-assembler/issues/3728 +--- + stages/org.osbuild.qemu | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/stages/org.osbuild.qemu b/stages/org.osbuild.qemu +index 642b5146..54e707d4 100755 +--- a/stages/org.osbuild.qemu ++++ b/stages/org.osbuild.qemu +@@ -231,6 +231,18 @@ def main(inputs, output, options): + cmd, check=True + ) + ++ # Sanity check that the image is 100% ++ cmd = [ ++ "qemu-img", "compare", ++ "-f", "raw", ++ "-F", fmt["type"], ++ source, target ++ ] ++ subprocess.run( ++ cmd, check=True ++ ) ++ ++ + return 0 + + +-- +2.43.0 + diff --git a/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch b/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch deleted file mode 100644 index 066d903d6c..0000000000 --- a/src/0001-support-user-defined-partition-numbers-for-GPT-disks.patch +++ /dev/null @@ -1,250 +0,0 @@ -From d84df08668b1e7350f9f151eff9d79817a1847ab Mon Sep 17 00:00:00 2001 -From: Nikita Dubrovskii -Date: Fri, 26 Jan 2024 14:18:21 +0100 -Subject: [PATCH 1/2] support user-defined partition numbers for GPT disks - -Partitions by default are indexed starting at 1, but in -some cases, such as CoreOS for IBM Z, it may be usefull -to set the 'partnum' for GPT disks explicitly, without -creating dummy partitions. - -Now user can define an image: - -``` - mpp-define-images: - - id: image - size: 10737418240 - table: - uuid: 00000000-0000-4000-a000-000000000001 - label: gpt - partitions: - - name: boot - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 3 - size: 786432 - - name: root - type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 - partnum: 4 - size: 4194304 -``` - -So target disk would look like: - -``` - Disklabel type: gpt - Disk identifier: 00000000-0000-4000-A000-000000000001 - Device Start End Sectors Size Type - /dev/loop0p3 2048 788479 786432 384M Linux filesystem - /dev/loop0p4 788480 4982783 4194304 2G Linux filesystem -``` - -This patch updates the osbuild-mpp tool and the sgdisk and sfdisk -stages to support this. - -Co-authored-by: Dusty Mabe ---- - stages/org.osbuild.sfdisk | 19 ++++++++++++++++--- - stages/org.osbuild.sgdisk | 9 ++++++++- - tools/osbuild-mpp | 18 +++++++++++++----- - 3 files changed, 37 insertions(+), 9 deletions(-) - -diff --git a/stages/org.osbuild.sfdisk b/stages/org.osbuild.sfdisk -index 9887a02e..108866c4 100755 ---- a/stages/org.osbuild.sfdisk -+++ b/stages/org.osbuild.sfdisk -@@ -4,6 +4,7 @@ Partition a target using sfdisk(8) - """ - - import json -+import re - import subprocess - import sys - from typing import Optional -@@ -50,6 +51,10 @@ SCHEMA_2 = r""" - "description": "The partition name (GPT)", - "type": "string" - }, -+ "partnum": { -+ "description": "The partition number", -+ "type": "integer" -+ }, - "size": { - "description": "The size of this partition", - "type": "integer" -@@ -88,6 +93,7 @@ SCHEMA_2 = r""" - class Partition: - def __init__(self, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -95,13 +101,15 @@ class Partition: - uuid: str = None, - attrs: int = None): - self.type = pttype -+ self.partnum = partnum - self.start = start - self.size = size - self.bootable = bootable - self.name = name - self.uuid = uuid -- self.index = None - self.attrs = attrs -+ self.index = partnum - 1 if partnum else None -+ self.partnum = partnum if partnum else None - - @property - def start_in_bytes(self): -@@ -167,7 +175,10 @@ class PartitionTable: - fields += [f'{field}="{value}"'] - if partition.bootable: - fields += ["bootable"] -- command += "\n" + ", ".join(fields) -+ if partition.partnum: -+ command += "\n" + f'{target}p{partition.partnum}: ' + ", ".join(fields) -+ else: -+ command += "\n" + ", ".join(fields) - - print(command) - -@@ -190,7 +201,8 @@ class PartitionTable: - - assert len(disk_parts) == len(self.partitions) - for i, part in enumerate(self.partitions): -- part.index = i -+ part.partnum = int(re.findall(r'\d+$', disk_parts[i]["node"])[0]) -+ part.index = part.partnum - 1 - part.start = disk_parts[i]["start"] - part.size = disk_parts[i]["size"] - part.type = disk_parts[i].get("type") -@@ -200,6 +212,7 @@ class PartitionTable: - - def partition_from_json(js) -> Partition: - p = Partition(pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -diff --git a/stages/org.osbuild.sgdisk b/stages/org.osbuild.sgdisk -index e11e0119..1627a90f 100755 ---- a/stages/org.osbuild.sgdisk -+++ b/stages/org.osbuild.sgdisk -@@ -50,6 +50,10 @@ SCHEMA_2 = r""" - "description": "The partition name", - "type": "string" - }, -+ "partnum": { -+ "description": "The partition number", -+ "type": "integer" -+ }, - "size": { - "description": "The size of this partition", - "type": "integer" -@@ -88,6 +92,7 @@ SCHEMA_2 = r""" - class Partition: - def __init__(self, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -95,6 +100,7 @@ class Partition: - uuid: str = None, - attrs: int = None): - self.type = pttype -+ self.partnum = partnum - self.start = start - self.size = size - self.name = name -@@ -129,7 +135,7 @@ class PartitionTable: - command += ["-U", self.uuid] - - for i, part in enumerate(self.partitions): -- idx = i + 1 # partitions are 1-indexed -+ idx = part.partnum if part.partnum else i + 1 # partitions are 1-indexed - - # format is 'partnum:start:end' - size = "0" -@@ -173,6 +179,7 @@ class PartitionTable: - - def partition_from_json(js) -> Partition: - p = Partition(pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -diff --git a/tools/osbuild-mpp b/tools/osbuild-mpp -index cf74488e..7bf93a18 100755 ---- a/tools/osbuild-mpp -+++ b/tools/osbuild-mpp -@@ -357,6 +357,7 @@ import hashlib - import json - import os - import pathlib -+import re - import string - import subprocess - import sys -@@ -815,6 +816,7 @@ class Partition: - def __init__(self, - uid: str = None, - pttype: str = None, -+ partnum: int = None, - start: int = None, - size: int = None, - bootable: bool = False, -@@ -830,8 +832,8 @@ class Partition: - self.name = name - self.uuid = uuid - self.attrs = attrs -- self.index = None -- self.partnum = None -+ self.index = partnum - 1 if partnum else None -+ self.partnum = partnum if partnum else None - - @property - def start_in_bytes(self): -@@ -845,6 +847,7 @@ class Partition: - def from_dict(cls, js): - p = cls(uid=js.get("id"), - pttype=js.get("type"), -+ partnum=js.get("partnum"), - start=js.get("start"), - size=js.get("size"), - bootable=js.get("bootable"), -@@ -858,6 +861,8 @@ class Partition: - - if self.start: - data["start"] = self.start -+ if self.partnum: -+ data["partnum"] = self.partnum - if self.size: - data["size"] = self.size - if self.type: -@@ -915,7 +920,10 @@ class PartitionTable: - fields += [f'{field}="{value}"'] - if partition.bootable: - fields += ["bootable"] -- command += "\n" + ", ".join(fields) -+ if partition.partnum: -+ command += "\n" + f'{target}p{partition.partnum}: ' + ", ".join(fields) -+ else: -+ command += "\n" + ", ".join(fields) - - subprocess.run(["sfdisk", "-q", "--no-tell-kernel", target], - input=command, -@@ -936,8 +944,8 @@ class PartitionTable: - - assert len(disk_parts) == len(self.partitions) - for i, part in enumerate(self.partitions): -- part.index = i -- part.partnum = i + 1 -+ part.partnum = int(re.findall(r'\d+$', disk_parts[i]["node"])[0]) -+ part.index = part.partnum - 1 - part.start = disk_parts[i]["start"] - part.size = disk_parts[i]["size"] - part.type = disk_parts[i].get("type") --- -2.43.0 - diff --git a/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch b/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch deleted file mode 100644 index 3bd536a4b8..0000000000 --- a/src/0002-fscache-add-FsCache._remove_lru-to-remove-entries.patch +++ /dev/null @@ -1,89 +0,0 @@ -From fff7dd5e1ab967165c16edec66b989adc70e39b7 Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Wed, 13 Dec 2023 10:07:32 +0100 -Subject: [PATCH 2/4] fscache: add FsCache._remove_lru() to remove entries - -The FsCache._remove_lru() removes the least recently used entry -from the cache. ---- - osbuild/util/fscache.py | 64 +++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 64 insertions(+) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index 58c9a310..32056a35 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -1086,6 +1086,70 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - objs.append(FsCacheObjectInfo(name=name, last_used=last_used)) - return sorted(objs, key=lambda obj: obj.last_used) - -+ def _remove_lru(self, required_size): -+ """" -+ Make room in the cache for "required_size" by remove the least -+ recently used entry from the cache. Note that the cache may -+ clear more than required_size. -+ """ -+ # To avoid having to take a global cache lock the strategy is: -+ # 1. Get list of (object, last_used) sorted from oldest to newest. -+ # This is racy so we need to take care of that in step(2). -+ # 2. Start with the oldest entry, try to take a write_lock -+ # (with O_NOATIME to be extra sure that atime information is -+ # correct). Get the "last_used" (atime) time and compare to what -+ # we expect in the list. If it diverges the object got load()ed -+ # while we iterated. Skip it and go to (2) again. -+ # 3. Remove entry, update cache size after the entry is removed. -+ # -+ # Note that there is a risk to get out-of-sync in (3). If the -+ # process dies while removing and before updating the cache -+ # size the cache will be over reported. -+ -+ # Try to clean at least twice the requested size to avoid having -+ # to do this all over again -+ try_to_free = required_size * 2 -+ freed_so_far = 0 -+ for name, last_used in self._last_used_objs(): -+ # take write lock for the indivdual object -+ rpath = os.path.join(self._dirname_objects, name) -+ rpath_lock = os.path.join(rpath, self._filename_object_lock) -+ # Ideally there would some lock helper instead of the low-level -+ # file manipulation to abstract this a bit more. -+ try: -+ with self._atomic_open( -+ rpath_lock, -+ wait=False, -+ write=True, -+ # atime carries the "last-used" data so don't alter it -+ oflags=os.O_EXCL | os.O_NOATIME, -+ ): -+ if last_used != self._last_used(name): -+ continue -+ # This is racy right now if the process is killed -+ # during "_rm_r_object(rpath)" because then the -+ # cache size is never reduced by the amount that -+ # was about to be deleted. -+ # -+ # To fix it we need to (atomic) rename the -+ # "object.info" file in _rm_r_object() to -+ # something like "object.removing". Then when -+ # opening the cache scan for leftover -+ # "object.removing" files and finish the cleanup -+ # and update the cache size based on the size -+ # recorded inside "object.removing". -+ size = self._calculate_space(self._path(rpath)) -+ self._rm_r_object(rpath) -+ self._update_cache_size(-size) -+ freed_so_far += size -+ if freed_so_far >= try_to_free: -+ break -+ except BlockingIOError: -+ continue -+ -+ # return True if at least the required size got freed -+ return freed_so_far > required_size -+ - @property - def info(self) -> FsCacheInfo: - """Query Cache Information --- -2.43.0 - diff --git a/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch b/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch deleted file mode 100644 index 7cde89da71..0000000000 --- a/src/0002-mounts-ostree.deployment-rename-var-root-deploy_root.patch +++ /dev/null @@ -1,49 +0,0 @@ -From e43abe1a9cbff3508eebc31d33efdc2fa44306aa Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 00:02:38 -0500 -Subject: [PATCH 2/5] mounts/ostree.deployment: rename var root -> deploy_root - -It makes things a little more clear to know the variable is pointing -to the path of the deployment. ---- - mounts/org.osbuild.ostree.deployment | 16 ++++++++-------- - 1 file changed, 8 insertions(+), 8 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index 9046fb05..f23a44da 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -105,22 +105,22 @@ class OSTreeDeploymentMount(mounts.MountService): - # - `mount(8)` - self.bind_mount(tree, tree) - -- root = ostree.deployment_path(tree, osname, ref, serial) -+ deploy_root = ostree.deployment_path(tree, osname, ref, serial) - -- print(f"Deployment root at '{os.path.relpath(root, tree)}'") -+ print(f"Deployment root at '{os.path.relpath(deploy_root, tree)}'") - - var = os.path.join(tree, "ostree", "deploy", osname, "var") - boot = os.path.join(tree, "boot") - -- self.mountpoint = root -- self.bind_mount(root, root) # prepare to move it later -+ self.mountpoint = deploy_root -+ self.bind_mount(deploy_root, deploy_root) # prepare to move it later - -- self.bind_mount(tree, os.path.join(root, "sysroot")) -- self.bind_mount(var, os.path.join(root, "var")) -- self.bind_mount(boot, os.path.join(root, "boot")) -+ self.bind_mount(tree, os.path.join(deploy_root, "sysroot")) -+ self.bind_mount(var, os.path.join(deploy_root, "var")) -+ self.bind_mount(boot, os.path.join(deploy_root, "boot")) - - subprocess.run([ -- "mount", "--move", root, tree, -+ "mount", "--move", deploy_root, tree, - ], check=True) - - self.mountpoint = tree --- -2.43.0 - diff --git a/src/0002-stages-sgdisk-support-label-option.patch b/src/0002-stages-sgdisk-support-label-option.patch deleted file mode 100644 index f66b4aa118..0000000000 --- a/src/0002-stages-sgdisk-support-label-option.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 92b7f81e1dde7c760327d0cc2170d6ce225e093b Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Tue, 30 Jan 2024 13:39:46 -0500 -Subject: [PATCH 2/2] stages(sgdisk): support label option - -We only support `gpt` here so it would seem this option doesn't -make much sense to add, but it will make it so that the mpp-define-images -from osbuild-mpp can be passed in to `org.osbuild.sgdisk` too as well -as `org.osbuild.sfdisk`. ---- - stages/org.osbuild.sgdisk | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/stages/org.osbuild.sgdisk b/stages/org.osbuild.sgdisk -index 1627a90f..f85a762a 100755 ---- a/stages/org.osbuild.sgdisk -+++ b/stages/org.osbuild.sgdisk -@@ -35,6 +35,11 @@ SCHEMA_2 = r""" - "type": "boolean", - "default": true - }, -+ "label": { -+ "description": "The type of the partition table. Only 'gpt' supported.", -+ "type": "string", -+ "enum": ["gpt"] -+ }, - "partitions": { - "description": "Partition layout ", - "type": "array", --- -2.43.0 - diff --git a/src/0002-stages-zipl.inst-support-appending-kernel-options.patch b/src/0002-stages-zipl.inst-support-appending-kernel-options.patch deleted file mode 100644 index 211f7f98f6..0000000000 --- a/src/0002-stages-zipl.inst-support-appending-kernel-options.patch +++ /dev/null @@ -1,52 +0,0 @@ -From 564e1e4841c914767ab2890bfe293cc397c4fca6 Mon Sep 17 00:00:00 2001 -From: Nikita Dubrovskii -Date: Tue, 30 Jan 2024 09:09:58 +0100 -Subject: [PATCH 2/2] stages(zipl.inst): support appending kernel options - -In some cases it could useful to add additional kernel options without -modifying BLS config, so any subsequent call to `zipl` would ignore them. -We are going to use this to append `ignition.firstboot` to kernel cmdline -for CoreOS on s390x. ---- - stages/org.osbuild.zipl.inst | 11 ++++++++++- - 1 file changed, 10 insertions(+), 1 deletion(-) - -diff --git a/stages/org.osbuild.zipl.inst b/stages/org.osbuild.zipl.inst -index 92dfd33e..96cf8cff 100755 ---- a/stages/org.osbuild.zipl.inst -+++ b/stages/org.osbuild.zipl.inst -@@ -28,6 +28,14 @@ SCHEMA_2 = r""" - "location": { - "type": "number" - }, -+ "kernel_opts_append": { -+ "description": "Additional kernel options to add to the discovered options", -+ "type": "array", -+ "items": { -+ "description": "A single kernel command line option", -+ "type": "string" -+ } -+ }, - "blocksize": { - "type": "number", - "default": 512 -@@ -93,6 +101,7 @@ def find_kernel(root, kernel: str): - def main(paths, devices, options): - kver = options["kernel"] - location = options["location"] -+ append_kopts = options.get("kernel_opts_append", []) - blocksize = options.get("blocksize", 512) - - root = paths["mounts"] -@@ -108,7 +117,7 @@ def main(paths, devices, options): - "--target", f"{root}/boot", - "--image", kernel, - "--ramdisk", initrd, -- "--parameters", kopts, -+ "--parameters", f"{kopts} {' '.join(append_kopts)}", - "--targetbase", device, - "--targettype", "SCSI", - "--targetblocksize", str(blocksize), --- -2.43.0 - diff --git a/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch b/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch deleted file mode 100644 index abb23f2088..0000000000 --- a/src/0003-fscache-use-remove_lru-to-reclaim-space-when-the-cac.patch +++ /dev/null @@ -1,79 +0,0 @@ -From 6ac1d20c8c52f3cbc6d890ed8a008f8f2d4c147b Mon Sep 17 00:00:00 2001 -From: Michael Vogt -Date: Wed, 13 Dec 2023 17:39:55 +0100 -Subject: [PATCH 3/4] fscache: use remove_lru() to reclaim space when the cache - is full - -This commit adds code that will remove the least recently used -entries when a store() operation does not succeeds because the -cache is full. To be more efficient it will try to free -twice the requested size (this can be configured in the code). ---- - osbuild/util/fscache.py | 32 +++++++++++++++++++++----------- - 1 file changed, 21 insertions(+), 11 deletions(-) - -diff --git a/osbuild/util/fscache.py b/osbuild/util/fscache.py -index 32056a35..59039522 100644 ---- a/osbuild/util/fscache.py -+++ b/osbuild/util/fscache.py -@@ -683,8 +683,11 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - self._info_maximum_size = -1 - elif isinstance(info.maximum_size, int): - self._info_maximum_size = info.maximum_size -- else: -+ elif info.maximum_size is None: - self._info_maximum_size = 0 -+ else: -+ raise ValueError( -+ f"maximum-size can only be set to 'unlimited' or an integer value, got {type(info.maximum_size)}") - - def _is_active(self): - # Internal helper to verify we are in an active context-manager. -@@ -942,19 +945,27 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - info["creation-boot-id"] = self._bootid - info["size"] = self._calculate_space(path_data) - -- # Update the total cache-size. If it exceeds the limits, bail out -- # but do not trigger an error. It behaves as if the entry was -- # committed and immediately deleted by racing cache management. No -- # need to tell the caller about it (if that is ever needed, we can -- # provide for it). -+ # Exit early if it never is going to fit -+ if self._info_maximum_size > -1 and info["size"] > self._info_maximum_size: -+ return -+ -+ # Update the total cache-size. If it exceeds the limits, remove -+ # least recently used objects until there is enough space. - # - # Note that if we crash after updating the total cache size, but - # before committing the object information, the total cache size -- # will be out of sync. However, it is never overcommitted, so we -- # will never violate any cache invariants. The cache-size will be -- # re-synchronized by any full cache-management operation. -+ # will be out of sync. -+ # -+ # However, it is never overcommitted, so we will never -+ # violate any cache invariants. Future code needs to resync -+ # the cache (e.g. on open with some simple journal strategy). - if not self._update_cache_size(info["size"]): -- return -+ # try to free space -+ self._remove_lru(info["size"]) -+ # and see if the update can happen now -+ if not self._update_cache_size(info["size"]): -+ # stil could not free enough space -+ return - - try: - # Commit the object-information, thus marking it as fully -@@ -1146,7 +1157,6 @@ class FsCache(contextlib.AbstractContextManager, os.PathLike): - break - except BlockingIOError: - continue -- - # return True if at least the required size got freed - return freed_so_far > required_size - --- -2.43.0 - diff --git a/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch b/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch deleted file mode 100644 index f1dd6f7b42..0000000000 --- a/src/0003-mounts-ostree.deployment-use-target-instead-of-tree.patch +++ /dev/null @@ -1,71 +0,0 @@ -From be90d8c36c0f0a6846a475f394843e898ad35a24 Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 00:12:01 -0500 -Subject: [PATCH 3/5] mounts/ostree.deployment: use target instead of tree - -We still target the tree here, but we open ourselves up to be able -to target something other than the tree in the future. This mostly -exchanges the `tree` variable for `target`. - -We also update the comment to try to enhance clarity. ---- - mounts/org.osbuild.ostree.deployment | 29 ++++++++++++++++------------ - 1 file changed, 17 insertions(+), 12 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index f23a44da..b200eca3 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -98,32 +98,37 @@ class OSTreeDeploymentMount(mounts.MountService): - ref = deployment["ref"] - serial = deployment.get("serial", 0) - -- # create a private mountpoint for the tree, which is needed -- # in order to be able to move the `root` mountpoint, which -- # is contained inside tree, since "moving a mount residing -- # under a shared mount is invalid and unsupported." -+ # The target path where we want the deployment to be mounted -+ # is the root of the tree. -+ target = tree -+ -+ # create a private mountpoint for the target path, which is -+ # needed in order to be able to move the deployment `root` -+ # mountpoint here, which is contained inside tree, since -+ # "moving a mount residing under a shared mount is invalid -+ # and unsupported." - # - `mount(8)` -- self.bind_mount(tree, tree) -+ self.bind_mount(target, target) - -- deploy_root = ostree.deployment_path(tree, osname, ref, serial) -+ deploy_root = ostree.deployment_path(target, osname, ref, serial) - -- print(f"Deployment root at '{os.path.relpath(deploy_root, tree)}'") -+ print(f"Deployment root at '{os.path.relpath(deploy_root, target)}'") - -- var = os.path.join(tree, "ostree", "deploy", osname, "var") -- boot = os.path.join(tree, "boot") -+ var = os.path.join(target, "ostree", "deploy", osname, "var") -+ boot = os.path.join(target, "boot") - - self.mountpoint = deploy_root - self.bind_mount(deploy_root, deploy_root) # prepare to move it later - -- self.bind_mount(tree, os.path.join(deploy_root, "sysroot")) -+ self.bind_mount(target, os.path.join(deploy_root, "sysroot")) - self.bind_mount(var, os.path.join(deploy_root, "var")) - self.bind_mount(boot, os.path.join(deploy_root, "boot")) - - subprocess.run([ -- "mount", "--move", deploy_root, tree, -+ "mount", "--move", deploy_root, target, - ], check=True) - -- self.mountpoint = tree -+ self.mountpoint = target - self.check = True - - def umount(self): --- -2.43.0 - diff --git a/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch b/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch deleted file mode 100644 index c9a67630c7..0000000000 --- a/src/0004-mounts-ostree.deployment-support-deployments-on-moun.patch +++ /dev/null @@ -1,149 +0,0 @@ -From bd6b8ffb83384e7f9e78dc42a9cee626830b990f Mon Sep 17 00:00:00 2001 -From: Dusty Mabe -Date: Wed, 10 Jan 2024 11:07:14 -0500 -Subject: [PATCH 4/5] mounts/ostree.deployment: support deployments on mount - -Instead of operating directly on the tree for a stage we can operate -on a mount too. This is useful in the case where operating on the -directory tree of files isn't sufficient and the modifications need -to be made directly to the filesystems on the disk image that we are -creating. - -One such example of this is we are having a problem right now where -the immutable bit being set on an OSTree deployment root doesn't -survive the `cp -a --reflink=auto` in the org.osbuild.copy stage when -being copied from the directory tree into the mounted XFS filesystem -we created on the disk image. Thus we have to workaround this loss -of attribute by applying the attribute directly on the mounted -filesystem from the disk. - -In this change here we also add a check in osbuild/mounts.py to not -attempt a umount of the root of the mounts directory if that path -is no longer a mountpoint, which can happen when the umount -R -from the mounts/org.osbuild.ostree.deployment also removes the -overmount. - -Here is an example of how this would be used: - -``` - - type: org.osbuild.chattr - options: - immutable: true - path: mount://root/ - devices: - disk: - type: org.osbuild.loopback - options: - filename: disk.img - partscan: true - mounts: - - name: root - type: org.osbuild.xfs - source: disk - partition: - mpp-format-int: '{image.layout[''root''].partnum}' - target: / - - name: ostree.deployment - type: org.osbuild.ostree.deployment - options: - source: mount - deployment: - osname: fedora-coreos - ref: ostree/1/1/0 -``` - -The initial mount on `/` is the filesystem from the root partition -on the disk. The second mount (of type org.osbuild.ostree.deployment) -then reconfigures things similar to how an OSTree system is set up. ---- - mounts/org.osbuild.ostree.deployment | 18 +++++++++++++++--- - osbuild/mounts.py | 13 ++++++++++--- - 2 files changed, 25 insertions(+), 6 deletions(-) - -diff --git a/mounts/org.osbuild.ostree.deployment b/mounts/org.osbuild.ostree.deployment -index b200eca3..24df7731 100755 ---- a/mounts/org.osbuild.ostree.deployment -+++ b/mounts/org.osbuild.ostree.deployment -@@ -32,6 +32,12 @@ SCHEMA_2 = """ - "type": "object", - "required": ["deployment"], - "properties": { -+ "source": { -+ "type": "string", -+ "pattern": "^(mount|tree)$", -+ "default": "tree", -+ "description": "The source of the OSTree filesystem tree. If 'mount', there should be a preceding mount defined that's mounted at /." -+ }, - "deployment": { - "type": "object", - "additionalProperties": false, -@@ -91,16 +97,21 @@ class OSTreeDeploymentMount(mounts.MountService): - def mount(self, args: Dict): - - tree = args["tree"] -+ mountroot = args["root"] - options = args["options"] - -+ source = options.get("source", "tree") - deployment = options["deployment"] - osname = deployment["osname"] - ref = deployment["ref"] - serial = deployment.get("serial", 0) - -- # The target path where we want the deployment to be mounted -- # is the root of the tree. -- target = tree -+ # The user could specify either the tree or mountroot as the -+ # place where we want the deployment to be mounted. -+ if source == "mount": -+ target = mountroot -+ else: -+ target = tree - - # create a private mountpoint for the target path, which is - # needed in order to be able to move the deployment `root` -@@ -113,6 +124,7 @@ class OSTreeDeploymentMount(mounts.MountService): - deploy_root = ostree.deployment_path(target, osname, ref, serial) - - print(f"Deployment root at '{os.path.relpath(deploy_root, target)}'") -+ print(f"mounting {deploy_root} -> {target}") - - var = os.path.join(target, "ostree", "deploy", osname, "var") - boot = os.path.join(target, "boot") -diff --git a/osbuild/mounts.py b/osbuild/mounts.py -index b938d21d..42b556ba 100644 ---- a/osbuild/mounts.py -+++ b/osbuild/mounts.py -@@ -181,6 +181,8 @@ class FileSystemMountService(MountService): - os.makedirs(mountpoint, exist_ok=True) - self.mountpoint = mountpoint - -+ print(f"mounting {source} -> {mountpoint}") -+ - try: - subprocess.run( - ["mount"] + -@@ -203,12 +205,17 @@ class FileSystemMountService(MountService): - if not self.mountpoint: - return - -- self.sync() -+ # It's possible this mountpoint has already been unmounted -+ # if a umount -R was run by another process, as is done in -+ # mounts/org.osbuild.ostree.deployment. -+ if not os.path.ismount(self.mountpoint): -+ print(f"already unmounted: {self.mountpoint}") -+ return - -- print("umounting") -+ self.sync() - - # We ignore errors here on purpose -- subprocess.run(["umount", self.mountpoint], -+ subprocess.run(["umount", "-v", self.mountpoint], - check=self.check) - self.mountpoint = None - --- -2.43.0 - diff --git a/src/0005-Create-stages-org.osbuild.chattr-stage.patch b/src/0005-Create-stages-org.osbuild.chattr-stage.patch deleted file mode 100644 index ac2a3a4c81..0000000000 --- a/src/0005-Create-stages-org.osbuild.chattr-stage.patch +++ /dev/null @@ -1,123 +0,0 @@ -From 477a21043eeb55468b1f75a9574e46e77cb5fdef Mon Sep 17 00:00:00 2001 -From: Luke Yang -Date: Mon, 22 Jan 2024 10:28:01 -0500 -Subject: [PATCH 5/5] Create stages/org.osbuild.chattr stage - -Add or remove the immutable bit to the specified mount directory. - -The need we have for this right now is for the CoreOS builds where -the immutable bit being set on an OSTree deployment root doesn't -survive the `cp -a --reflink=auto` in the org.osbuild.copy stage when -being copied from the directory tree into the mounted XFS filesystem -we created on the disk image. Thus we have to workaround this loss -of attribute by applying the attribute directly on the mounted -filesystem from the disk. ---- - stages/org.osbuild.chattr | 95 +++++++++++++++++++++++++++++++++++++++ - 1 file changed, 95 insertions(+) - create mode 100755 stages/org.osbuild.chattr - -diff --git a/stages/org.osbuild.chattr b/stages/org.osbuild.chattr -new file mode 100755 -index 00000000..0bc8aa0e ---- /dev/null -+++ b/stages/org.osbuild.chattr -@@ -0,0 +1,95 @@ -+#!/usr/bin/python3 -+""" -+Runs `chattr` to set file/directory attributes. -+""" -+ -+import os -+import subprocess -+import sys -+from typing import Dict -+from urllib.parse import ParseResult, urlparse -+ -+import osbuild.api -+ -+SCHEMA_2 = r""" -+"options": { -+ "additionalProperties": false, -+ "properties": { -+ "items": { -+ "type": "object", -+ "additionalProperties": false, -+ "patternProperties": { -+ "^mount:\/\/[^\/]+\/|^tree:\/\/\/": { -+ "type": "object", -+ "required": ["immutable"], -+ "properties": { -+ "immutable": { -+ "type": "boolean", -+ "description": "Make the file/directory immutable", -+ "default": true -+ } -+ } -+ } -+ } -+ } -+ } -+}, -+"devices": { -+ "type": "object", -+ "additionalProperties": true -+}, -+"mounts": { -+ "type": "array" -+} -+""" -+ -+ -+def parse_mount(url: ParseResult, args: Dict): -+ name = url.netloc -+ if name: -+ root = args["mounts"].get(name, {}).get("path") -+ if not root: -+ raise ValueError(f"Unknown mount '{name}'") -+ else: -+ root = args["paths"]["mounts"] -+ -+ return root -+ -+ -+def parse_location(location, args): -+ url = urlparse(location) -+ -+ scheme = url.scheme -+ if scheme == "tree": -+ root = args["tree"] -+ elif scheme == "mount": -+ root = parse_mount(url, args) -+ else: -+ raise ValueError(f"Unsupported scheme '{scheme}'") -+ -+ assert url.path.startswith("/") -+ -+ path = os.path.relpath(url.path, "/") -+ path = os.path.join(root, path) -+ path = os.path.normpath(path) -+ -+ if url.path.endswith("/"): -+ path = os.path.join(path, ".") -+ -+ return path -+ -+ -+def main(args, options): -+ for path, cmdargs in options["items"].items(): -+ immutable = cmdargs["immutable"] -+ dst = parse_location(path, args) -+ op = '+' if immutable else '-' -+ subprocess.run(["chattr", f"{op}i", dst], check=True) -+ -+ return 0 -+ -+ -+if __name__ == '__main__': -+ _args = osbuild.api.arguments() -+ r = main(_args, _args["options"]) -+ sys.exit(r) --- -2.43.0 - diff --git a/src/cmdlib.sh b/src/cmdlib.sh index be616a912d..95b20a0ece 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -607,7 +607,7 @@ runvm_with_cache_snapshot() { ( # shellcheck source=src/libguestfish.sh source /usr/lib/coreos-assembler/libguestfish.sh - virt-format --filesystem=xfs --label=cosa-cache -a cache2.qcow2.tmp) + virt-format --filesystem=ext4 --label=cosa-cache -a cache2.qcow2.tmp) mv -T cache2.qcow2.tmp "${workdir}"/cache/cache2.qcow2 fi # And remove the old one @@ -729,13 +729,6 @@ runvm() { # include COSA in the image find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles" - # include new patched in osbuild stage in the image. - # can drop this once the upstream PRs are merged: - # https://github.com/osbuild/osbuild/pull/1519 - # https://github.com/osbuild/osbuild/pull/1535 - # shellcheck disable=SC2129 - echo /usr/lib/osbuild/stages/org.osbuild.chattr >> "${vmpreparedir}/hostfiles" - # and include all GPG keys find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles" @@ -753,17 +746,24 @@ export USER=$(id -u) export RUNVM_NONET=${RUNVM_NONET:-} $(cat "${DIR}"/supermin-init-prelude.sh) rc=0 -# tee to the virtio port so its output is also part of the supermin output in -# case e.g. a key msg happens in dmesg when the command does a specific operation +# - tee to the virtio port so its output is also part of the supermin output in +# case e.g. a key msg happens in dmesg when the command does a specific operation. +# - Use a subshell because otherwise init will use workdir as its cwd and we won't +# be able to unmount the virtiofs mount cleanly. This leads to consistency issues. if [ -z "${RUNVM_SHELL:-}" ]; then - bash ${tmp_builddir}/cmd.sh |& tee /dev/virtio-ports/cosa-cmdout || rc=\$? + (cd ${workdir}; bash ${tmp_builddir}/cmd.sh |& tee /dev/virtio-ports/cosa-cmdout) || rc=\$? else - bash; poweroff -f -f; sleep infinity + (cd ${workdir}; bash) fi echo \$rc > ${rc_file} if [ -n "\${cachedev}" ]; then /sbin/fstrim -v ${workdir}/cache + mount -o remount,ro ${workdir}/cache + fsfreeze -f ${workdir}/cache + fsfreeze -u ${workdir}/cache + umount ${workdir}/cache fi +umount ${workdir} /sbin/reboot -f EOF chmod a+x "${vmpreparedir}"/init diff --git a/src/coreos.osbuild.aarch64.mpp.yaml b/src/coreos.osbuild.aarch64.mpp.yaml index 2eb460e2db..8d652580df 100644 --- a/src/coreos.osbuild.aarch64.mpp.yaml +++ b/src/coreos.osbuild.aarch64.mpp.yaml @@ -674,5 +674,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.ppc64le.mpp.yaml b/src/coreos.osbuild.ppc64le.mpp.yaml index 669eabd7a0..f74cd68aa5 100644 --- a/src/coreos.osbuild.ppc64le.mpp.yaml +++ b/src/coreos.osbuild.ppc64le.mpp.yaml @@ -641,5 +641,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.s390x.mpp.yaml b/src/coreos.osbuild.s390x.mpp.yaml index 42f7e1f1a6..99a307c6b3 100644 --- a/src/coreos.osbuild.s390x.mpp.yaml +++ b/src/coreos.osbuild.s390x.mpp.yaml @@ -640,5 +640,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/coreos.osbuild.x86_64.mpp.yaml b/src/coreos.osbuild.x86_64.mpp.yaml index 1d2569f353..ff34d0953d 100644 --- a/src/coreos.osbuild.x86_64.mpp.yaml +++ b/src/coreos.osbuild.x86_64.mpp.yaml @@ -680,5 +680,5 @@ pipelines: mpp-format-string: '{filename}' format: type: qcow2 - compression: true + compression: false compat: '1.1' diff --git a/src/runvm-osbuild b/src/runvm-osbuild index 348f5925cb..1a651d3e18 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -65,10 +65,11 @@ fi # Since it doesn't exist create loop-control mknod /dev/loop-control c 10 237 -# Tell osbuild to write out artifacts into a file in the root -# filesystem of the supermin VM, which is ephemeral. -mkdir /var/osbuild -outdir=/var/osbuild/out +# Put the store and the output dir on the cache. At the end we'll mv +# out the created artifact from the output dir to the place it's supposed +# to go. +outdir=cache/osbuild/out +storedir=cache/osbuild/store # Run through the preprocessor # Note: don't quote the size arguements since they are numbers, not strings @@ -82,17 +83,18 @@ osbuild-mpp \ -D metal_image_size_mb="${metal_image_size_mb}" \ -D cloud_image_size_mb="${cloud_image_size_mb}" \ "${mppyaml}" \ - /var/osbuild/processed.json + /processed.json # Build the image osbuild \ - --out "$outdir" \ - --store cache/osbuild/store/ \ + --out "$outdir" \ + --store "$storedir" \ --cache-max-size 9GiB \ --checkpoint tree \ --checkpoint raw-image \ - --export "$platform" /var/osbuild/processed.json + --export "$platform" /processed.json -# Copy it out to the specified location -cp "${outdir}/${platform}/${filename}" "${filepath}" +# Copy it out to the specified location. Use mv here so we remove it +# from the cache qcow2 so we don't cache it. +mv "${outdir}/${platform}/${filename}" "${filepath}" diff --git a/src/supermin-init-prelude.sh b/src/supermin-init-prelude.sh index 8cf8150d50..24eed7ea76 100644 --- a/src/supermin-init-prelude.sh +++ b/src/supermin-init-prelude.sh @@ -69,6 +69,3 @@ touch /etc/cosa-supermin # the missing link. Hehe. update-alternatives --install /etc/alternatives/iptables iptables /usr/sbin/iptables-legacy 1 update-alternatives --install /etc/alternatives/ip6tables ip6tables /usr/sbin/ip6tables-legacy 1 - -# https://github.com/koalaman/shellcheck/wiki/SC2164 -cd "${workdir}" || exit