From ed1467e15f49f27d83a2d58dea73cbe14cc6c053 Mon Sep 17 00:00:00 2001
From: Jakob Meng <code@jakobmeng.de>
Date: Thu, 24 Aug 2023 10:52:26 +0200
Subject: [PATCH] Implement workaround to allow SNO installations for OKD/FCOS

OKD/FCOS uses FCOS as its bootimage, i.e. when booting cluster nodes
the first time during installation. FCOS does not provide tools such
as OpenShift Client (oc) or hyperkube which are used during
single-node cluster installation at first boot (e.g. oc in
bootkube.sh [0]). RHCOS and SCOS include these tools, but FCOS has to
pivot the root fs [1] to okd-machine-os [2] first in order to make
those tools available.

Pivoting uses 'rpm-ostree rebase' but during SNO installation the node
will be booted from a FCOS Live ISO where the root fs and /sysroot are
mounted read-only. Thus 'rpm-ostree rebase' fails and necessary tools
for SNO installation will not be available, causing the setup to stall.

This patch works around this issue by mounting /usr and /etc as
overlay filesystems which contains both the content of the live iso as
well as from the okd-machine-os container image.

[0] https://github.com/openshift/installer/blob/master/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
[1] https://github.com/openshift/installer/blob/master/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template
[2] https://github.com/openshift/okd-machine-os
---
 .../usr/local/bin/bootstrap-pivot.sh.template     | 15 +++++++++++++++
 .../units/release-image-pivot.service.template    |  4 ++++
 2 files changed, 19 insertions(+)

diff --git a/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template b/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template
index e0a3aa7cd55..86a42bb9cc2 100644
--- a/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template
+++ b/data/data/bootstrap/files/usr/local/bin/bootstrap-pivot.sh.template
@@ -42,10 +42,25 @@ if [ ! -f /opt/openshift/.pivot-done ]; then
   record_service_stage_start "rebase-to-okd-os-image"
 {{if .IsFCOS -}}
   mnt="$(podman image mount "${MACHINE_OS_IMAGE}")"
+{{- if .BootstrapInPlace }}
+  # SNO setup boots into Live ISO which cannot be rebased
+  # https://github.com/coreos/rpm-ostree/issues/4547
+  mkdir /var/mnt/{upper,worker}
+  mount -t overlay overlay -o "lowerdir=/usr:$mnt/usr" /usr
+  mount -t overlay overlay -o "lowerdir=/etc:$mnt/etc,upperdir=/var/mnt/upper,workdir=/var/mnt/worker" /etc
+  systemctl daemon-reload
+
+  # Workaround for SELinux denials when launching crio.service from overlayfs
+  setenforce Permissive
+
+  systemctl start crio.service
+  # No reboot necessary because SNO setup will reboot system
+{{ else }}
   pushd "${mnt}/bootstrap"
     # shellcheck disable=SC1091
     . ./pre-pivot.sh
   popd
+{{ end -}}
 {{else if .IsSCOS -}}
   chmod 0644 /etc/containers/registries.conf
   rpm-ostree rebase --experimental "ostree-unverified-registry:${MACHINE_OS_IMAGE}"
diff --git a/data/data/bootstrap/systemd/units/release-image-pivot.service.template b/data/data/bootstrap/systemd/units/release-image-pivot.service.template
index aac2f76ece1..e278091cc50 100644
--- a/data/data/bootstrap/systemd/units/release-image-pivot.service.template
+++ b/data/data/bootstrap/systemd/units/release-image-pivot.service.template
@@ -3,7 +3,11 @@
 Description=Pivot bootstrap to the OpenShift Release Image
 Wants=release-image.service
 After=release-image.service
+{{- if .BootstrapInPlace }}
+Before=bootkube.service kubelet.service
+{{ else }}
 Before=bootkube.service
+{{ end -}}
 
 [Service]
 Type=oneshot