Add boot state markers: ssh-ready and boot-done

The boot scripts must terminate an existing user session after
/etc/environment has been updated to make sure the user session
(which may linger) has the updated values.

This reset will break the SSH control path, breaking sshfs mounts,
so the hostagent must wait until the instance is "ss-ready" for
persistent connections.

A similar "boot-done" status check is added as a "final" requirement
so that `limactl start` doesn't return until all the boot scripts
have finished.

Signed-off-by: Jan Dubois <[email protected]>

Author: jandubois

Diff for: pkg/cidata/cidata.TEMPLATE.d/boot.sh

@@ -59,5 +59,9 @@ if [ -d "${LIMA_CIDATA_MNT}"/provision.user ]; then
 	done
 fi
 
+# Signal that provisioning is done. The instance-id in the meta-data file changes on every boot,
+# so any copy from a previous boot cycle will have different content.
+cp "${LIMA_CIDATA_MNT}"/meta-data /etc/lima-boot-done
+
 INFO "Exiting with code $CODE"
 exit "$CODE"

Diff for: pkg/cidata/cidata.TEMPLATE.d/boot/07-etc-environment.sh

@@ -16,3 +16,10 @@ cat "${LIMA_CIDATA_MNT}/etc_environment" >>/etc/environment
 if command -v loginctl >/dev/null 2>&1; then
 	loginctl terminate-user "${LIMA_CIDATA_USER}" || true
 fi
+
+# Make sure the guestagent socket from a previous boot is removed before we open the "lima-ssh-ready" gate.
+rm -f /run/lima-guest-agent.sock
+
+# Signal that provisioning is done. The instance-id in the meta-data file changes on every boot,
+# so any copy from a previous boot cycle will have different content.
+cp "${LIMA_CIDATA_MNT}"/meta-data /etc/lima-ssh-ready

Diff for: pkg/hostagent/hostagent.go

@@ -388,6 +388,9 @@ func (a *HostAgent) startHostAgentRoutines(ctx context.Context) error {
 	if err := a.waitForRequirements(ctx, "optional", a.optionalRequirements()); err != nil {
 		mErr = multierror.Append(mErr, err)
 	}
+	if err := a.waitForRequirements(ctx, "final", a.finalRequirements()); err != nil {
+		mErr = multierror.Append(mErr, err)
+	}
 	return mErr
 }
 

Diff for: pkg/hostagent/requirements.go

@@ -60,16 +60,33 @@ type requirement struct {
 
 func (a *HostAgent) essentialRequirements() []requirement {
 	req := make([]requirement, 0)
-	req = append(req, requirement{
-		description: "ssh",
-		script: `#!/bin/bash
+	req = append(req,
+		requirement{
+			description: "ssh",
+			script: `#!/bin/bash
 true
 `,
-		debugHint: `Failed to SSH into the guest.
+			debugHint: `Failed to SSH into the guest.
 Make sure that the YAML field "ssh.localPort" is not used by other processes on the host.
 If any private key under ~/.ssh is protected with a passphrase, you need to have ssh-agent to be running.
 `,
-	})
+		},
+		requirement{
+			description: "user session is ready for ssh",
+			script: `#!/bin/bash
+set -eux -o pipefail
+if ! timeout 30s bash -c "until sudo diff -q /etc/lima-ssh-ready /mnt/lima-cidata/meta-data 2>/dev/null; do sleep 3; done"; then
+	echo >&2 "not ready to start persistent ssh session"
+	exit 1
+fi
+`,
+			debugHint: `The boot sequence will terminate any existing user session after updating
+/etc/environment to make sure the session includes the new values.
+Terminating the session will break the persistent SSH tunnel, so
+it must not be created until the session reset is done.
+`,
+		})
+
 	if len(a.y.Mounts) > 0 {
 		req = append(req, requirement{
 			description: "sshfs binary to be installed",
@@ -164,3 +181,23 @@ Also see "/var/log/cloud-init-output.log" in the guest.
 	}
 	return req
 }
+
+func (a *HostAgent) finalRequirements() []requirement {
+	req := make([]requirement, 0)
+	req = append(req,
+		requirement{
+			description: "boot scripts must have finished",
+			script: `#!/bin/bash
+set -eux -o pipefail
+if ! timeout 30s bash -c "until sudo diff -q /etc/lima-boot-done /mnt/lima-cidata/meta-data 2>/dev/null; do sleep 3; done"; then
+	echo >&2 "boot scripts have not finished"
+	exit 1
+fi
+`,
+			debugHint: `All boot scripts, provisioning scripts, and readiness probes must
+finish before the instance is considered "ready".
+Check "/var/log/cloud-init-output.log" in the guest to see where the process is blocked!
+`,
+		})
+	return req
+}