bootstrap: record progress of services

data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/bootstrap-in-place.sh

@@ -12,6 +12,7 @@ bootkube_podman_run() {
 }
 
 if [ ! -f stop-etcd.done ]; then
+  record_service_stage_start "stop-etcd"
   echo "Stop etcd static pod by moving the manifest"
   mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes || echo "already moved etcd-member-pod.yaml"
 
@@ -21,9 +22,11 @@ if [ ! -f stop-etcd.done ]; then
   done
 
   touch stop-etcd.done
+  record_service_stage_success
 fi
 
 if [ ! -f master-ignition.done ]; then
+  record_service_stage_start "master-ignition"
   echo "Creating master ignition and writing it to disk"
   # Get the master ignition from MCS
   curl --header 'Accept:application/vnd.coreos.ignition+json;version=3.2.0' \
@@ -50,4 +53,5 @@ if [ ! -f master-ignition.done ]; then
     --output /assets/master.ign
 
   touch master-ignition.done
+  record_service_stage_success
 fi

data/data/bootstrap/bootstrap-in-place/files/usr/local/bin/install-to-disk.sh.template

@@ -3,20 +3,28 @@ set -euoE pipefail ## -E option will cause functions to inherit trap
 
 # This script is executed by install-to-disk service when installing single node with bootstrap in place
 
+. /usr/local/bin/bootstrap-service-record.sh
+
+record_service_stage_start "wait-for-bootkube"
 echo "Waiting for /opt/openshift/.bootkube.done"
 until [ -f /opt/openshift/.bootkube.done ]; do
   sleep 5
 done
+record_service_stage_success
 
 if [ ! -f coreos-installer.done ]; then
+  record_service_stage_start "coreos-installer"
   # Write image + ignition to disk
   echo "Executing coreos-installer with the following options: install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}"
   coreos-installer install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}
 
   touch coreos-installer.done
+  record_service_stage_success
 fi
 
+record_service_stage_start "reboot"
 echo "Going to reboot"
 shutdown -r +1 "Bootstrap completed, server is going to reboot."
 touch /opt/openshift/.install-to-disk.done
 echo "Done"
+record_service_stage_success

data/data/bootstrap/files/usr/local/bin/approve-csr.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# shellcheck disable=SC1091  # using path on bootstrap machine
+. /usr/local/bin/bootstrap-service-record.sh
+
 KUBECONFIG="${1}"
 
 echo "Approving all CSR requests until bootstrapping is complete..."

data/data/bootstrap/files/usr/local/bin/bootkube.sh.template

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euoE pipefail ## -E option will cause functions to inherit trap
 
+. /usr/local/bin/bootstrap-service-record.sh
+
 . /usr/local/bin/release-image.sh
 
 mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
@@ -68,13 +70,16 @@ mkdir --parents ./{bootstrap-manifests,manifests}
 
 if [ ! -f openshift-manifests.done ]
 then
+    record_service_stage_start "openshift-manifests"
 	echo "Moving OpenShift manifests in with the rest of them"
 	cp openshift/* manifests/
 	touch openshift-manifests.done
+    record_service_stage_success
 fi
 
 if [ ! -f cvo-bootstrap.done ]
 then
+    record_service_stage_start "cvo-bootstrap"
 	echo "Rendering Cluster Version Operator Manifests..."
 
 	rm --recursive --force cvo-bootstrap
@@ -95,11 +100,13 @@ then
 	cp auth/kubeconfig-loopback /etc/kubernetes/kubeconfig
 
 	touch cvo-bootstrap.done
+    record_service_stage_success
 fi
 
 ETCD_ENDPOINTS=https://localhost:2379
 if [ ! -f etcd-bootstrap.done ]
 then
+    record_service_stage_start "etcd-bootstrap"
 	echo "Rendering CEO Manifests..."
 	bootkube_podman_run \
 		--volume "$PWD:/assets:z" \
@@ -131,10 +138,12 @@ then
 	fi
 
 	touch etcd-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f config-bootstrap.done ]
 then
+    record_service_stage_start "config-bootstrap"
 	echo "Rendering cluster config manifests..."
 
 	rm --recursive --force config-bootstrap
@@ -158,10 +167,12 @@ then
 	cp config-bootstrap/manifests/* manifests/
 
 	touch config-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f kube-apiserver-bootstrap.done ]
 then
+    record_service_stage_start "kube-apiserver-bootstrap"
 	echo "Rendering Kubernetes API server core manifests..."
 
 	rm --recursive --force kube-apiserver-bootstrap
@@ -185,10 +196,12 @@ then
 	cp kube-apiserver-bootstrap/manifests/* manifests/
 
 	touch kube-apiserver-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f kube-controller-manager-bootstrap.done ]
 then
+    record_service_stage_start "kube-controller-manager-bootstrap"
 	echo "Rendering Kubernetes Controller Manager core manifests..."
 
 	rm --recursive --force kube-controller-manager-bootstrap
@@ -211,10 +224,12 @@ then
 	cp kube-controller-manager-bootstrap/manifests/* manifests/
 
 	touch kube-controller-manager-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f kube-scheduler-bootstrap.done ]
 then
+    record_service_stage_start "kube-scheduler-bootstrap"
 	echo "Rendering Kubernetes Scheduler core manifests..."
 
 	rm --recursive --force kube-scheduler-bootstrap
@@ -233,10 +248,12 @@ then
 	cp kube-scheduler-bootstrap/manifests/* manifests/
 
 	touch kube-scheduler-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f ingress-operator-bootstrap.done ]
 then
+    record_service_stage_start "ingress-operator-bootstrap"
 	echo "Rendering Ingress Operator core manifests..."
 
 	rm --recursive --force ingress-operator-bootstrap
@@ -251,10 +268,12 @@ then
 	cp ingress-operator-manifests/* manifests/
 
 	touch ingress-operator-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f mco-bootstrap.done ]
 then
+    record_service_stage_start "mco-bootstrap"
 	echo "Rendering MCO manifests..."
 
 	rm --recursive --force mco-bootstrap
@@ -323,10 +342,12 @@ then
 	cp tls/machine-config-server.key /etc/ssl/mcs/tls.key
 
 	touch mco-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ ! -f cco-bootstrap.done ]
 then
+    record_service_stage_start "cco-bootstrap"
 	echo "Rendering CCO manifests..."
 
 	rm --recursive --force cco-bootstrap
@@ -349,13 +370,16 @@ then
 	fi
 
 	touch cco-bootstrap.done
+    record_service_stage_success
 fi
 
 # in case of single node, if we removed etcd, there is no point to wait for it on restart
 if [ ! -f stop-etcd.done ]
 then
+    record_service_stage_start "wait-for-etcd"
 	# Wait for the etcd cluster to come up.
 	wait_for_etcd_cluster
+    record_service_stage_success
 fi
 
 REQUIRED_PODS="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
@@ -368,6 +392,7 @@ echo "Starting cluster-bootstrap..."
 
 if [ ! -f cb-bootstrap.done ]
 then
+    record_service_stage_start "cb-bootstrap"
     bootkube_podman_run \
         --rm \
         --volume "$PWD:/assets:z" \
@@ -376,6 +401,7 @@ then
         start --tear-down-early=false --asset-dir=/assets --required-pods="${REQUIRED_PODS}"
 
     touch cb-bootstrap.done
+    record_service_stage_success
 fi
 
 if [ "$BOOTSTRAP_INPLACE" = true ]
@@ -386,15 +412,18 @@ else
 
     if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ]
     then
+        record_service_stage_start "wait-for-ceo"
         echo "Waiting for CEO to finish..."
         bootkube_podman_run \
             --volume "$PWD:/assets:z" \
             "${CLUSTER_ETCD_OPERATOR_IMAGE}" \
             /usr/bin/cluster-etcd-operator \
             wait-for-ceo \
             --kubeconfig /assets/auth/kubeconfig
+        record_service_stage_success
     fi
 fi
+
 # Workaround for https://github.com/opencontainers/runc/pull/1807
 touch /opt/openshift/.bootkube.done
 echo "bootkube.service complete"