Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ bootkube_podman_run() {
}

if [ ! -f stop-etcd.done ]; then
record_service_stage_start "stop-etcd"
echo "Stop etcd static pod by moving the manifest"
mv /etc/kubernetes/manifests/etcd-member-pod.yaml /etc/kubernetes || echo "already moved etcd-member-pod.yaml"

Expand All @@ -21,9 +22,11 @@ if [ ! -f stop-etcd.done ]; then
done

touch stop-etcd.done
record_service_stage_success
fi

if [ ! -f master-ignition.done ]; then
record_service_stage_start "master-ignition"
echo "Creating master ignition and writing it to disk"
# Get the master ignition from MCS
curl --header 'Accept:application/vnd.coreos.ignition+json;version=3.2.0' \
Expand All @@ -50,4 +53,5 @@ if [ ! -f master-ignition.done ]; then
--output /assets/master.ign

touch master-ignition.done
record_service_stage_success
fi
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,28 @@ set -euoE pipefail ## -E option will cause functions to inherit trap

# This script is executed by install-to-disk service when installing single node with bootstrap in place

. /usr/local/bin/bootstrap-service-record.sh

record_service_stage_start "wait-for-bootkube"
echo "Waiting for /opt/openshift/.bootkube.done"
until [ -f /opt/openshift/.bootkube.done ]; do
sleep 5
done
record_service_stage_success

if [ ! -f coreos-installer.done ]; then
record_service_stage_start "coreos-installer"
# Write image + ignition to disk
echo "Executing coreos-installer with the following options: install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}"
coreos-installer install -i /opt/openshift/master.ign {{.BootstrapInPlace.InstallationDisk}}

touch coreos-installer.done
record_service_stage_success
fi

record_service_stage_start "reboot"
echo "Going to reboot"
shutdown -r +1 "Bootstrap completed, server is going to reboot."
touch /opt/openshift/.install-to-disk.done
echo "Done"
record_service_stage_success
2 changes: 2 additions & 0 deletions data/data/bootstrap/files/usr/local/bin/approve-csr.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/usr/bin/env bash

. /usr/local/bin/bootstrap-service-record.sh

KUBECONFIG="${1}"

echo "Approving all CSR requests until bootstrapping is complete..."
Expand Down
29 changes: 29 additions & 0 deletions data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#!/usr/bin/env bash
set -euoE pipefail ## -E option will cause functions to inherit trap

. /usr/local/bin/bootstrap-service-record.sh

. /usr/local/bin/release-image.sh

mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests}
Expand Down Expand Up @@ -68,13 +70,16 @@ mkdir --parents ./{bootstrap-manifests,manifests}

if [ ! -f openshift-manifests.done ]
then
record_service_stage_start "openshift-manifests"
echo "Moving OpenShift manifests in with the rest of them"
cp openshift/* manifests/
touch openshift-manifests.done
record_service_stage_success
fi

if [ ! -f cvo-bootstrap.done ]
then
record_service_stage_start "cvo-bootstrap"
echo "Rendering Cluster Version Operator Manifests..."

rm --recursive --force cvo-bootstrap
Expand All @@ -95,11 +100,13 @@ then
cp auth/kubeconfig-loopback /etc/kubernetes/kubeconfig

touch cvo-bootstrap.done
record_service_stage_success
fi

ETCD_ENDPOINTS=https://localhost:2379
if [ ! -f etcd-bootstrap.done ]
then
record_service_stage_start "etcd-bootstrap"
echo "Rendering CEO Manifests..."
bootkube_podman_run \
--volume "$PWD:/assets:z" \
Expand Down Expand Up @@ -131,10 +138,12 @@ then
fi

touch etcd-bootstrap.done
record_service_stage_success
fi

if [ ! -f config-bootstrap.done ]
then
record_service_stage_start "config-bootstrap"
echo "Rendering cluster config manifests..."

rm --recursive --force config-bootstrap
Expand All @@ -158,10 +167,12 @@ then
cp config-bootstrap/manifests/* manifests/

touch config-bootstrap.done
record_service_stage_success
fi

if [ ! -f kube-apiserver-bootstrap.done ]
then
record_service_stage_start "kube-apiserver-bootstrap"
echo "Rendering Kubernetes API server core manifests..."

rm --recursive --force kube-apiserver-bootstrap
Expand All @@ -185,10 +196,12 @@ then
cp kube-apiserver-bootstrap/manifests/* manifests/

touch kube-apiserver-bootstrap.done
record_service_stage_success
fi

if [ ! -f kube-controller-manager-bootstrap.done ]
then
record_service_stage_start "kube-controller-manager-bootstrap"
echo "Rendering Kubernetes Controller Manager core manifests..."

rm --recursive --force kube-controller-manager-bootstrap
Expand All @@ -211,10 +224,12 @@ then
cp kube-controller-manager-bootstrap/manifests/* manifests/

touch kube-controller-manager-bootstrap.done
record_service_stage_success
fi

if [ ! -f kube-scheduler-bootstrap.done ]
then
record_service_stage_start "kube-scheduler-bootstrap"
echo "Rendering Kubernetes Scheduler core manifests..."

rm --recursive --force kube-scheduler-bootstrap
Expand All @@ -233,10 +248,12 @@ then
cp kube-scheduler-bootstrap/manifests/* manifests/

touch kube-scheduler-bootstrap.done
record_service_stage_success
fi

if [ ! -f ingress-operator-bootstrap.done ]
then
record_service_stage_start "ingress-operator-bootstrap"
echo "Rendering Ingress Operator core manifests..."

rm --recursive --force ingress-operator-bootstrap
Expand All @@ -251,10 +268,12 @@ then
cp ingress-operator-manifests/* manifests/

touch ingress-operator-bootstrap.done
record_service_stage_success
fi

if [ ! -f mco-bootstrap.done ]
then
record_service_stage_start "mco-bootstrap"
echo "Rendering MCO manifests..."

rm --recursive --force mco-bootstrap
Expand Down Expand Up @@ -323,10 +342,12 @@ then
cp tls/machine-config-server.key /etc/ssl/mcs/tls.key

touch mco-bootstrap.done
record_service_stage_success
fi

if [ ! -f cco-bootstrap.done ]
then
record_service_stage_start "cco-bootstrap"
echo "Rendering CCO manifests..."

rm --recursive --force cco-bootstrap
Expand All @@ -349,13 +370,16 @@ then
fi

touch cco-bootstrap.done
record_service_stage_success
fi

# in case of single node, if we removed etcd, there is no point to wait for it on restart
if [ ! -f stop-etcd.done ]
then
record_service_stage_start "wait-for-etcd"
# Wait for the etcd cluster to come up.
wait_for_etcd_cluster
record_service_stage_success
fi

REQUIRED_PODS="openshift-kube-apiserver/kube-apiserver,openshift-kube-scheduler/openshift-kube-scheduler,openshift-kube-controller-manager/kube-controller-manager,openshift-cluster-version/cluster-version-operator"
Expand All @@ -368,6 +392,7 @@ echo "Starting cluster-bootstrap..."

if [ ! -f cb-bootstrap.done ]
then
record_service_stage_start "cb-bootstrap"
bootkube_podman_run \
--rm \
--volume "$PWD:/assets:z" \
Expand All @@ -376,6 +401,7 @@ then
start --tear-down-early=false --asset-dir=/assets --required-pods="${REQUIRED_PODS}"

touch cb-bootstrap.done
record_service_stage_success
fi

if [ "$BOOTSTRAP_INPLACE" = true ]
Expand All @@ -386,15 +412,18 @@ else

if [ ! -z "$CLUSTER_ETCD_OPERATOR_IMAGE" ]
then
record_service_stage_start "wait-for-ceo"
echo "Waiting for CEO to finish..."
bootkube_podman_run \
--volume "$PWD:/assets:z" \
"${CLUSTER_ETCD_OPERATOR_IMAGE}" \
/usr/bin/cluster-etcd-operator \
wait-for-ceo \
--kubeconfig /assets/auth/kubeconfig
record_service_stage_success
fi
fi

# Workaround for https://github.com/opencontainers/runc/pull/1807
touch /opt/openshift/.bootkube.done
echo "bootkube.service complete"
133 changes: 133 additions & 0 deletions data/data/bootstrap/files/usr/local/bin/bootstrap-service-record.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#!/usr/bin/env bash
# This library provides a helper functions for recording when a service
# and its stages start and end.

# SERVICE_RECORDS_DIR is the directory under which service records will be stored.
SERVICE_RECORDS_DIR="${SERVICE_RECORDS_DIR:-/var/log/openshift/}"
# SYSTEMD_UNIT_NAME is the name of the systemd unit for the service
SYSTEMD_UNIT_NAME="$(ps -o unit= $$)"
# SERVICE_NAME is the name of the service
SERVICE_NAME="${SERVICE_NAME:-${SYSTEMD_UNIT_NAME%.service}}"
# STAGE_NAME is the name of the current stage in the service
STAGE_NAME=""


# add_service_record_entry adds a record entry to the service records file.
# PHASE - phase being recorded; one of "start", "end", "stage start", or "stage end"

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

consider
start -> service start
end -> service end

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds reasonable.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll change the phase for pre- and post-commands to pre-command start, pre-command end, post-command start, and post-command end, too.

# RESULT - result of the action
# MESSAGE - message giving more detail about the result of the action
add_service_record_entry() {
local FILENAME="${SERVICE_RECORDS_DIR}/${SERVICE_NAME}.json"
mkdir --parents $(dirname "${FILENAME}")
# Append the new entry to the existing array in the file.
# If the file does not already exist, start with an empty array.
# The new entry contains only the fields that have non-empty values, since the
# stage, result, error line, and error message are all optional.
([ -f "${FILENAME}" ] && cat "${FILENAME}" || echo '[]') | \
jq \
--arg timestamp "$(date +"%Y-%m-%dT%H:%M:%SZ")" \
--arg preCommand "${PRE_COMMAND-}" \
--arg postCommand "${POST_COMMAND-}" \
--arg stage "${STAGE_NAME-}" \
--arg phase "${PHASE}" \
--arg result "${RESULT-}" \
--arg errorLine "${ERROR_LINE-}" \
--arg errorMessage "${ERROR_MESSAGE-}" \
'. += [
{$timestamp,$preCommand,$postCommand,$stage,$phase,$result,$errorLine,$errorMessage} |
reduce keys[] as $k (.; if .[$k] == "" then del(.[$k]) else . end)
]' \
> "${FILENAME}.tmp" && \
mv "${FILENAME}.tmp" "${FILENAME}"
}

# record_service_start() records the start of a service.
record_service_start() {
local PHASE="start"

add_service_record_entry
}

# record_service_end(result) records the end of a service.
# ERROR_LINE - line where the error occurred, if there was an error
# ERROR_MESSAGE - error message, if there was an error
record_service_end() {
local PHASE="end"
local RESULT=${1:?Must specify a result}

add_service_record_entry
}

# record_service_stage_start(stage_name) records the start of a stage of a service.
record_service_stage_start() {
if [ -n "${STAGE_NAME}" ]
then
echo "attempt to record the start of a stage without ending the previous one"
exit 1
fi

local PHASE="stage start"
STAGE_NAME=${1:?Must specify a stage name}

add_service_record_entry
}

# record_service_stage_end(result) records the end of a stage of a service.
# ERROR_LINE - line where the error occurred, if there was an error
# ERROR_MESSAGE - error message, if there was an error
record_service_stage_end() {
if [ -z "${STAGE_NAME}" ]
then
echo "attempt to record the end of a stage without starting one"
exit 1
fi

local PHASE="stage end"
local RESULT=${1:?Must specify a result}

add_service_record_entry

STAGE_NAME=""
}

# record_service_stage_success records the successful end of a stage of a service.
record_service_stage_success() {
record_service_stage_end "success"
}

record_service_stage_failure() {
local ERROR_LINE
local ERROR_MESSAGE
get_error_info ERROR_LINE ERROR_MESSAGE
record_service_stage_end "failure"
}

record_service_exit() {
if [ "$1" -eq 0 ]
then
local RESULT="success"
else
local RESULT="failure"
local ERROR_LINE
local ERROR_MESSAGE
get_error_info ERROR_LINE ERROR_MESSAGE
fi

if [ -n "${STAGE_NAME}" ]
then
record_service_stage_end "${RESULT}"
fi

record_service_end "${RESULT}"
}

get_error_info() {
local -n error_line=$1
local -n error_message=$2
error_line="$(caller 1)"
error_message="$(journalctl --unit="${SYSTEMD_UNIT_NAME}" --lines=3 --output=cat)"
}

record_service_start

trap 'record_service_exit $?' EXIT
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ set -euo pipefail
# Perhaps down the line we change this to run something like:
# podman run machine-config-daemon bootstrap ... (passing the release image and the host rootfs)

. /usr/local/bin/bootstrap-service-record.sh

. /usr/local/bin/release-image.sh

MACHINE_CONFIG_INFRA_IMAGE=$(image_for pod)
Expand Down
Loading