kic entrypoint: retryable update-alternatives, move earlier into execution

.github/workflows/kic_image.yml

@@ -2,7 +2,7 @@ name: KIC_IMAGE
 on:
   pull_request:
     paths:
-      - "hack/images/**"
+      - "deploy/kicbase/**"
 env:
   GOPROXY: https://proxy.golang.org
 jobs:

deploy/kicbase/Dockerfile

@@ -31,6 +31,7 @@ RUN sh -c "echo 'deb https://dl.bintray.com/afbjorklund/podman eoan main' > /etc
 # install varlink
 RUN apt-get install -y --no-install-recommends varlink
 
+COPY entrypoint /usr/local/bin/entrypoint
 # automount service
 COPY automount/minikube-automount /usr/sbin/minikube-automount
 COPY automount/minikube-automount.service /usr/lib/systemd/system/minikube-automount.service
@@ -47,8 +48,6 @@ RUN mkdir /var/run/sshd
 RUN echo 'root:root' |chpasswd
 RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
 RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
-# Add set -x to entrypoint file
-RUN sed -i "20i set -x" /usr/local/bin/entrypoint
 
 EXPOSE 22
 # create docker user for minikube ssh. to match VM using "docker" as username

deploy/kicbase/entrypoint

@@ -17,30 +17,50 @@
 set -o errexit
 set -o nounset
 set -o pipefail
+set -x
+
+update-alternatives() {
+    echo "retryable update-alternatives: $*"
+    local args=$*
+
+    for i in $(seq 0 15); do
+      /usr/bin/update-alternatives $args && return || echo "update-alternatives $args failed (retry $i)"
+
+      echo "update-alternatives diagnostics information below:"
+      mount
+      df -h /var
+      find /var/lib/dpkg
+      dmesg | tail
+
+      sleep 1
+    done
+
+    exit 30
+}
 
 fix_mount() {
-  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap' 
+  echo 'INFO: ensuring we can execute mount/umount even with userns-remap' 
   # necessary only when userns-remap is enabled on the host, but harmless
   # The binary /bin/mount should be owned by root and have the setuid bit
-  chown root:root /bin/mount
-  chmod -s /bin/mount
+  chown root:root "$(which mount)" "$(which umount)"
+  chmod -s "$(which mount)" "$(which umount)"
 
   # This is a workaround to an AUFS bug that might cause `Text file
   # busy` on `mount` command below. See more details in
   # https://github.com/moby/moby/issues/9547
   if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
-    echo 'INFO: detected aufs, calling sync'
+    echo 'INFO: detected aufs, calling sync' >&2
     sync
   fi
 
   echo 'INFO: remounting /sys read-only'
   # systemd-in-a-container should have read only /sys
-  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+  # https://systemd.io/CONTAINER_INTERFACE/
   # however, we need other things from `docker run --privileged` ...
   # and this flag also happens to make /sys rw, amongst other things
   mount -o remount,ro /sys
 
-  echo 'INFO: making mounts shared'
+  echo 'INFO: making mounts shared' >&2
   # for mount propagation
   mount --make-rshared /
 }
@@ -70,13 +90,25 @@ fix_cgroup() {
       mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
     done
   fi
+  local podman_cgroup_mounts
+  podman_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep libpod_parent || true)
+  if [[ -n "${podman_cgroup_mounts}" ]]; then
+    local podman_cgroup cgroup_subsystems subsystem
+    podman_cgroup=$(echo "${podman_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
+    cgroup_subsystems=$(echo "${podman_cgroup_mounts}" | cut -d' ' -f 5)
+    echo "${cgroup_subsystems}" |
+    while IFS= read -r subsystem; do
+      mkdir -p "${subsystem}${podman_cgroup}"
+      mount --bind "${subsystem}" "${subsystem}${podman_cgroup}"
+    done
+  fi
 }
 
 fix_machine_id() {
   # Deletes the machine-id embedded in the node image and generates a new one.
   # This is necessary because both kubelet and other components like weave net
   # use machine-id internally to distinguish nodes.
-  echo 'INFO: clearing and regenerating /etc/machine-id'
+  echo 'INFO: clearing and regenerating /etc/machine-id' >&2
   rm -f /etc/machine-id
   systemd-machine-id-setup
 }
@@ -85,7 +117,7 @@ fix_product_name() {
   # this is a small fix to hide the underlying hardware and fix issue #426
   # https://github.com/kubernetes-sigs/kind/issues/426
   if [[ -f /sys/class/dmi/id/product_name ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"'
+    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"' >&2
     echo 'kind' > /kind/product_name
     mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
   fi
@@ -101,11 +133,11 @@ fix_product_uuid() {
   # https://github.com/kubernetes-sigs/kind/issues/1027
   [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
   if [[ -f /sys/class/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random'
+    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random' >&2
     mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
   fi
   if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well'
+    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well' >&2
     mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
   fi
 }
@@ -149,7 +181,7 @@ select_iptables() {
     fi
   fi
 
-  echo "INFO: setting iptables to detected mode: ${mode}"
+  echo "INFO: setting iptables to detected mode: ${mode}" >&2
   update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
   update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
 }
@@ -179,17 +211,56 @@ enable_network_magic(){
   # now we can ensure that DNS is configured to use our IP
   cp /etc/resolv.conf /etc/resolv.conf.original
   sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
+
+  # fixup IPs in manifests ...
+  curr_ipv4="$( (getent ahostsv4 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
+  echo "INFO: Detected IPv4 address: ${curr_ipv4}" >&2
+  if [ -f /kind/old-ipv4 ]; then
+      old_ipv4=$(cat /kind/old-ipv4)
+      echo "INFO: Detected old IPv4 address: ${old_ipv4}" >&2
+      # sanity check that we have a current address
+      if [[ -z $curr_ipv4 ]]; then
+        echo "ERROR: Have an old IPv4 address but no current IPv4 address (!)" >&2
+        exit 1
+      fi
+      # kubernetes manifests are only present on control-plane nodes
+      sed -i "s#${old_ipv4}#${curr_ipv4}#" /etc/kubernetes/manifests/*.yaml || true
+      # this is no longer required with autodiscovery
+      sed -i "s#${old_ipv4}#${curr_ipv4}#" /var/lib/kubelet/kubeadm-flags.env || true
+  fi
+  if [[ -n $curr_ipv4 ]]; then
+    echo -n "${curr_ipv4}" >/kind/old-ipv4
+  fi
+
+  # do IPv6
+  curr_ipv6="$( (getent ahostsv6 $(hostname) | head -n1 | cut -d' ' -f1) || true)"
+  echo "INFO: Detected IPv6 address: ${curr_ipv6}" >&2
+  if [ -f /kind/old-ipv6 ]; then
+      old_ipv6=$(cat /kind/old-ipv6)
+      echo "INFO: Detected old IPv6 address: ${old_ipv6}" >&2
+      # sanity check that we have a current address
+      if [[ -z $curr_ipv6 ]]; then
+        echo "ERROR: Have an old IPv6 address but no current IPv6 address (!)" >&2
+      fi
+      # kubernetes manifests are only present on control-plane nodes
+      sed -i "s#${old_ipv6}#${curr_ipv6}#" /etc/kubernetes/manifests/*.yaml || true
+      # this is no longer required with autodiscovery
+      sed -i "s#${old_ipv6}#${curr_ipv6}#" /var/lib/kubelet/kubeadm-flags.env || true
+  fi
+  if [[ -n $curr_ipv6 ]]; then
+    echo -n "${curr_ipv6}" >/kind/old-ipv6
+  fi
 }
 
 # run pre-init fixups
+select_iptables
 fix_kmsg
 fix_mount
 fix_cgroup
 fix_machine_id
 fix_product_name
 fix_product_uuid
 configure_proxy
-select_iptables
 enable_network_magic
 
 # we want the command (expected to be systemd) to be PID1, so exec to it