WIP: deploy workers with the live-iso flow

04_setup_ironic.sh

@@ -144,8 +144,27 @@ if [ ! -f "${CACHED_MACHINE_OS_IMAGE}" ]; then
   curl -g --insecure -L -o "${CACHED_MACHINE_OS_IMAGE}" "${MACHINE_OS_IMAGE_URL}"
   echo "${MACHINE_OS_IMAGE_SHA256} ${CACHED_MACHINE_OS_IMAGE}" | tee ${CACHED_MACHINE_OS_IMAGE}.sha256sum
   sha256sum --strict --check ${CACHED_MACHINE_OS_IMAGE}.sha256sum || ( rm -f "${CACHED_MACHINE_OS_IMAGE}" ; exit 1 )
+fi
 
+if [ ! -z "${TEST_LIVE_ISO:-}" ]; then
+  CACHED_MACHINE_OS_ISO_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_ISO_IMAGE_NAME}"
+  if [ ! -f "${CACHED_MACHINE_OS_ISO_IMAGE}" ]; then
+    curl -g --insecure -L -o "${CACHED_MACHINE_OS_ISO_IMAGE}" "${MACHINE_OS_ISO_IMAGE_URL}"
+    echo "${MACHINE_OS_ISO_IMAGE_SHA256} ${CACHED_MACHINE_OS_ISO_IMAGE}" | tee ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum
+    sha256sum --strict --check ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum || ( rm -f "${CACHED_MACHINE_OS_ISO_IMAGE}" ; exit 1 )
+    # Modify the cached image to enable the openstack ignition datasource
+    # We need this so the live-iso can access ignition user_data via the
+    # configdrive ref https://review.opendev.org/c/openstack/ironic/+/764333
+    # This process is from https://coreos.github.io/coreos-installer/getting-started/#run-from-a-container
+    COREOS_INSTALL="sudo podman run --pull=always --privileged --rm -v /dev:/dev \
+        -v /run/udev:/run/udev -v ${IRONIC_DATA_DIR}/html/images:/data \
+        -w /data quay.io/coreos/coreos-installer:release"
+    ${COREOS_INSTALL} iso kargs modify -r ignition.platform.id=metal=openstack /data/${MACHINE_OS_ISO_IMAGE_NAME}
+    ${COREOS_INSTALL} iso kargs show /data/${MACHINE_OS_ISO_IMAGE_NAME}
+    sha256sum ${CACHED_MACHINE_OS_ISO_IMAGE} > ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum
+  fi
 fi
+
 CACHED_MACHINE_OS_BOOTSTRAP_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_BOOTSTRAP_IMAGE_NAME}"
 if [ ! -f "${CACHED_MACHINE_OS_BOOTSTRAP_IMAGE}" ]; then
   curl -g --insecure -L -o "${CACHED_MACHINE_OS_BOOTSTRAP_IMAGE}" "${MACHINE_OS_BOOTSTRAP_IMAGE_URL}"

05_create_install_config.sh

@@ -21,7 +21,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
         API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}')
         INGRESS_VIP=$(nth_ip $EXTERNAL_SUBNET_V4 4)
     fi
-    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
+    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
     echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
     echo "listen-address=::1" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf
 
@@ -58,5 +58,5 @@ fi
 # Generate the assets for extra worker VMs
 if [ -f "${EXTRA_NODES_FILE}" ]; then
     jq '.nodes' "${EXTRA_NODES_FILE}" | tee "${EXTRA_BAREMETALHOSTS_FILE}"
-    generate_ocp_host_manifest ${OCP_DIR} ${EXTRA_BAREMETALHOSTS_FILE} extra_host_manifests.yaml openshift-machine-api
+    generate_extra_host_manifest ${OCP_DIR} ${EXTRA_BAREMETALHOSTS_FILE} extra_host_manifests.yaml openshift-machine-api
 fi

common.sh

@@ -287,3 +287,7 @@ fi
 
 # Defaults the variable to enable testing a custom machine-api-operator image
 export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false}
+
+# Enable testing with a live-iso image (experimental)
+export TEST_LIVE_ISO=${TEST_LIVE_ISO:-}
+export LIVE_ISO_CONFIG_EMBED=${LIVE_ISO_CONFIG_EMBED:-}

config_example.sh

@@ -214,3 +214,19 @@ set -x
 # certificate store. This trust bundle may also be used when a proxy has 
 # been configured.
 # export ADDITIONAL_TRUST_BUNDLE=/path/to/ca_file
+
+
+##
+## Experimental/unsupported options
+##
+
+# Enable dev-testing with the installer live-iso
+#export TEST_LIVE_ISO=true
+
+# Indicate the extra hosts should be annotated to skip inspection
+#EXTRA_HOST_INSPECT_DISABLED=true
+
+# When using TEST_LIVE_ISO=true embed the ignition config
+# in the iso, instead of the default which is to provide it
+# via the ironic config drive (using the BMH userData field)
+# export LIVE_ISO_CONFIG_EMBED=true

host_cleanup.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -x
+set -xe
 
 source logging.sh
 source common.sh

metal3-dev/local-bmo.sh

@@ -67,7 +67,9 @@ oc apply -f $OUTDIR/bmo-deployment-dev.yaml -n openshift-machine-api
 # Set some variables the operator expects to have in order to work
 export OPERATOR_NAME=baremetal-operator
 
-CLUSTER_IP=$(wrap_if_ipv6 $CLUSTER_PROVISIONING_IP)
+oc wait --for=condition=Ready pod -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state --timeout=90s
+CLUSTER_IRONIC_IP=$(oc get pods -n openshift-machine-api -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state -o jsonpath="{.items[0].status.hostIP}")
+CLUSTER_IP=$(wrap_if_ipv6 ${CLUSTER_IRONIC_IP})
 for var in IRONIC_ENDPOINT IRONIC_INSPECTOR_ENDPOINT DEPLOY_KERNEL_URL DEPLOY_RAMDISK_URL; do
     export "$var"=$(cat $OUTDIR/bmo-deployment-full.yaml | yq -r ".spec.template.spec.containers[] | select(.name == \"metal3-baremetal-operator\").env[] | select(.name == \"${var}\").value" | sed "s/localhost/${CLUSTER_IP}/g")
 done

ocp_cleanup.sh

@@ -50,13 +50,13 @@ for vm in $(sudo virsh list --all --name | grep "^${CLUSTER_NAME}.*bootstrap");
   sudo virsh undefine $vm --remove-all-storage
 done
 
-# For some reason --remove-all-storage doesn't actually remove the storage...
+# For some reason --remove-all-storage doesn't actually remove the storage
+# so we do some extra cleanup of volumes
 if [ -d /var/lib/libvirt/openshift-images ]; then
   sudo rm -fr /var/lib/libvirt/openshift-images/${CLUSTER_NAME}-*
 fi
 
-# The .ign volume isn't deleted via --remove-all-storage
-VOLS="$(sudo virsh vol-list --pool default | awk '{print $1}' | grep "^${CLUSTER_NAME}.*bootstrap")"
+VOLS="$(sudo virsh vol-list --pool default | awk '{print $1}' | grep -e "^${CLUSTER_NAME}.*bootstrap" -e "^configdrive-" -e "boot-*-iso-*")"
 for v in $VOLS; do
   sudo virsh vol-delete $v --pool default
 done

ocp_install_env.sh

@@ -245,7 +245,108 @@ EOF
     cp "${outdir}/install-config.yaml" "${outdir}/install-config.yaml.save"
 }
 
-function generate_ocp_host_manifest() {
+function extra_host_annotations() {
+  name=$1
+  vmname=${name#"${CLUSTER_NAME}-"}
+  vmmac=$(grep ${vmname} /var/lib/libvirt/dnsmasq/ostestbm.hostsfile | cut -d"," -f1)
+  vmip=$(grep ${vmname} /var/lib/libvirt/dnsmasq/ostestbm.hostsfile | cut -d"," -f2)
+  if [ ! -z "${EXTRA_HOST_INSPECT_DISABLED:-}" ]; then
+cat <<EOF
+  annotations:
+    inspect.metal3.io: disabled
+    inspect.metal3.io/hardwaredetails: '{"nics":[{"name":"enp1s0","mac": "$vmmac", "ip":"${vmip}"}],"hostname":"${name}"}'
+EOF
+  fi
+}
+
+function extra_host_image() {
+  name=$1
+  outdir="$2"
+  IMAGE_NAME=${MACHINE_OS_ISO_IMAGE_NAME}
+  if [ ! -z "${TEST_LIVE_ISO:-}" ]; then
+    if [ ! -z "${LIVE_ISO_CONFIG_EMBED:-}" ]; then
+      IMAGE_NAME=${MACHINE_OS_ISO_IMAGE_NAME%.iso}-${name}.iso
+      cp ${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_ISO_IMAGE_NAME} ${IRONIC_DATA_DIR}/html/images/${IMAGE_NAME}
+      sudo podman run --pull=always --privileged --rm -v /dev:/dev -v /run/udev:/run/udev -v ./${outdir}:/data -v ${IRONIC_DATA_DIR}/html/images:/images -w /data quay.io/coreos/coreos-installer:release iso ignition embed -i /data/${name}-userData.json /images/${IMAGE_NAME} -f
+    fi
+cat <<EOF
+  image:
+    url: http://$(wrap_if_ipv6 $MIRROR_IP)/images/${IMAGE_NAME}
+    format: live-iso
+EOF
+  fi
+}
+
+function extra_host_userdata() {
+  name=$1
+  if [ ! -z "${TEST_LIVE_ISO:-}" -a -z "${LIVE_ISO_CONFIG_EMBED:-}" ]; then
+cat <<EOF
+  userData:
+    name: ${name}-userdata-secret
+    namespace: openshift-machine-api
+EOF
+  fi
+}
+
+function extra_host_userdata_secret() {
+  name="$1"
+  outdir="$2"
+  if [ ! -z "${TEST_LIVE_ISO:-}" ]; then
+    EXTRA_HOST_IGN=$(oc get secret worker-user-data -n openshift-machine-api -o json | jq -r .data.userData)
+    set +x
+    EXTRA_HOST_PASSWORD=FIXME
+    EXTRA_HOST_PWHASH=$(echo ${EXTRA_HOST_PASSWORD} | openssl passwd -6 -stdin)
+    set -x
+    EXTRA_HOST_USERDATA=$(cat <<EOF
+{
+  "ignition": { "version": "3.2.0" },
+  "passwd": {
+    "users": [
+      {
+        "name": "core",
+        "passwordHash": "${EXTRA_HOST_PWHASH}",
+        "groups": [ "sudo" ]
+      }
+    ]
+  },
+  "systemd": {
+    "units": [{
+      "name": "coreos_installer.service",
+      "enabled": true,
+      "contents": "[Unit]\nAfter=network.target\nAfter=network-online.target\n[Service]\nType=oneshot\nExecStart=/bin/sh -c 'while ! /usr/bin/coreos-installer install --insecure -i /home/core/config.ign ${ROOT_DISK_NAME}; do sleep 5; done'\nExecStartPost=/bin/sh -c 'efibootmgr -d ${ROOT_DISK_NAME} -p 2 -c -L fcos -l \"\\\EFI\\\BOOT\\\BOOTX64.EFI\"; efibootmgr -n 3; reboot'\n\n[Install]\nWantedBy=multi-user.target"
+}]
+},
+"storage": {
+    "files": [{
+      "filesystem": "root",
+      "path": "/home/core/config.ign",
+      "mode": 644,
+      "contents": { "source": "data:text/plain;base64,${EXTRA_HOST_IGN}" }
+    }]
+  }
+}
+EOF)
+
+# Write the unencoded userData for easier debugging and so it
+# can optionally be embedded into the iso
+echo "${EXTRA_HOST_USERDATA}" > ${outdir}/${name}-userData.json
+
+    if [ -z "${LIVE_ISO_CONFIG_EMBED:-}" ]; then
+      cat <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${name}-userdata-secret
+  namespace: openshift-machine-api
+type: Opaque
+data:
+  userData: $(echo ${EXTRA_HOST_USERDATA} | base64 -w 0)
+EOF
+    fi
+  fi
+}
+
+function generate_extra_host_manifest() {
     local outdir
 
     outdir="$1"
@@ -255,6 +356,7 @@ function generate_ocp_host_manifest() {
 
     mkdir -p "${outdir}"
     rm -f "${outdir}/extra_hosts.yaml"
+    rm -f "${outdir}/${host_output}"
 
     jq --raw-output '.[] | .name + " " + .ports[0].address + " " + .driver_info.username + " " + .driver_info.password + " " + .driver_info.address' $host_input \
        | while read name mac username password address ; do
@@ -273,19 +375,23 @@ type: Opaque
 data:
   username: $encoded_username
   password: $encoded_password
-
+---
+$(extra_host_userdata_secret ${name} ${outdir})
 ---
 apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   name: $name
   namespace: $namespace
+$(extra_host_annotations ${name})
 spec:
   online: ${EXTRA_WORKERS_ONLINE_STATUS}
   bootMACAddress: $mac
   bmc:
     address: $address
     credentialsName: ${name}-bmc-secret
+$(extra_host_image ${name} ${outdir})
+$(extra_host_userdata ${name})
 EOF
 
     done

podman_cleanup.sh

@@ -8,3 +8,4 @@ source validation.sh
 early_cleanup_validation
 
 sudo podman image prune --all
+sudo podman volume prune

rhcos.sh

@@ -23,6 +23,14 @@ export MACHINE_OS_IMAGE_URL=${MACHINE_OS_IMAGE_URL:-${MACHINE_OS_INSTALLER_IMAGE
 export MACHINE_OS_IMAGE_NAME=$(basename ${MACHINE_OS_IMAGE_URL})
 export MACHINE_OS_IMAGE_SHA256=${MACHINE_OS_IMAGE_SHA256:-${MACHINE_OS_INSTALLER_IMAGE_SHA256}}
 
+if [ ! -z "${TEST_LIVE_ISO:-}" ]; then
+  export MACHINE_OS_ISO_INSTALLER_IMAGE_URL=$(jq -r '.baseURI + .images["live-iso"].path' $OCP_DIR/rhcos.json)
+  export MACHINE_OS_ISO_INSTALLER_IMAGE_SHA256=$(jq -r '.images["live-iso"].sha256' $OCP_DIR/rhcos.json)
+  export MACHINE_OS_ISO_IMAGE_URL=${MACHINE_OS_ISO_IMAGE_URL:-${MACHINE_OS_ISO_INSTALLER_IMAGE_URL}}
+  export MACHINE_OS_ISO_IMAGE_NAME=$(basename ${MACHINE_OS_ISO_IMAGE_URL})
+  export MACHINE_OS_ISO_IMAGE_SHA256=${MACHINE_OS_ISO_IMAGE_SHA256:-${MACHINE_OS_ISO_INSTALLER_IMAGE_SHA256}}
+fi
+
 export MACHINE_OS_INSTALLER_BOOTSTRAP_IMAGE_URL=$(jq -r '.baseURI + .images.qemu.path' $OCP_DIR/rhcos.json)
 export MACHINE_OS_INSTALLER_BOOTSTRAP_IMAGE_SHA256=$(jq -r '.images.qemu.sha256' $OCP_DIR/rhcos.json)
 export MACHINE_OS_BOOTSTRAP_IMAGE_URL=${MACHINE_OS_BOOTSTRAP_IMAGE_URL:-${MACHINE_OS_INSTALLER_BOOTSTRAP_IMAGE_URL}}

utils.sh

@@ -249,15 +249,22 @@ function generate_auth_template {
         INSPECTOR_USER=$((oc -n openshift-machine-api  get secret/metal3-ironic-inspector-password -o template --template '{{.data.username}}' || echo "") | base64 -d)
         INSPECTOR_PASSWORD=$((oc -n openshift-machine-api  get secret/metal3-ironic-inspector-password -o template --template '{{.data.password}}' || echo "") | base64 -d)
         INSPECTOR_CREDS="$INSPECTOR_USER:$INSPECTOR_PASSWORD"
+        CLUSTER_IRONIC_IP=$(oc get pods -n openshift-machine-api -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state -o jsonpath="{.items[0].status.hostIP}" || echo "")
 
-        go run metal3-templater.go "http_basic" -ironic-basic-auth="$IRONIC_CREDS" -inspector-basic-auth="$INSPECTOR_CREDS" -template-file=clouds.yaml.template -provisioning-interface="$CLUSTER_PRO_IF" -provisioning-network="$PROVISIONING_NETWORK" -image-url="$MACHINE_OS_IMAGE_URL" -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" -cluster-ip="$CLUSTER_PROVISIONING_IP" > clouds.yaml
+        if [ ! -z "${CLUSTER_IRONIC_IP}" ]; then
+            go run metal3-templater.go "http_basic" -ironic-basic-auth="$IRONIC_CREDS" -inspector-basic-auth="$INSPECTOR_CREDS" -template-file=clouds.yaml.template -provisioning-interface="$CLUSTER_PRO_IF" -provisioning-network="$PROVISIONING_NETWORK" -image-url="$MACHINE_OS_IMAGE_URL" -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" -cluster-ip="$CLUSTER_IRONIC_IP" > clouds.yaml
+        else
+            echo "Unable to read CLUSTER_IRONIC_IP - you may need to run generate_clouds_yaml.sh manually"
+        fi
 
         BOOTSTRAP_VM_IP=$(bootstrap_ip)
         if [ ! -z "${BOOTSTRAP_VM_IP}" ]; then
             if ping -c 1 ${BOOTSTRAP_VM_IP}; then
                 # From 4.7 basic_auth is also enabled on the bootstrap VM
                 # There's a clouds.yaml we can copy in that case
-                ($SSH core@${BOOTSTRAP_VM_IP} sudo cat /opt/metal3/auth/clouds.yaml || echo "") | sed "s/^clouds://" >> clouds.yaml
+                # FIXME: the sed of the URL is a workaround for
+                # https://bugzilla.redhat.com/show_bug.cgi?id=1930240
+                ($SSH core@${BOOTSTRAP_VM_IP} sudo cat /opt/metal3/auth/clouds.yaml || echo "") | sed "s/^clouds://" | sed "s/http:\/\/:/http:\/\/${BOOTSTRAP_VM_IP}:/" >> clouds.yaml
             fi
         fi
     fi

vm_setup_vars.yml

@@ -67,3 +67,6 @@ external_network:
           addr: "127.0.0.1"
 
 networks: "{{ (provisioning_network if lookup('env', 'PROVISIONING_NETWORK_PROFILE') != 'Disabled' else []) + external_network }}"
+
+# Allow overriding boot order with efibootmgr when testing with the live-iso
+sushy_ignore_boot_device: "{{ lookup('env', 'TEST_LIVE_ISO') | default(False, True) }}"