diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index 10d42ab1e..813ab5468 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -144,8 +144,27 @@ if [ ! -f "${CACHED_MACHINE_OS_IMAGE}" ]; then curl -g --insecure -L -o "${CACHED_MACHINE_OS_IMAGE}" "${MACHINE_OS_IMAGE_URL}" echo "${MACHINE_OS_IMAGE_SHA256} ${CACHED_MACHINE_OS_IMAGE}" | tee ${CACHED_MACHINE_OS_IMAGE}.sha256sum sha256sum --strict --check ${CACHED_MACHINE_OS_IMAGE}.sha256sum || ( rm -f "${CACHED_MACHINE_OS_IMAGE}" ; exit 1 ) +fi +if [ ! -z "${TEST_LIVE_ISO:-}" ]; then + CACHED_MACHINE_OS_ISO_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_ISO_IMAGE_NAME}" + if [ ! -f "${CACHED_MACHINE_OS_ISO_IMAGE}" ]; then + curl -g --insecure -L -o "${CACHED_MACHINE_OS_ISO_IMAGE}" "${MACHINE_OS_ISO_IMAGE_URL}" + echo "${MACHINE_OS_ISO_IMAGE_SHA256} ${CACHED_MACHINE_OS_ISO_IMAGE}" | tee ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum + sha256sum --strict --check ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum || ( rm -f "${CACHED_MACHINE_OS_ISO_IMAGE}" ; exit 1 ) + # Modify the cached image to enable the openstack ignition datasource + # We need this so the live-iso can access ignition user_data via the + # configdrive ref https://review.opendev.org/c/openstack/ironic/+/764333 + # This process is from https://coreos.github.io/coreos-installer/getting-started/#run-from-a-container + COREOS_INSTALL="sudo podman run --pull=always --privileged --rm -v /dev:/dev \ + -v /run/udev:/run/udev -v ${IRONIC_DATA_DIR}/html/images:/data \ + -w /data quay.io/coreos/coreos-installer:release" + ${COREOS_INSTALL} iso kargs modify -r ignition.platform.id=metal=openstack /data/${MACHINE_OS_ISO_IMAGE_NAME} + ${COREOS_INSTALL} iso kargs show /data/${MACHINE_OS_ISO_IMAGE_NAME} + sha256sum ${CACHED_MACHINE_OS_ISO_IMAGE} > ${CACHED_MACHINE_OS_ISO_IMAGE}.sha256sum + fi fi + CACHED_MACHINE_OS_BOOTSTRAP_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_BOOTSTRAP_IMAGE_NAME}" if [ ! -f "${CACHED_MACHINE_OS_BOOTSTRAP_IMAGE}" ]; then curl -g --insecure -L -o "${CACHED_MACHINE_OS_BOOTSTRAP_IMAGE}" "${MACHINE_OS_BOOTSTRAP_IMAGE_URL}" diff --git a/05_create_install_config.sh b/05_create_install_config.sh index 46e479805..f47050266 100755 --- a/05_create_install_config.sh +++ b/05_create_install_config.sh @@ -21,7 +21,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip ${BAREMETAL_NETWORK_NAME}) | awk '{print $NF}') INGRESS_VIP=$(nth_ip $EXTERNAL_SUBNET_V4 4) fi - echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf + echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf echo "listen-address=::1" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf @@ -58,5 +58,5 @@ fi # Generate the assets for extra worker VMs if [ -f "${EXTRA_NODES_FILE}" ]; then jq '.nodes' "${EXTRA_NODES_FILE}" | tee "${EXTRA_BAREMETALHOSTS_FILE}" - generate_ocp_host_manifest ${OCP_DIR} ${EXTRA_BAREMETALHOSTS_FILE} extra_host_manifests.yaml openshift-machine-api + generate_extra_host_manifest ${OCP_DIR} ${EXTRA_BAREMETALHOSTS_FILE} extra_host_manifests.yaml openshift-machine-api fi diff --git a/common.sh b/common.sh index 4b699e752..e496d91bb 100644 --- a/common.sh +++ b/common.sh @@ -287,3 +287,7 @@ fi # Defaults the variable to enable testing a custom machine-api-operator image export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false} + +# Enable testing with a live-iso image (experimental) +export TEST_LIVE_ISO=${TEST_LIVE_ISO:-} +export LIVE_ISO_CONFIG_EMBED=${LIVE_ISO_CONFIG_EMBED:-} diff --git a/config_example.sh b/config_example.sh index 5559b3632..b461249ee 100755 --- a/config_example.sh +++ b/config_example.sh @@ -214,3 +214,19 @@ set -x # certificate store. This trust bundle may also be used when a proxy has # been configured. # export ADDITIONAL_TRUST_BUNDLE=/path/to/ca_file + + +## +## Experimental/unsupported options +## + +# Enable dev-testing with the installer live-iso +#export TEST_LIVE_ISO=true + +# Indicate the extra hosts should be annotated to skip inspection +#EXTRA_HOST_INSPECT_DISABLED=true + +# When using TEST_LIVE_ISO=true embed the ignition config +# in the iso, instead of the default which is to provide it +# via the ironic config drive (using the BMH userData field) +# export LIVE_ISO_CONFIG_EMBED=true diff --git a/host_cleanup.sh b/host_cleanup.sh index 99c4e51ec..b80583e8a 100755 --- a/host_cleanup.sh +++ b/host_cleanup.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -x +set -xe source logging.sh source common.sh diff --git a/metal3-dev/local-bmo.sh b/metal3-dev/local-bmo.sh index 9f8ac9425..b7f7a6e4f 100755 --- a/metal3-dev/local-bmo.sh +++ b/metal3-dev/local-bmo.sh @@ -67,7 +67,9 @@ oc apply -f $OUTDIR/bmo-deployment-dev.yaml -n openshift-machine-api # Set some variables the operator expects to have in order to work export OPERATOR_NAME=baremetal-operator -CLUSTER_IP=$(wrap_if_ipv6 $CLUSTER_PROVISIONING_IP) +oc wait --for=condition=Ready pod -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state --timeout=90s +CLUSTER_IRONIC_IP=$(oc get pods -n openshift-machine-api -l baremetal.openshift.io/cluster-baremetal-operator=metal3-state -o jsonpath="{.items[0].status.hostIP}") +CLUSTER_IP=$(wrap_if_ipv6 ${CLUSTER_IRONIC_IP}) for var in IRONIC_ENDPOINT IRONIC_INSPECTOR_ENDPOINT DEPLOY_KERNEL_URL DEPLOY_RAMDISK_URL; do export "$var"=$(cat $OUTDIR/bmo-deployment-full.yaml | yq -r ".spec.template.spec.containers[] | select(.name == \"metal3-baremetal-operator\").env[] | select(.name == \"${var}\").value" | sed "s/localhost/${CLUSTER_IP}/g") done diff --git a/ocp_cleanup.sh b/ocp_cleanup.sh index 5914e6fc2..dd172a176 100755 --- a/ocp_cleanup.sh +++ b/ocp_cleanup.sh @@ -50,13 +50,13 @@ for vm in $(sudo virsh list --all --name | grep "^${CLUSTER_NAME}.*bootstrap"); sudo virsh undefine $vm --remove-all-storage done -# For some reason --remove-all-storage doesn't actually remove the storage... +# For some reason --remove-all-storage doesn't actually remove the storage +# so we do some extra cleanup of volumes if [ -d /var/lib/libvirt/openshift-images ]; then sudo rm -fr /var/lib/libvirt/openshift-images/${CLUSTER_NAME}-* fi -# The .ign volume isn't deleted via --remove-all-storage -VOLS="$(sudo virsh vol-list --pool default | awk '{print $1}' | grep "^${CLUSTER_NAME}.*bootstrap")" +VOLS="$(sudo virsh vol-list --pool default | awk '{print $1}' | grep -e "^${CLUSTER_NAME}.*bootstrap" -e "^configdrive-" -e "boot-*-iso-*")" for v in $VOLS; do sudo virsh vol-delete $v --pool default done diff --git a/ocp_install_env.sh b/ocp_install_env.sh index ccd674e9c..e395babf5 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -245,7 +245,108 @@ EOF cp "${outdir}/install-config.yaml" "${outdir}/install-config.yaml.save" } -function generate_ocp_host_manifest() { +function extra_host_annotations() { + name=$1 + vmname=${name#"${CLUSTER_NAME}-"} + vmmac=$(grep ${vmname} /var/lib/libvirt/dnsmasq/ostestbm.hostsfile | cut -d"," -f1) + vmip=$(grep ${vmname} /var/lib/libvirt/dnsmasq/ostestbm.hostsfile | cut -d"," -f2) + if [ ! -z "${EXTRA_HOST_INSPECT_DISABLED:-}" ]; then +cat < ${outdir}/${name}-userData.json + + if [ -z "${LIVE_ISO_CONFIG_EMBED:-}" ]; then + cat < clouds.yaml + if [ ! -z "${CLUSTER_IRONIC_IP}" ]; then + go run metal3-templater.go "http_basic" -ironic-basic-auth="$IRONIC_CREDS" -inspector-basic-auth="$INSPECTOR_CREDS" -template-file=clouds.yaml.template -provisioning-interface="$CLUSTER_PRO_IF" -provisioning-network="$PROVISIONING_NETWORK" -image-url="$MACHINE_OS_IMAGE_URL" -bootstrap-ip="$BOOTSTRAP_PROVISIONING_IP" -cluster-ip="$CLUSTER_IRONIC_IP" > clouds.yaml + else + echo "Unable to read CLUSTER_IRONIC_IP - you may need to run generate_clouds_yaml.sh manually" + fi BOOTSTRAP_VM_IP=$(bootstrap_ip) if [ ! -z "${BOOTSTRAP_VM_IP}" ]; then if ping -c 1 ${BOOTSTRAP_VM_IP}; then # From 4.7 basic_auth is also enabled on the bootstrap VM # There's a clouds.yaml we can copy in that case - ($SSH core@${BOOTSTRAP_VM_IP} sudo cat /opt/metal3/auth/clouds.yaml || echo "") | sed "s/^clouds://" >> clouds.yaml + # FIXME: the sed of the URL is a workaround for + # https://bugzilla.redhat.com/show_bug.cgi?id=1930240 + ($SSH core@${BOOTSTRAP_VM_IP} sudo cat /opt/metal3/auth/clouds.yaml || echo "") | sed "s/^clouds://" | sed "s/http:\/\/:/http:\/\/${BOOTSTRAP_VM_IP}:/" >> clouds.yaml fi fi fi diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index 57a1d0241..8f68277a2 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -67,3 +67,6 @@ external_network: addr: "127.0.0.1" networks: "{{ (provisioning_network if lookup('env', 'PROVISIONING_NETWORK_PROFILE') != 'Disabled' else []) + external_network }}" + +# Allow overriding boot order with efibootmgr when testing with the live-iso +sushy_ignore_boot_device: "{{ lookup('env', 'TEST_LIVE_ISO') | default(False, True) }}"