diff --git a/01_install_requirements.sh b/01_install_requirements.sh index 5bc157d51..1adb9adc3 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -4,6 +4,7 @@ set -ex source logging.sh source common.sh source utils.sh +source ocp_install_env.sh if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null ; then RHEL8="True" diff --git a/02_configure_host.sh b/02_configure_host.sh index 83f229b1d..7451bbc04 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -86,6 +86,9 @@ if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then echo -e "DEVICE=$PRO_IF\nTYPE=Ethernet\nONBOOT=yes\nNM_CONTROLLED=no\nBRIDGE=provisioning" | sudo dd of=/etc/sysconfig/network-scripts/ifcfg-$PRO_IF sudo ifdown $PRO_IF || true sudo ifup $PRO_IF + # Need to ifup the provisioning bridge again because ifdown $PRO_IF + # will bring down the bridge as well. + sudo ifup provisioning fi fi @@ -176,7 +179,7 @@ if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then # create authfile for local registry sudo podman login --authfile ${REGISTRY_CREDS} \ -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \ - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT} + ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT} else # Create a blank authfile in order to have something valid when we read it in 04_setup_ironic.sh echo '{}' | sudo dd of=${REGISTRY_CREDS} diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index b78edc23c..227d29093 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -23,11 +23,9 @@ export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX") COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX") jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE} -_local_images= DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX") echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do - _local_images=1 IMAGE=${!IMAGE_VAR} sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE @@ -39,7 +37,7 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH cd $REPOPATH export $IMAGE_VAR=${IMAGE##*/}:latest - export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} + export $IMAGE_VAR=$LOCAL_REGISTRY_DNS_NAME:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} . cd - sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR} @@ -65,23 +63,22 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then --insecure=true \ -a ${COMBINED_AUTH_FILE} \ --from ${OPENSHIFT_RELEASE_IMAGE} \ - --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ - --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} + --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest \ + --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, #you must extract the installation program from the mirrored content: if [ -z "$KNI_INSTALL_FROM_GIT" ]; then oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ - "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ fi rm -rf "${EXTRACT_DIR}" -fi -if [ "${_local_images}" == "1" ]; then + # Build a local release image, if no *_LOCAL_IMAGE env variables are set then this is just a copy of the release image sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE fi diff --git a/06_create_cluster.sh b/06_create_cluster.sh index b90f8df74..2d334084c 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -23,13 +23,22 @@ fi # NOTE: This is equivalent to the external API DNS record pointing the API to the API VIP if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + fi INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))") - echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf + echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf + echo "listen-address=::1" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf sudo systemctl reload NetworkManager else - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + fi INGRESS_VIP=$(dig +noall +answer "test.apps.${CLUSTER_DOMAIN}" | awk '{print $NF}') fi diff --git a/common.sh b/common.sh index f9468396b..a31c2e9bc 100644 --- a/common.sh +++ b/common.sh @@ -36,13 +36,25 @@ if [ -z "${CONFIG:-}" ]; then fi source $CONFIG +export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} +export CLUSTER_NAME=${CLUSTER_NAME:-ostest} +export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" +export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" +export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} +export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} +export CLUSTER_SUBNET=${CLUSTER_SUBNET:-"10.128.0.0/14"} +export CLUSTER_HOST_PREFIX=${CLUSTER_HOST_PREFIX:-"23"} +export SERVICE_SUBNET=${SERVICE_SUBNET:-"172.30.0.0/16"} +export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} +export DNS_VIP=${DNS_VIP:-"192.168.111.2"} +export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} + # mirror images for installation in restricted network export MIRROR_IMAGES=${MIRROR_IMAGES:-} WORKING_DIR=${WORKING_DIR:-"/opt/dev-scripts"} # variables for local registry configuration -export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"} export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"} export REGISTRY_USER=${REGISTRY_USER:-ocp-user} export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass} @@ -94,7 +106,7 @@ fi if [ -n "$MIRROR_IMAGES" ]; then # We're going to be using a locally modified release image - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" fi # Set variables diff --git a/config_example.sh b/config_example.sh index 5fcf22b54..5cb844ab6 100644 --- a/config_example.sh +++ b/config_example.sh @@ -76,7 +76,7 @@ set -x # Name of branch in the above repo which contains the custom MAO changes #export MAO_BRANCH="mao-fix" -#export LOCAL_REGISTRY_ADDRESS="192.168.111.1" +#export LOCAL_REGISTRY_DNS_NAME="virthost.ostest.test.metalkube.org" #export LOCAL_REGISTRY_PORT="5000" # configure username for registry diff --git a/config_ipv6.sh b/config_ipv6.sh new file mode 100644 index 000000000..f72a5b2bd --- /dev/null +++ b/config_ipv6.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Get a valid pull secret (json string) from +# You can get this secret from https://cloud.openshift.com/clusters/install#pull-secret +export PULL_SECRET=$(cat ~/pull-secret.json) + +# Local checkout with https://github.com/metal3-io/metal3-dev-env/pull/160 applied +export METAL3_DEV_ENV="$WORKING_DIR/metal3-dev-env" +export MIRROR_IMAGES=true +export OPENSHIFT_RELEASE_IMAGE="registry.svc.ci.openshift.org/ipv6/release:4.3.0-0.nightly-2020-01-16-123848-ipv6.5" +export EXTERNAL_SUBNET="fd2e:6f44:5dd8:c956::/120" +export DNS_VIP="fd2e:6f44:5dd8:c956:0:0:0:2" +export NETWORK_TYPE="OVNKubernetes" +export CLUSTER_SUBNET="fd01::/48" +export CLUSTER_HOST_PREFIX="64" +export SERVICE_SUBNET="fd02::/112" + +# Modify downloaded rhcos images to work around: +# https://bugzilla.redhat.com/show_bug.cgi?id=1787620 +# +# cd $WORKING_DIR/ironic/html/images +# gunzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz +# virt-edit -a rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 -m /dev/sda1 -e "s/ip=any/ip=ens3:dhcp6/g" /grub2/grub.cfg +# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 +# gzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 +# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz +# update .sha256sum file + +export MACHINE_OS_BOOTSTRAP_IMAGE_UNCOMPRESSED_SHA256="21e424f6ebfef68171d38088caeb1365d3b11e3d9f492304a3b7d6704c2b59fa" +export MACHINE_OS_BOOTSTRAP_IMAGE_SHA256="307ed6f8675fcc5b5f62f4711ebc55b54bdfb9175decff357241f3fb979d564c" +#export MACHINE_OS_IMAGE_UNCOMPRESSED_SHA256="d41814d65f300222dde0cf3c59a04c37b459cb3f4fb696da02832af212cdc8d3" +#export MACHINE_OS_IMAGE_SHA256="6c7016ec68d46c07937949644d6bbc74d4f55ebdb83872bce77f47ce2ee6d671" diff --git a/ocp_cleanup.sh b/ocp_cleanup.sh index 7fa10abb3..16d1bfa53 100755 --- a/ocp_cleanup.sh +++ b/ocp_cleanup.sh @@ -35,6 +35,10 @@ for vm in $(sudo virsh list --all --name | grep "^${CLUSTER_NAME}.*bootstrap"); sudo virsh destroy $vm sudo virsh undefine $vm --remove-all-storage done +# The .ign volume isn't deleted via --remove-all-storage +for v in $(sudo virsh vol-list --pool default | grep "^${CLUSTER_NAME}.*bootstrap" | awk '{print $1}'); do + sudo virsh vol-delete $v --pool default +done if [ -d assets/generated ]; then rm -rf assets/generated diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 517cfef5a..12f4c7926 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -1,14 +1,5 @@ eval "$(go env)" -export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} -export CLUSTER_NAME=${CLUSTER_NAME:-ostest} -export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" -export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" -export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} -export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} -export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"} -export DNS_VIP=${DNS_VIP:-"192.168.111.2"} - function extract_command() { local release_image local cmd @@ -53,6 +44,11 @@ function clone_installer() { if [[ ! -d $OPENSHIFT_INSTALL_PATH ]]; then sync_repo_and_patch go/src/github.com/openshift/installer https://github.com/openshift/installer.git fi + pushd $OPENSHIFT_INSTALL_PATH + git remote add kni https://github.com/openshift-kni/installer.git || true + git fetch -v kni + git checkout -b kni/4.3-ipv6-$$ kni/4.3-ipv6 + popd } function build_installer() { @@ -85,12 +81,25 @@ function generate_ocp_install_config() { fi mkdir -p "${outdir}" + + # IPv6 network config validation + if [[ "${EXTERNAL_SUBNET}" =~ .*:.* ]]; then + if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then + echo "NETWORK_TYPE must be OVNKubernetes when using IPv6" + exit 1 + fi + fi cat > "${outdir}/install-config.yaml" << EOF apiVersion: v1 baseDomain: ${BASE_DOMAIN} networking: networkType: ${NETWORK_TYPE} machineCIDR: ${EXTERNAL_SUBNET} + clusterNetwork: + - cidr: ${CLUSTER_SUBNET} + hostPrefix: ${CLUSTER_HOST_PREFIX} + serviceNetwork: + - ${SERVICE_SUBNET} metadata: name: ${CLUSTER_NAME} compute: diff --git a/utils.sh b/utils.sh index 76f77de70..50a8cc6c8 100644 --- a/utils.sh +++ b/utils.sh @@ -207,10 +207,10 @@ function image_mirror_config { cat << EOF imageContentSources: - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${RELEASE} - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${TAGGED} additionalTrustBundle: | ${INDENTED_CERT} @@ -235,13 +235,13 @@ function setup_local_registry() { sudo chown -R $USER:$USER ${REGISTRY_DIR} pushd $REGISTRY_DIR/certs - SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" + SSL_HOST_NAME="${LOCAL_REGISTRY_DNS_NAME}" - if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then + if ipcalc -c $SSL_HOST_NAME; then SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" else - SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}" + SSL_EXT_8="subjectAltName = DNS:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}" fi @@ -304,7 +304,7 @@ EOF if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then sudo podman rm registry -f || true - sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \ + sudo podman run -d --name registry --net=host --privileged \ -v ${REGISTRY_DIR}/data:/var/lib/registry:z \ -v ${REGISTRY_DIR}/auth:/auth:z \ -e "REGISTRY_AUTH=htpasswd" \ diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index 7f0062ada..46923dd80 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -22,8 +22,41 @@ flavors: vcpu: 4 extradisks: false -# For OpenShift we create some additional DNS records for the API/DNS VIPs + baremetal_network_cidr: "{{ lookup('env', 'EXTERNAL_SUBNET') | default('192.168.111.0/24', true) }}" + +# Describe our virtual networks. These networks will be attached to +# the vm nodes in the order in which they are defined with the following caveats: +# * The first bridge network defined will be used for pxe booting +manage_baremetal: 'y' +networks: + - name: provisioning + bridge: provisioning + forward_mode: bridge + - name: baremetal + bridge: baremetal + forward_mode: "{% if manage_baremetal == 'y' %}nat{% else %}bridge{% endif %}" + address: "{{ baremetal_network_cidr|nthhost(1) }}" + netmask: "{{ baremetal_network_cidr|ipaddr('netmask') }}" + prefix: "{{ baremetal_network_cidr|ipaddr('prefix') }}" + dhcp_range: + - "{{ baremetal_network_cidr|nthhost(20) }}" + - "{{ baremetal_network_cidr|nthhost(60) }}" + nat_port_range: + - 1024 + - 65535 + domain: "{{ cluster_domain }}" + dns: + # http://www.thekelleys.org.uk/dnsmasq/docs/dnsmasq-man.html + # [fd00::] is replaced with the Unique Local Address + #options: 'dhcp-option=option6:dns-server,[fd00::]' + hosts: "{{dns_extrahosts | default([])}}" + forwarders: + - domain: "apps.{{ cluster_domain }}" + addr: "{% if baremetal_network_cidr|ipv6 != False %}::1{% else %}127.0.0.1{% endif %}" + + +# For OpenShift we create some additional DNS records for the API/DNS VIPs dns_extrahosts: - ip: "{{ baremetal_network_cidr | nthhost(5) }}" hostnames: @@ -31,3 +64,6 @@ dns_extrahosts: - ip: "{{ baremetal_network_cidr | nthhost(2) }}" hostnames: - "ns1" + - ip: "{{ baremetal_network_cidr | nthhost(1) }}" + hostnames: + - "virthost"