WIP ipv6 testing

01_install_requirements.sh

@@ -4,6 +4,7 @@ set -ex
 source logging.sh
 source common.sh
 source utils.sh
+source ocp_install_env.sh
 
 if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null ; then
     RHEL8="True"

02_configure_host.sh

@@ -86,6 +86,9 @@ if [ "$MANAGE_PRO_BRIDGE" == "y" ]; then
         echo -e "DEVICE=$PRO_IF\nTYPE=Ethernet\nONBOOT=yes\nNM_CONTROLLED=no\nBRIDGE=provisioning" | sudo dd of=/etc/sysconfig/network-scripts/ifcfg-$PRO_IF
         sudo ifdown $PRO_IF || true
         sudo ifup $PRO_IF
+        # Need to ifup the provisioning bridge again because ifdown $PRO_IF
+        # will bring down the bridge as well.
+        sudo ifup provisioning
     fi
 fi
 
@@ -176,7 +179,7 @@ if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then
     # create authfile for local registry
     sudo podman login --authfile ${REGISTRY_CREDS} \
         -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \
-        ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}
+        ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}
 else
     # Create a blank authfile in order to have something valid when we read it in 04_setup_ironic.sh
     echo '{}' | sudo dd of=${REGISTRY_CREDS}

04_setup_ironic.sh

@@ -23,11 +23,9 @@ export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX")
 COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX")
 jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE}
 
-_local_images=
 DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX")
 echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE
 for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
-    _local_images=1
     IMAGE=${!IMAGE_VAR}
 
     sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE
@@ -39,7 +37,7 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
         [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH
         cd $REPOPATH
         export $IMAGE_VAR=${IMAGE##*/}:latest
-        export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR}
+        export $IMAGE_VAR=$LOCAL_REGISTRY_DNS_NAME:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR}
         sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} .
         cd -
         sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR}
@@ -65,23 +63,22 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then
        --insecure=true \
         -a ${COMBINED_AUTH_FILE}  \
         --from ${OPENSHIFT_RELEASE_IMAGE} \
-        --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \
-        --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE}
+        --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest \
+        --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE}
 
     #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected,
     #you must extract the installation program from the mirrored content:
     if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
       oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \
         --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \
-        "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}"
+        "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
 
       mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/
     fi
 
     rm -rf "${EXTRACT_DIR}"
-fi
 
-if [ "${_local_images}" == "1" ]; then
+    # Build a local release image, if no *_LOCAL_IMAGE env variables are set then this is just a copy of the release image
     sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE
     sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
 fi

06_create_cluster.sh

@@ -23,13 +23,22 @@ fi
 
 # NOTE: This is equivalent to the external API DNS record pointing the API to the API VIP
 if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
-    API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then
+        API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    else
+        API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    fi
     INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))")
-    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf
+    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf
     echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf
+    echo "listen-address=::1" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf
     sudo systemctl reload NetworkManager
 else
-    API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then
+        API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    else
+        API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    fi
     INGRESS_VIP=$(dig +noall +answer "test.apps.${CLUSTER_DOMAIN}" | awk '{print $NF}')
 fi
 

common.sh

@@ -36,13 +36,25 @@ if [ -z "${CONFIG:-}" ]; then
 fi
 source $CONFIG
 
+export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org}
+export CLUSTER_NAME=${CLUSTER_NAME:-ostest}
+export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}"
+export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}"
+export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"}
+export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"}
+export CLUSTER_SUBNET=${CLUSTER_SUBNET:-"10.128.0.0/14"}
+export CLUSTER_HOST_PREFIX=${CLUSTER_HOST_PREFIX:-"23"}
+export SERVICE_SUBNET=${SERVICE_SUBNET:-"172.30.0.0/16"}
+export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"}
+export DNS_VIP=${DNS_VIP:-"192.168.111.2"}
+export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"}
+
 # mirror images for installation in restricted network
 export MIRROR_IMAGES=${MIRROR_IMAGES:-}
 
 WORKING_DIR=${WORKING_DIR:-"/opt/dev-scripts"}
 
 # variables for local registry configuration
-export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"}
 export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"}
 export REGISTRY_USER=${REGISTRY_USER:-ocp-user}
 export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass}
@@ -94,7 +106,7 @@ fi
 
 if [ -n "$MIRROR_IMAGES" ]; then
     # We're going to be using a locally modified release image
-    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
+    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
 fi
 
 # Set variables

config_example.sh

@@ -76,7 +76,7 @@ set -x
 # Name of branch in the above repo which contains the custom MAO changes
 #export MAO_BRANCH="mao-fix"
 
-#export LOCAL_REGISTRY_ADDRESS="192.168.111.1"
+#export LOCAL_REGISTRY_DNS_NAME="virthost.ostest.test.metalkube.org"
 #export LOCAL_REGISTRY_PORT="5000"
 
 # configure username for registry

config_ipv6.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Get a valid pull secret (json string) from
+# You can get this secret from https://cloud.openshift.com/clusters/install#pull-secret
+export PULL_SECRET=$(cat ~/pull-secret.json)
+
+# Local checkout with https://github.com/metal3-io/metal3-dev-env/pull/160 applied
+export METAL3_DEV_ENV="$WORKING_DIR/metal3-dev-env"
+export MIRROR_IMAGES=true
+export OPENSHIFT_RELEASE_IMAGE="registry.svc.ci.openshift.org/ipv6/release:4.3.0-0.nightly-2020-01-16-123848-ipv6.5"
+export EXTERNAL_SUBNET="fd2e:6f44:5dd8:c956::/120"
+export DNS_VIP="fd2e:6f44:5dd8:c956:0:0:0:2"
+export NETWORK_TYPE="OVNKubernetes"
+export CLUSTER_SUBNET="fd01::/48"
+export CLUSTER_HOST_PREFIX="64"
+export SERVICE_SUBNET="fd02::/112"
+
+# Modify downloaded rhcos images to work around:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1787620
+#
+# cd $WORKING_DIR/ironic/html/images
+# gunzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz
+# virt-edit -a rhcos-43.81.201912131630.0-qemu.x86_64.qcow2 -m /dev/sda1 -e "s/ip=any/ip=ens3:dhcp6/g" /grub2/grub.cfg 
+# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2
+# gzip rhcos-43.81.201912131630.0-qemu.x86_64.qcow2
+# sha256sum rhcos-43.81.201912131630.0-qemu.x86_64.qcow2.gz
+# update .sha256sum file
+
+export MACHINE_OS_BOOTSTRAP_IMAGE_UNCOMPRESSED_SHA256="21e424f6ebfef68171d38088caeb1365d3b11e3d9f492304a3b7d6704c2b59fa"
+export MACHINE_OS_BOOTSTRAP_IMAGE_SHA256="307ed6f8675fcc5b5f62f4711ebc55b54bdfb9175decff357241f3fb979d564c"
+#export MACHINE_OS_IMAGE_UNCOMPRESSED_SHA256="d41814d65f300222dde0cf3c59a04c37b459cb3f4fb696da02832af212cdc8d3"
+#export MACHINE_OS_IMAGE_SHA256="6c7016ec68d46c07937949644d6bbc74d4f55ebdb83872bce77f47ce2ee6d671"

ocp_cleanup.sh

@@ -35,6 +35,10 @@ for vm in $(sudo virsh list --all --name | grep "^${CLUSTER_NAME}.*bootstrap");
   sudo virsh destroy $vm
   sudo virsh undefine $vm --remove-all-storage
 done
+# The .ign volume isn't deleted via --remove-all-storage
+for v in $(sudo virsh vol-list --pool default | grep "^${CLUSTER_NAME}.*bootstrap" | awk '{print $1}'); do
+  sudo virsh vol-delete $v --pool default
+done
 
 if [ -d assets/generated ]; then
   rm -rf assets/generated

ocp_install_env.sh

@@ -1,14 +1,5 @@
 eval "$(go env)"
 
-export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org}
-export CLUSTER_NAME=${CLUSTER_NAME:-ostest}
-export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}"
-export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}"
-export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"}
-export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"}
-export MIRROR_IP=${MIRROR_IP:-"172.22.0.1"}
-export DNS_VIP=${DNS_VIP:-"192.168.111.2"}
-
 function extract_command() {
     local release_image
     local cmd
@@ -53,6 +44,11 @@ function clone_installer() {
   if [[ ! -d $OPENSHIFT_INSTALL_PATH ]]; then
     sync_repo_and_patch go/src/github.com/openshift/installer https://github.com/openshift/installer.git
   fi
+  pushd $OPENSHIFT_INSTALL_PATH
+  git remote add kni https://github.com/openshift-kni/installer.git || true
+  git fetch -v kni
+  git checkout -b kni/4.3-ipv6-$$ kni/4.3-ipv6
+  popd
 }
 
 function build_installer() {
@@ -85,12 +81,25 @@ function generate_ocp_install_config() {
     fi
 
     mkdir -p "${outdir}"
+
+    # IPv6 network config validation
+    if [[ "${EXTERNAL_SUBNET}" =~ .*:.* ]]; then
+      if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then
+        echo "NETWORK_TYPE must be OVNKubernetes when using IPv6"
+        exit 1
+      fi
+    fi
     cat > "${outdir}/install-config.yaml" << EOF
 apiVersion: v1
 baseDomain: ${BASE_DOMAIN}
 networking:
   networkType: ${NETWORK_TYPE}
   machineCIDR: ${EXTERNAL_SUBNET}
+  clusterNetwork:
+  - cidr: ${CLUSTER_SUBNET}
+    hostPrefix: ${CLUSTER_HOST_PREFIX}
+  serviceNetwork:
+  - ${SERVICE_SUBNET}
 metadata:
   name: ${CLUSTER_NAME}
 compute:

utils.sh

@@ -207,10 +207,10 @@ function image_mirror_config {
             cat << EOF
 imageContentSources:
 - mirrors:
-    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
   source: ${RELEASE}
 - mirrors:
-    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
   source: ${TAGGED}
 additionalTrustBundle: |
 ${INDENTED_CERT}
@@ -235,13 +235,13 @@ function setup_local_registry() {
     sudo chown -R $USER:$USER ${REGISTRY_DIR}
 
     pushd $REGISTRY_DIR/certs
-    SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}"
+    SSL_HOST_NAME="${LOCAL_REGISTRY_DNS_NAME}"
 
-    if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then
+    if ipcalc -c $SSL_HOST_NAME; then
         SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}"
         SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}"
     else
-        SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}"
+        SSL_EXT_8="subjectAltName = DNS:${SSL_HOST_NAME}"
         SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}"
     fi
 
@@ -304,7 +304,7 @@ EOF
     if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then
         sudo podman rm registry -f || true
 
-        sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \
+        sudo podman run -d --name registry --net=host --privileged \
             -v ${REGISTRY_DIR}/data:/var/lib/registry:z \
             -v ${REGISTRY_DIR}/auth:/auth:z \
             -e "REGISTRY_AUTH=htpasswd" \

vm_setup_vars.yml

@@ -22,12 +22,48 @@ flavors:
     vcpu: 4
     extradisks: false
 
-# For OpenShift we create some additional DNS records for the API/DNS VIPs
+
 baremetal_network_cidr: "{{ lookup('env', 'EXTERNAL_SUBNET') | default('192.168.111.0/24', true) }}"
+
+# Describe our virtual networks.  These networks will be attached to
+# the vm nodes in the order in which they are defined with the following caveats:
+#   *  The first bridge network defined will be used for pxe booting
+manage_baremetal: 'y'
+networks:
+  - name: provisioning
+    bridge: provisioning
+    forward_mode: bridge
+  - name: baremetal
+    bridge: baremetal
+    forward_mode: "{% if manage_baremetal == 'y' %}nat{% else %}bridge{% endif %}"
+    address: "{{ baremetal_network_cidr|nthhost(1) }}"
+    netmask: "{{ baremetal_network_cidr|ipaddr('netmask') }}"
+    prefix: "{{ baremetal_network_cidr|ipaddr('prefix') }}"
+    dhcp_range:
+      - "{{ baremetal_network_cidr|nthhost(20) }}"
+      - "{{ baremetal_network_cidr|nthhost(60) }}"
+    nat_port_range:
+      - 1024
+      - 65535
+    domain: "{{ cluster_domain }}"
+    dns:
+      # http://www.thekelleys.org.uk/dnsmasq/docs/dnsmasq-man.html
+      # [fd00::] is replaced with the Unique Local Address
+      #options: 'dhcp-option=option6:dns-server,[fd00::]'
+      hosts: "{{dns_extrahosts | default([])}}"
+      forwarders:
+        - domain: "apps.{{ cluster_domain }}"
+          addr: "{% if baremetal_network_cidr|ipv6 != False %}::1{% else %}127.0.0.1{% endif %}"
+
+
+# For OpenShift we create some additional DNS records for the API/DNS VIPs
 dns_extrahosts:
   - ip: "{{ baremetal_network_cidr | nthhost(5) }}"
     hostnames:
       - "api"
   - ip: "{{ baremetal_network_cidr | nthhost(2) }}"
     hostnames:
       - "ns1"
+  - ip: "{{ baremetal_network_cidr | nthhost(1) }}"
+    hostnames:
+      - "virthost"