Changes to enable testing with an IPv6 controlplane

01_install_requirements.sh

@@ -4,6 +4,7 @@ set -ex
 source logging.sh
 source common.sh
 source utils.sh
+source ocp_install_env.sh
 
 if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null ; then
     RHEL8="True"

02_configure_host.sh

@@ -173,14 +173,18 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
   fi
 fi
 
+# Add a /etc/hosts entry for $LOCAL_REGISTRY_DNS_NAME
+sudo sed -i "/${LOCAL_REGISTRY_DNS_NAME}/d" /etc/hosts
+echo "${PROVISIONING_HOST_EXTERNAL_IP} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts
+
 # Remove any previous file, or podman login panics when reading the
 # blank authfile with a "assignment to entry in nil map" error
 rm -f ${REGISTRY_CREDS}
 if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then
     # create authfile for local registry
     sudo podman login --authfile ${REGISTRY_CREDS} \
         -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \
-        ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}
+        ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}
 else
     # Create a blank authfile in order to have something valid when we read it in 04_setup_ironic.sh
     echo '{}' | sudo dd of=${REGISTRY_CREDS}

04_setup_ironic.sh

@@ -37,7 +37,7 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
         [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH
         cd $REPOPATH
         export $IMAGE_VAR=${IMAGE##*/}:latest
-        export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR}
+        export $IMAGE_VAR=$LOCAL_REGISTRY_DNS_NAME:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR}
         sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} .
         cd -
         sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR}
@@ -63,15 +63,15 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then
        --insecure=true \
         -a ${COMBINED_AUTH_FILE}  \
         --from ${OPENSHIFT_RELEASE_IMAGE} \
-        --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \
-        --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE}
+        --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \
+        --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE}
 
     #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected,
     #you must extract the installation program from the mirrored content:
     if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
       oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \
         --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \
-        "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}"
+        "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}"
 
       mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/
     fi

06_create_cluster.sh

@@ -23,13 +23,21 @@ fi
 
 # NOTE: This is equivalent to the external API DNS record pointing the API to the API VIP
 if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
-    API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then
+        API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    else
+        API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}')
+    fi
     INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))")
-    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf
+    echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf
     echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf
     sudo systemctl reload NetworkManager
 else
-    API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then
+        API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    else
+        API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}"  | awk '{print $NF}')
+    fi
     INGRESS_VIP=$(dig +noall +answer "test.apps.${CLUSTER_DOMAIN}" | awk '{print $NF}')
 fi
 

common.sh

@@ -39,13 +39,35 @@ if [ -z "${CONFIG:-}" ]; then
 fi
 source $CONFIG
 
+# Provisioning network information
+export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-172.22.0.0/24}
+export PROVISIONING_NETMASK=${PROVISIONING_NETMASK:-$(ipcalc --netmask $PROVISIONING_NETWORK | cut -d= -f2)}
+export CLUSTER_PRO_IF=${CLUSTER_PRO_IF:-enp1s0}
+
+export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org}
+export CLUSTER_NAME=${CLUSTER_NAME:-ostest}
+export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}"
+export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}"
+export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"}
+export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"}
+export CLUSTER_SUBNET=${CLUSTER_SUBNET:-"10.128.0.0/14"}
+export CLUSTER_HOST_PREFIX=${CLUSTER_HOST_PREFIX:-"23"}
+export SERVICE_SUBNET=${SERVICE_SUBNET:-"172.30.0.0/16"}
+export DNS_VIP=${DNS_VIP:-"192.168.111.2"}
+export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"}
+
+# ipcalc on CentOS 7 doesn't support the 'minaddr' option, so use python
+# instead to get the first address in the network:
+export PROVISIONING_HOST_IP=${PROVISIONING_HOST_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$PROVISIONING_NETWORK\").hosts()))")}
+export PROVISIONING_HOST_EXTERNAL_IP=${PROVISIONING_HOST_EXTERNAL_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$EXTERNAL_SUBNET\").hosts()))")}
+export MIRROR_IP=${MIRROR_IP:-$PROVISIONING_HOST_IP}
+
 # mirror images for installation in restricted network
 export MIRROR_IMAGES=${MIRROR_IMAGES:-}
 
 WORKING_DIR=${WORKING_DIR:-"/opt/dev-scripts"}
 
 # variables for local registry configuration
-export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"}
 export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"}
 export REGISTRY_USER=${REGISTRY_USER:-ocp-user}
 export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass}
@@ -97,7 +119,7 @@ fi
 
 if [ -n "$MIRROR_IMAGES" ]; then
     # We're going to be using a locally modified release image
-    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
+    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
 fi
 
 # Set variables
@@ -117,16 +139,6 @@ INT_IF=${INT_IF:-}
 #Root disk to deploy coreOS - use /dev/sda on BM
 ROOT_DISK_NAME=${ROOT_DISK_NAME-"/dev/sda"}
 
-# Provisioning network information
-export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-172.22.0.0/24}
-export PROVISIONING_NETMASK=${PROVISIONING_NETMASK:-$(ipcalc --netmask $PROVISIONING_NETWORK | cut -d= -f2)}
-
-export CLUSTER_PRO_IF=${CLUSTER_PRO_IF:-enp1s0}
-
-# ipcalc on CentOS 7 doesn't support the 'minaddr' option, so use python
-# instead to get the first address in the network:
-export PROVISIONING_HOST_IP=${PROVISIONING_HOST_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$PROVISIONING_NETWORK\").hosts()))")}
-
 FILESYSTEM=${FILESYSTEM:="/"}
 
 NODES_FILE=${NODES_FILE:-"${WORKING_DIR}/ironic_nodes.json"}

config_example.sh

@@ -67,6 +67,15 @@ set -x
 # External subnet
 #export EXTERNAL_SUBNET="11.0.0.0/24"
 
+# Cluster Subnet
+# export CLUSTER_SUBNET="10.128.0.0/14"
+
+# Cluster Host Prefix
+#export CLUSTER_HOST_PREFIX="23"
+
+# Service Subnet
+#export SERVICE_SUBNET="172.30.0.0/16"
+
 # Enable testing of custom machine-api-operator-image
 #export TEST_CUSTOM_MAO=true
 
@@ -79,7 +88,7 @@ set -x
 # Name of branch in the above repo which contains the custom MAO changes
 #export MAO_BRANCH="mao-fix"
 
-#export LOCAL_REGISTRY_ADDRESS="192.168.111.1"
+#export LOCAL_REGISTRY_DNS_NAME="virthost.ostest.test.metalkube.org"
 #export LOCAL_REGISTRY_PORT="5000"
 
 # configure username for registry

ocp_install_env.sh

@@ -1,14 +1,5 @@
 eval "$(go env)"
 
-export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org}
-export CLUSTER_NAME=${CLUSTER_NAME:-ostest}
-export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}"
-export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}"
-export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"}
-export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"}
-export MIRROR_IP=${MIRROR_IP:-$PROVISIONING_HOST_IP}
-export DNS_VIP=${DNS_VIP:-"192.168.111.2"}
-
 function extract_command() {
     local release_image
     local cmd
@@ -78,12 +69,25 @@ function generate_ocp_install_config() {
     fi
 
     mkdir -p "${outdir}"
+
+    # IPv6 network config validation
+    if [[ "${EXTERNAL_SUBNET}" =~ .*:.* ]]; then
+      if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then
+        echo "NETWORK_TYPE must be OVNKubernetes when using IPv6"
+        exit 1
+      fi
+    fi
     cat > "${outdir}/install-config.yaml" << EOF
 apiVersion: v1
 baseDomain: ${BASE_DOMAIN}
 networking:
   networkType: ${NETWORK_TYPE}
   machineCIDR: ${EXTERNAL_SUBNET}
+  clusterNetwork:
+  - cidr: ${CLUSTER_SUBNET}
+    hostPrefix: ${CLUSTER_HOST_PREFIX}
+  serviceNetwork:
+  - ${SERVICE_SUBNET}
 metadata:
   name: ${CLUSTER_NAME}
 compute:

utils.sh

@@ -219,10 +219,10 @@ function image_mirror_config {
             cat << EOF
 imageContentSources:
 - mirrors:
-    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
   source: ${RELEASE}
 - mirrors:
-    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+    - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
   source: ${TAGGED}
 additionalTrustBundle: |
 ${INDENTED_CERT}
@@ -247,13 +247,13 @@ function setup_local_registry() {
     sudo chown -R $USER:$USER ${REGISTRY_DIR}
 
     pushd $REGISTRY_DIR/certs
-    SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}"
+    SSL_HOST_NAME="${LOCAL_REGISTRY_DNS_NAME}"
 
-    if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then
+    if ipcalc -c $SSL_HOST_NAME; then
         SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}"
         SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}"
     else
-        SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}"
+        SSL_EXT_8="subjectAltName = DNS:${SSL_HOST_NAME}"
         SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}"
     fi
 
@@ -316,7 +316,7 @@ EOF
     if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then
         sudo podman rm registry -f || true
 
-        sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \
+        sudo podman run -d --name registry --net=host --privileged \
             -v ${REGISTRY_DIR}/data:/var/lib/registry:z \
             -v ${REGISTRY_DIR}/auth:/auth:z \
             -e "REGISTRY_AUTH=htpasswd" \

vm_setup_vars.yml

@@ -31,3 +31,6 @@ dns_extrahosts:
   - ip: "{{ baremetal_network_cidr | nthhost(2) }}"
     hostnames:
       - "ns1"
+  - ip: "{{ baremetal_network_cidr | nthhost(1) }}"
+    hostnames:
+      - "virthost"