diff --git a/01_install_requirements.sh b/01_install_requirements.sh index a0d7c3ce6..715ec0a6a 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -4,6 +4,7 @@ set -ex source logging.sh source common.sh source utils.sh +source ocp_install_env.sh if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null ; then RHEL8="True" diff --git a/02_configure_host.sh b/02_configure_host.sh index 702321dd5..068c46e7f 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -173,6 +173,10 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then fi fi +# Add a /etc/hosts entry for $LOCAL_REGISTRY_DNS_NAME +sudo sed -i "/${LOCAL_REGISTRY_DNS_NAME}/d" /etc/hosts +echo "${PROVISIONING_HOST_EXTERNAL_IP} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts + # Remove any previous file, or podman login panics when reading the # blank authfile with a "assignment to entry in nil map" error rm -f ${REGISTRY_CREDS} @@ -180,7 +184,7 @@ if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then # create authfile for local registry sudo podman login --authfile ${REGISTRY_CREDS} \ -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \ - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT} + ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT} else # Create a blank authfile in order to have something valid when we read it in 04_setup_ironic.sh echo '{}' | sudo dd of=${REGISTRY_CREDS} diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index a961e17a1..4c5a78c2a 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -37,7 +37,7 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH cd $REPOPATH export $IMAGE_VAR=${IMAGE##*/}:latest - export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} + export $IMAGE_VAR=$LOCAL_REGISTRY_DNS_NAME:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} . cd - sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR} @@ -63,15 +63,15 @@ if [ ! -z "${MIRROR_IMAGES}" ]; then --insecure=true \ -a ${COMBINED_AUTH_FILE} \ --from ${OPENSHIFT_RELEASE_IMAGE} \ - --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ - --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} + --to-release-image ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ + --to ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, #you must extract the installation program from the mirrored content: if [ -z "$KNI_INSTALL_FROM_GIT" ]; then oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ - "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + "${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ fi diff --git a/06_create_cluster.sh b/06_create_cluster.sh index 5af35b7cd..96cff9f93 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -23,13 +23,21 @@ fi # NOTE: This is equivalent to the external API DNS record pointing the API to the API VIP if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @$(network_ip baremetal) | awk '{print $NF}') + fi INGRESS_VIP=$(python -c "from ansible.plugins.filter import ipaddr; print(ipaddr.nthhost('"$EXTERNAL_SUBNET"', 4))") - echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee /etc/NetworkManager/dnsmasq.d/openshift.conf + echo "address=/api.${CLUSTER_DOMAIN}/${API_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf echo "address=/.apps.${CLUSTER_DOMAIN}/${INGRESS_VIP}" | sudo tee -a /etc/NetworkManager/dnsmasq.d/openshift.conf sudo systemctl reload NetworkManager else - API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + if [[ $EXTERNAL_SUBNET =~ .*:.* ]]; then + API_VIP=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + else + API_VIP=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" | awk '{print $NF}') + fi INGRESS_VIP=$(dig +noall +answer "test.apps.${CLUSTER_DOMAIN}" | awk '{print $NF}') fi diff --git a/common.sh b/common.sh index 51ff1de5e..4ace992a4 100644 --- a/common.sh +++ b/common.sh @@ -39,13 +39,35 @@ if [ -z "${CONFIG:-}" ]; then fi source $CONFIG +# Provisioning network information +export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-172.22.0.0/24} +export PROVISIONING_NETMASK=${PROVISIONING_NETMASK:-$(ipcalc --netmask $PROVISIONING_NETWORK | cut -d= -f2)} +export CLUSTER_PRO_IF=${CLUSTER_PRO_IF:-enp1s0} + +export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} +export CLUSTER_NAME=${CLUSTER_NAME:-ostest} +export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" +export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" +export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} +export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} +export CLUSTER_SUBNET=${CLUSTER_SUBNET:-"10.128.0.0/14"} +export CLUSTER_HOST_PREFIX=${CLUSTER_HOST_PREFIX:-"23"} +export SERVICE_SUBNET=${SERVICE_SUBNET:-"172.30.0.0/16"} +export DNS_VIP=${DNS_VIP:-"192.168.111.2"} +export LOCAL_REGISTRY_DNS_NAME=${LOCAL_REGISTRY_DNS_NAME:-"virthost.${CLUSTER_NAME}.${BASE_DOMAIN}"} + +# ipcalc on CentOS 7 doesn't support the 'minaddr' option, so use python +# instead to get the first address in the network: +export PROVISIONING_HOST_IP=${PROVISIONING_HOST_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$PROVISIONING_NETWORK\").hosts()))")} +export PROVISIONING_HOST_EXTERNAL_IP=${PROVISIONING_HOST_EXTERNAL_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$EXTERNAL_SUBNET\").hosts()))")} +export MIRROR_IP=${MIRROR_IP:-$PROVISIONING_HOST_IP} + # mirror images for installation in restricted network export MIRROR_IMAGES=${MIRROR_IMAGES:-} WORKING_DIR=${WORKING_DIR:-"/opt/dev-scripts"} # variables for local registry configuration -export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"} export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"} export REGISTRY_USER=${REGISTRY_USER:-ocp-user} export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass} @@ -97,7 +119,7 @@ fi if [ -n "$MIRROR_IMAGES" ]; then # We're going to be using a locally modified release image - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" fi # Set variables @@ -117,16 +139,6 @@ INT_IF=${INT_IF:-} #Root disk to deploy coreOS - use /dev/sda on BM ROOT_DISK_NAME=${ROOT_DISK_NAME-"/dev/sda"} -# Provisioning network information -export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-172.22.0.0/24} -export PROVISIONING_NETMASK=${PROVISIONING_NETMASK:-$(ipcalc --netmask $PROVISIONING_NETWORK | cut -d= -f2)} - -export CLUSTER_PRO_IF=${CLUSTER_PRO_IF:-enp1s0} - -# ipcalc on CentOS 7 doesn't support the 'minaddr' option, so use python -# instead to get the first address in the network: -export PROVISIONING_HOST_IP=${PROVISIONING_HOST_IP:-$(python -c "import ipaddress; print(next(ipaddress.ip_network(u\"$PROVISIONING_NETWORK\").hosts()))")} - FILESYSTEM=${FILESYSTEM:="/"} NODES_FILE=${NODES_FILE:-"${WORKING_DIR}/ironic_nodes.json"} diff --git a/config_example.sh b/config_example.sh index f524e4751..bdb4f2b31 100644 --- a/config_example.sh +++ b/config_example.sh @@ -67,6 +67,15 @@ set -x # External subnet #export EXTERNAL_SUBNET="11.0.0.0/24" +# Cluster Subnet +# export CLUSTER_SUBNET="10.128.0.0/14" + +# Cluster Host Prefix +#export CLUSTER_HOST_PREFIX="23" + +# Service Subnet +#export SERVICE_SUBNET="172.30.0.0/16" + # Enable testing of custom machine-api-operator-image #export TEST_CUSTOM_MAO=true @@ -79,7 +88,7 @@ set -x # Name of branch in the above repo which contains the custom MAO changes #export MAO_BRANCH="mao-fix" -#export LOCAL_REGISTRY_ADDRESS="192.168.111.1" +#export LOCAL_REGISTRY_DNS_NAME="virthost.ostest.test.metalkube.org" #export LOCAL_REGISTRY_PORT="5000" # configure username for registry diff --git a/ocp_install_env.sh b/ocp_install_env.sh index cedddeec7..f80709a67 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -1,14 +1,5 @@ eval "$(go env)" -export BASE_DOMAIN=${BASE_DOMAIN:-test.metalkube.org} -export CLUSTER_NAME=${CLUSTER_NAME:-ostest} -export CLUSTER_DOMAIN="${CLUSTER_NAME}.${BASE_DOMAIN}" -export SSH_PUB_KEY="${SSH_PUB_KEY:-$(cat $HOME/.ssh/id_rsa.pub)}" -export NETWORK_TYPE=${NETWORK_TYPE:-"OpenShiftSDN"} -export EXTERNAL_SUBNET=${EXTERNAL_SUBNET:-"192.168.111.0/24"} -export MIRROR_IP=${MIRROR_IP:-$PROVISIONING_HOST_IP} -export DNS_VIP=${DNS_VIP:-"192.168.111.2"} - function extract_command() { local release_image local cmd @@ -78,12 +69,25 @@ function generate_ocp_install_config() { fi mkdir -p "${outdir}" + + # IPv6 network config validation + if [[ "${EXTERNAL_SUBNET}" =~ .*:.* ]]; then + if [[ "${NETWORK_TYPE}" != "OVNKubernetes" ]]; then + echo "NETWORK_TYPE must be OVNKubernetes when using IPv6" + exit 1 + fi + fi cat > "${outdir}/install-config.yaml" << EOF apiVersion: v1 baseDomain: ${BASE_DOMAIN} networking: networkType: ${NETWORK_TYPE} machineCIDR: ${EXTERNAL_SUBNET} + clusterNetwork: + - cidr: ${CLUSTER_SUBNET} + hostPrefix: ${CLUSTER_HOST_PREFIX} + serviceNetwork: + - ${SERVICE_SUBNET} metadata: name: ${CLUSTER_NAME} compute: diff --git a/utils.sh b/utils.sh index be8961602..8f73ea0c1 100644 --- a/utils.sh +++ b/utils.sh @@ -219,10 +219,10 @@ function image_mirror_config { cat << EOF imageContentSources: - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${RELEASE} - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + - ${LOCAL_REGISTRY_DNS_NAME}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${TAGGED} additionalTrustBundle: | ${INDENTED_CERT} @@ -247,13 +247,13 @@ function setup_local_registry() { sudo chown -R $USER:$USER ${REGISTRY_DIR} pushd $REGISTRY_DIR/certs - SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" + SSL_HOST_NAME="${LOCAL_REGISTRY_DNS_NAME}" - if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then + if ipcalc -c $SSL_HOST_NAME; then SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" else - SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}" + SSL_EXT_8="subjectAltName = DNS:${SSL_HOST_NAME}" SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}" fi @@ -316,7 +316,7 @@ EOF if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then sudo podman rm registry -f || true - sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \ + sudo podman run -d --name registry --net=host --privileged \ -v ${REGISTRY_DIR}/data:/var/lib/registry:z \ -v ${REGISTRY_DIR}/auth:/auth:z \ -e "REGISTRY_AUTH=htpasswd" \ diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml index 7f0062ada..b95f96505 100644 --- a/vm_setup_vars.yml +++ b/vm_setup_vars.yml @@ -31,3 +31,6 @@ dns_extrahosts: - ip: "{{ baremetal_network_cidr | nthhost(2) }}" hostnames: - "ns1" + - ip: "{{ baremetal_network_cidr | nthhost(1) }}" + hostnames: + - "virthost"