Add SSL support to local mirrored registry

01_install_requirements.sh

@@ -30,11 +30,6 @@ popd
 if sudo systemctl is-active docker-distribution.service; then
   sudo systemctl disable --now docker-distribution.service
 fi
-reg_state=$(sudo podman inspect registry --format  "{{.State.Status}}" || echo "error")
-if [[ "$reg_state" != "running" ]]; then
-  sudo podman rm registry -f || true
-  sudo podman run -d -p 5000:5000 --name registry docker.io/registry:latest
-fi
 
 # Install oc client
 oc_version=4.4
@@ -66,3 +61,7 @@ if ! which dep 2>&1 >/dev/null ; then
     mkdir -p $GOPATH/bin
     curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
 fi
+
+if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then
+    setup_local_registry
+fi

02_configure_host.sh

@@ -124,7 +124,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
 fi
 
 # Add firewall rules to ensure the image caches can be reached on the host
-for PORT in 80 5000 ; do
+for PORT in 80 ${LOCAL_REGISTRY_PORT} ; do
     if [ "${RHEL8}" = "True" ] ; then
         sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp
         sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp --permanent
@@ -168,3 +168,10 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
     sudo systemctl restart NetworkManager
   fi
 fi
+
+if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then
+    # create authfile for local registry
+    sudo podman login --authfile ${REGISTRY_CREDS} \
+        -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \
+        ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}
+fi

04_setup_ironic.sh

@@ -19,12 +19,16 @@ rm -f assets/templates/99_local-registry.yaml $OPENSHIFT_INSTALL_PATH/data/data/
 export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX")
 { echo "${PULL_SECRET}" ; } 2> /dev/null > $REGISTRY_AUTH_FILE
 
+# Combine pull-secret with registry's password
+COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX")
+jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE}
+
 DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX")
 echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE
 for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
     IMAGE=${!IMAGE_VAR}
 
-    sudo -E podman pull $OPENSHIFT_RELEASE_IMAGE
+    sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE
 
     # Is it a git repo?
     if [[ "$IMAGE" =~ "://" ]] ; then
@@ -33,26 +37,56 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
         [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH
         cd $REPOPATH
         export $IMAGE_VAR=${IMAGE##*/}:latest
-        export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS/localimages/${!IMAGE_VAR}
-        sudo podman build -t ${!IMAGE_VAR} .
+        export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR}
+        sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} .
         cd -
-        sudo podman push --tls-verify=false ${!IMAGE_VAR} ${!IMAGE_VAR}
+        sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR}
     fi
 
     # Update the bootstrap and master nodes to treat LOCAL_REGISTRY_ADDRESS as insecure
     mkdir -p $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers
-    echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf
+    echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf
     cp assets/templates/99_local-registry.yaml.optional assets/templates/99_local-registry.yaml
 
     IMAGE_NAME=$(echo ${IMAGE_VAR/_LOCAL_IMAGE} | tr '[:upper:]_' '[:lower:]-')
     OLDIMAGE=$(sudo podman run --rm $OPENSHIFT_RELEASE_IMAGE image $IMAGE_NAME)
     echo "RUN sed -i 's%$OLDIMAGE%${!IMAGE_VAR}%g' /release-manifests/*" >> $DOCKERFILE
 done
 
+if [ ! -z "${MIRROR_IMAGES}" ]; then
+
+    # combine global and local secrets
+    # pull from one registry and push to local one
+    # hence credentials are different
+
+    EXTRACT_DIR=$(mktemp -d "mirror-installer--XXXXXXXXXX")
+
+    TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' )
+    MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log
+
+    oc adm release mirror \
+       --insecure=true \
+        -a ${COMBINED_AUTH_FILE}  \
+        --from ${OPENSHIFT_RELEASE_IMAGE} \
+        --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \
+        --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE}
+
+    #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected,
+    #you must extract the installation program from the mirrored content:
+
+    oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \
+        --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \
+        "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}"
+
+    mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/
+
+    rm -rf "${EXTRACT_DIR}"
+fi
+
 if [ -f assets/templates/99_local-registry.yaml ] ; then
     build_installer
-    sudo podman image build -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE
-    sudo podman push --tls-verify=false $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
+    sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE
+    sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
 fi
 rm -f $DOCKERFILE
 
@@ -62,12 +96,12 @@ for name in ironic ironic-api ironic-conductor ironic-inspector dnsmasq httpd ma
 done
 
 # Remove existing pod
-if  sudo podman pod exists ironic-pod ; then 
+if  sudo podman pod exists ironic-pod ; then
     sudo podman pod rm ironic-pod -f
 fi
 
 # Create pod
-sudo podman pod create -n ironic-pod 
+sudo podman pod create -n ironic-pod
 
 IRONIC_IMAGE=${IRONIC_LOCAL_IMAGE:-$IRONIC_IMAGE}
 IRONIC_IPA_DOWNLOADER_IMAGE=${IRONIC_IPA_DOWNLOADER_LOCAL_IMAGE:-$IRONIC_IPA_DOWNLOADER_IMAGE}
@@ -76,7 +110,8 @@ for IMAGE in ${IRONIC_IMAGE} ${IRONIC_IPA_DOWNLOADER_IMAGE} ${VBMC_IMAGE} ${SUSH
     sudo -E podman pull $([[ $IMAGE =~ $LOCAL_REGISTRY_ADDRESS.* ]] && echo "--tls-verify=false" ) $IMAGE
 done
 
-rm -rf $REGISTRY_AUTH_FILE
+rm -rf ${REGISTRY_AUTH_FILE}
+rm -rf ${COMBINED_AUTH_FILE}
 
 CACHED_MACHINE_OS_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_IMAGE_NAME}"
 if [ ! -f "${CACHED_MACHINE_OS_IMAGE}" ]; then
@@ -102,7 +137,7 @@ if [ "$NODES_PLATFORM" = "libvirt" ]; then
     sudo podman run -d --net host --privileged --name vbmc --pod ironic-pod \
          -v "$WORKING_DIR/virtualbmc/vbmc":/root/.vbmc -v "/root/.ssh":/root/ssh \
          "${VBMC_IMAGE}"
-    
+
     sudo podman run -d --net host --privileged --name sushy-tools --pod ironic-pod \
          -v "$WORKING_DIR/virtualbmc/sushy-tools":/root/sushy -v "/root/.ssh":/root/ssh \
          "${SUSHY_TOOLS_IMAGE}"

common.sh

@@ -36,8 +36,6 @@ if [ -z "${CONFIG:-}" ]; then
 fi
 source $CONFIG
 
-export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1:5000"}
-
 #
 # See https://openshift-release.svc.ci.openshift.org for release details
 #
@@ -80,7 +78,7 @@ fi
 
 if env | grep -q "_LOCAL_IMAGE=" ; then
     # We're going to be using a locally modified release image
-    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image:latest"
+    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest"
 fi
 
 # Set variables
@@ -215,3 +213,14 @@ fi
 
 # Defaults the variable to enable testing a custom machine-api-operator image
 export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false}
+
+# mirror images for installation in restricted network
+export MIRROR_IMAGES=${MIRROR_IMAGES:-}
+
+# variables for local registry configuration
+export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"}
+export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"}
+export REGISTRY_USER=${REGISTRY_USER:-ocp-user}
+export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass}
+export REGISTRY_DIR=${REGISTRY_DIR:-$WORKING_DIR/registry}
+export REGISTRY_CREDS=${REGISTRY_CREDS:-$HOME/private-mirror.json}

config_example.sh

@@ -11,6 +11,9 @@ set -x
 #export IRONIC_LOCAL_IMAGE=quay.io/username/ironic
 #export MACHINE_CONFIG_OPERATOR_LOCAL_IMAGE=https://github.com/openshift/machine-config-operator
 
+# Mirror latest ci images to local registry
+#export MIRROR_IMAGES=true
+
 # Switch to upstream metal3-io ironic images instead of openshift ones.
 #export UPSTREAM_IRONIC=true
 
@@ -73,4 +76,17 @@ set -x
 # Name of branch in the above repo which contains the custom MAO changes
 #export MAO_BRANCH="mao-fix"
 
-#export LOCAL_REGISTRY_ADDRESS="192.168.111.1:5000"
+#export LOCAL_REGISTRY_ADDRESS="192.168.111.1"
+#export LOCAL_REGISTRY_PORT="5000"
+
+# configure username for registry
+#export REGISTRY_USER=some-user
+
+# congiugre password for registry user
+#export REGISTRY_PASS=some-pass
+
+# configure base directory for registry
+#export REGISTRY_DIR=/opt/registry
+
+# configure location of mirror's creds
+#export REGISTRY_CREDS=${REGISTRY_CREDS:-$USER/private-mirror.json}

ocp_install_env.sh

@@ -78,6 +78,12 @@ function generate_ocp_install_config() {
     # TODO - Change worker replicas to ${NUM_WORKERS} once the machine-api-operator
     # deploys the baremetal-operator
 
+    # when using local mirror set pull secret to this mirror
+    # also this should ensure we don't accidentally pull from upstream
+    if [ ! -z "${MIRROR_IMAGES}" ]; then
+        export PULL_SECRET=$(cat ${REGISTRY_CREDS})
+    fi
+
     mkdir -p "${outdir}"
     cat > "${outdir}/install-config.yaml" << EOF
 apiVersion: v1
@@ -102,6 +108,7 @@ platform:
     dnsVIP: ${DNS_VIP}
     hosts:
 $(master_node_map_to_install_config $NUM_MASTERS)
+$(image_mirror_config)
 pullSecret: |
   $(echo $PULL_SECRET | jq -c .)
 sshKey: |

utils.sh

@@ -195,3 +195,125 @@ function bmo_config_map {
 
     cp ocp/deploy/metal3-config.yaml assets/generated/99_metal3-config.yaml
 }
+
+function image_mirror_config {
+    if [ ! -z "${MIRROR_IMAGES}" ]; then
+        TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' )
+        TAGGED=$(echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/release://')
+        RELEASE=$(echo $OPENSHIFT_RELEASE_IMAGE | grep -o 'registry.svc.ci.openshift.org[^":]\+')
+        INDENTED_CERT=$( cat $REGISTRY_DIR/certs/registry.crt | awk '{ print " ", $0 }' )
+        MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log
+        if [ ! -s ${MIRROR_LOG_FILE} ]; then
+            cat << EOF
+imageContentSources:
+- mirrors:
+    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+  source: ${RELEASE}
+- mirrors:
+    - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image
+  source: ${TAGGED}
+additionalTrustBundle: |
+${INDENTED_CERT}
+EOF
+        else
+            cat ${MIRROR_LOG_FILE} | sed -n '/To use the new mirrored repository to install/,/To use the new mirrored repository for upgrades/p' |\
+                sed -e '/^$/d' -e '/To use the new mirrored repository/d'
+            cat << EOF
+additionalTrustBundle: |
+${INDENTED_CERT}
+EOF
+        fi
+    fi
+}
+
+function setup_local_registry() {
+
+    # httpd-tools provides htpasswd utility
+    sudo yum install -y httpd-tools
+
+    sudo mkdir -pv ${REGISTRY_DIR}/{auth,certs,data}
+    sudo chown -R $USER:$USER ${REGISTRY_DIR}
+
+    pushd $REGISTRY_DIR/certs
+    SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}"
+
+    if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then
+        SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}"
+        SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}"
+    else
+        SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}"
+        SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}"
+    fi
+
+    #
+    # registry key and cert are generated if they don't exist
+    #
+    if [[ ! -s ${REGISTRY_DIR}/certs/registry.key ]]; then
+        openssl genrsa -out ${REGISTRY_DIR}/certs/registry.key 2048
+    fi
+
+    if [[ ! -s ${REGISTRY_DIR}/certs/registry.crt ]]; then
+
+        if [ "${RHEL8}" = "True" ] ; then
+            openssl req -x509 \
+                -key ${REGISTRY_DIR}/certs/registry.key \
+                -out ${REGISTRY_DIR}/certs/registry.crt \
+                -days 365 \
+                -addext "${SSL_EXT_8}" \
+                -subj "/C=US/ST=NC/L=Raleigh/O=Test Company/OU=Testing/CN=${SSL_HOST_NAME}"
+        else
+            SSL_TMP_CONF=$(mktemp 'my-ssl-conf.XXXXXX')
+            cat > ${SSL_TMP_CONF} <<EOF
+[req]
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+CN = ${SSL_HOST_NAME}
+
+[SAN]
+basicConstraints=CA:TRUE,pathlen:0
+${SSL_EXT_7}
+EOF
+
+            openssl req -x509 \
+                -key ${REGISTRY_DIR}/certs/registry.key \
+                -out  ${REGISTRY_DIR}/certs/registry.crt \
+                -days 365 \
+                -config ${SSL_TMP_CONF} \
+                -extensions SAN \
+                -subj "/C=US/ST=NC/L=Raleigh/O=Test Company/OU=Testing/CN=${SSL_HOST_NAME}"
+        fi
+    fi
+
+    # get MD5 hashes for SSL cert on a disk and one used in running registry
+    SSL_CERT_MD5_HASH=$( md5sum ${REGISTRY_DIR}/certs/registry.crt | awk '{print $1}' )
+    MD5_HASH_RUNNING=$( sudo podman exec registry /bin/sh -c "md5sum /certs/registry.crt || echo not_exist" | awk '{print $1}' || echo "error" )
+
+    popd
+
+    htpasswd -bBc ${REGISTRY_DIR}/auth/htpasswd ${REGISTRY_USER} ${REGISTRY_PASS}
+
+    sudo cp ${REGISTRY_DIR}/certs/registry.crt /etc/pki/ca-trust/source/anchors/
+    sudo update-ca-trust
+
+    reg_state=$(sudo podman inspect registry --format  "{{.State.Status}}" || echo "error")
+
+    # if container doesn't run or has different SSL cert that preent in ${REGISTRY_DIR}/certs/
+    #   restart it
+
+    if [[ "$reg_state" != "running" || "$SSL_CERT_MD5_HASH" != "$MD5_HASH_RUNNING" ]]; then
+        sudo podman rm registry -f || true
+
+        sudo podman run -d --name registry -p ${LOCAL_REGISTRY_PORT}:5000 \
+            -v ${REGISTRY_DIR}/data:/var/lib/registry:z \
+            -v ${REGISTRY_DIR}/auth:/auth:z \
+            -e "REGISTRY_AUTH=htpasswd" \
+            -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
+            -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
+            -v ${REGISTRY_DIR}/certs:/certs:z \
+            -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \
+            -e REGISTRY_HTTP_TLS_KEY=/certs/registry.key \
+            docker.io/registry:latest
+    fi
+
+}