diff --git a/01_install_requirements.sh b/01_install_requirements.sh index e95655abe..9271e621c 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -30,11 +30,6 @@ popd if sudo systemctl is-active docker-distribution.service; then sudo systemctl disable --now docker-distribution.service fi -reg_state=$(sudo podman inspect registry --format "{{.State.Status}}" || echo "error") -if [[ "$reg_state" != "running" ]]; then - sudo podman rm registry -f || true - sudo podman run -d -p 5000:5000 --name registry docker.io/registry:latest -fi # Install oc client oc_version=4.4 @@ -66,3 +61,7 @@ if ! which dep 2>&1 >/dev/null ; then mkdir -p $GOPATH/bin curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh fi + +if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then + setup_local_registry +fi diff --git a/02_configure_host.sh b/02_configure_host.sh index 5ac756483..a82f02372 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -124,7 +124,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then fi # Add firewall rules to ensure the image caches can be reached on the host -for PORT in 80 5000 ; do +for PORT in 80 ${LOCAL_REGISTRY_PORT} ; do if [ "${RHEL8}" = "True" ] ; then sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp --permanent @@ -168,3 +168,10 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then sudo systemctl restart NetworkManager fi fi + +if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then + # create authfile for local registry + sudo podman login --authfile ${REGISTRY_CREDS} \ + -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \ + ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT} +fi diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index 190466482..ef7ddca22 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -19,12 +19,16 @@ rm -f assets/templates/99_local-registry.yaml $OPENSHIFT_INSTALL_PATH/data/data/ export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX") { echo "${PULL_SECRET}" ; } 2> /dev/null > $REGISTRY_AUTH_FILE +# Combine pull-secret with registry's password +COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX") +jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE} + DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX") echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do IMAGE=${!IMAGE_VAR} - sudo -E podman pull $OPENSHIFT_RELEASE_IMAGE + sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE # Is it a git repo? if [[ "$IMAGE" =~ "://" ]] ; then @@ -33,15 +37,15 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH cd $REPOPATH export $IMAGE_VAR=${IMAGE##*/}:latest - export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS/localimages/${!IMAGE_VAR} - sudo podman build -t ${!IMAGE_VAR} . + export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} + sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} . cd - - sudo podman push --tls-verify=false ${!IMAGE_VAR} ${!IMAGE_VAR} + sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR} fi # Update the bootstrap and master nodes to treat LOCAL_REGISTRY_ADDRESS as insecure mkdir -p $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers - echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf + echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf cp assets/templates/99_local-registry.yaml.optional assets/templates/99_local-registry.yaml IMAGE_NAME=$(echo ${IMAGE_VAR/_LOCAL_IMAGE} | tr '[:upper:]_' '[:lower:]-') @@ -49,10 +53,40 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do echo "RUN sed -i 's%$OLDIMAGE%${!IMAGE_VAR}%g' /release-manifests/*" >> $DOCKERFILE done +if [ ! -z "${MIRROR_IMAGES}" ]; then + + # combine global and local secrets + # pull from one registry and push to local one + # hence credentials are different + + EXTRACT_DIR=$(mktemp -d "mirror-installer--XXXXXXXXXX") + + TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' ) + MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log + + oc adm release mirror \ + --insecure=true \ + -a ${COMBINED_AUTH_FILE} \ + --from ${OPENSHIFT_RELEASE_IMAGE} \ + --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ + --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} + + #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, + #you must extract the installation program from the mirrored content: + + oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ + --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ + "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + + mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ + + rm -rf "${EXTRACT_DIR}" +fi + if [ -f assets/templates/99_local-registry.yaml ] ; then build_installer - sudo podman image build -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE - sudo podman push --tls-verify=false $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE + sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE + sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE fi rm -f $DOCKERFILE @@ -62,12 +96,12 @@ for name in ironic ironic-api ironic-conductor ironic-inspector dnsmasq httpd ma done # Remove existing pod -if sudo podman pod exists ironic-pod ; then +if sudo podman pod exists ironic-pod ; then sudo podman pod rm ironic-pod -f fi # Create pod -sudo podman pod create -n ironic-pod +sudo podman pod create -n ironic-pod IRONIC_IMAGE=${IRONIC_LOCAL_IMAGE:-$IRONIC_IMAGE} IRONIC_IPA_DOWNLOADER_IMAGE=${IRONIC_IPA_DOWNLOADER_LOCAL_IMAGE:-$IRONIC_IPA_DOWNLOADER_IMAGE} @@ -76,7 +110,8 @@ for IMAGE in ${IRONIC_IMAGE} ${IRONIC_IPA_DOWNLOADER_IMAGE} ${VBMC_IMAGE} ${SUSH sudo -E podman pull $([[ $IMAGE =~ $LOCAL_REGISTRY_ADDRESS.* ]] && echo "--tls-verify=false" ) $IMAGE done -rm -rf $REGISTRY_AUTH_FILE +rm -rf ${REGISTRY_AUTH_FILE} +rm -rf ${COMBINED_AUTH_FILE} CACHED_MACHINE_OS_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_IMAGE_NAME}" if [ ! -f "${CACHED_MACHINE_OS_IMAGE}" ]; then @@ -102,7 +137,7 @@ if [ "$NODES_PLATFORM" = "libvirt" ]; then sudo podman run -d --net host --privileged --name vbmc --pod ironic-pod \ -v "$WORKING_DIR/virtualbmc/vbmc":/root/.vbmc -v "/root/.ssh":/root/ssh \ "${VBMC_IMAGE}" - + sudo podman run -d --net host --privileged --name sushy-tools --pod ironic-pod \ -v "$WORKING_DIR/virtualbmc/sushy-tools":/root/sushy -v "/root/.ssh":/root/ssh \ "${SUSHY_TOOLS_IMAGE}" diff --git a/common.sh b/common.sh index 956b4a2d2..a3133a8d5 100644 --- a/common.sh +++ b/common.sh @@ -36,8 +36,6 @@ if [ -z "${CONFIG:-}" ]; then fi source $CONFIG -export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1:5000"} - # # See https://openshift-release.svc.ci.openshift.org for release details # @@ -80,7 +78,7 @@ fi if env | grep -q "_LOCAL_IMAGE=" ; then # We're going to be using a locally modified release image - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image:latest" + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" fi # Set variables @@ -215,3 +213,14 @@ fi # Defaults the variable to enable testing a custom machine-api-operator image export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false} + +# mirror images for installation in restricted network +export MIRROR_IMAGES=${MIRROR_IMAGES:-} + +# variables for local registry configuration +export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"} +export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"} +export REGISTRY_USER=${REGISTRY_USER:-ocp-user} +export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass} +export REGISTRY_DIR=${REGISTRY_DIR:-$WORKING_DIR/registry} +export REGISTRY_CREDS=${REGISTRY_CREDS:-$HOME/private-mirror.json} diff --git a/config_example.sh b/config_example.sh index 1517e7268..5fcf22b54 100644 --- a/config_example.sh +++ b/config_example.sh @@ -11,6 +11,9 @@ set -x #export IRONIC_LOCAL_IMAGE=quay.io/username/ironic #export MACHINE_CONFIG_OPERATOR_LOCAL_IMAGE=https://github.com/openshift/machine-config-operator +# Mirror latest ci images to local registry +#export MIRROR_IMAGES=true + # Switch to upstream metal3-io ironic images instead of openshift ones. #export UPSTREAM_IRONIC=true @@ -73,4 +76,17 @@ set -x # Name of branch in the above repo which contains the custom MAO changes #export MAO_BRANCH="mao-fix" -#export LOCAL_REGISTRY_ADDRESS="192.168.111.1:5000" +#export LOCAL_REGISTRY_ADDRESS="192.168.111.1" +#export LOCAL_REGISTRY_PORT="5000" + +# configure username for registry +#export REGISTRY_USER=some-user + +# congiugre password for registry user +#export REGISTRY_PASS=some-pass + +# configure base directory for registry +#export REGISTRY_DIR=/opt/registry + +# configure location of mirror's creds +#export REGISTRY_CREDS=${REGISTRY_CREDS:-$USER/private-mirror.json} diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 8dcde5232..517cfef5a 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -78,6 +78,12 @@ function generate_ocp_install_config() { # TODO - Change worker replicas to ${NUM_WORKERS} once the machine-api-operator # deploys the baremetal-operator + # when using local mirror set pull secret to this mirror + # also this should ensure we don't accidentally pull from upstream + if [ ! -z "${MIRROR_IMAGES}" ]; then + export PULL_SECRET=$(cat ${REGISTRY_CREDS}) + fi + mkdir -p "${outdir}" cat > "${outdir}/install-config.yaml" << EOF apiVersion: v1 @@ -102,6 +108,7 @@ platform: dnsVIP: ${DNS_VIP} hosts: $(master_node_map_to_install_config $NUM_MASTERS) +$(image_mirror_config) pullSecret: | $(echo $PULL_SECRET | jq -c .) sshKey: | diff --git a/utils.sh b/utils.sh index b0a7cc000..76f77de70 100644 --- a/utils.sh +++ b/utils.sh @@ -195,3 +195,125 @@ function bmo_config_map { cp ocp/deploy/metal3-config.yaml assets/generated/99_metal3-config.yaml } + +function image_mirror_config { + if [ ! -z "${MIRROR_IMAGES}" ]; then + TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' ) + TAGGED=$(echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/release://') + RELEASE=$(echo $OPENSHIFT_RELEASE_IMAGE | grep -o 'registry.svc.ci.openshift.org[^":]\+') + INDENTED_CERT=$( cat $REGISTRY_DIR/certs/registry.crt | awk '{ print " ", $0 }' ) + MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log + if [ ! -s ${MIRROR_LOG_FILE} ]; then + cat << EOF +imageContentSources: +- mirrors: + - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + source: ${RELEASE} +- mirrors: + - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image + source: ${TAGGED} +additionalTrustBundle: | +${INDENTED_CERT} +EOF + else + cat ${MIRROR_LOG_FILE} | sed -n '/To use the new mirrored repository to install/,/To use the new mirrored repository for upgrades/p' |\ + sed -e '/^$/d' -e '/To use the new mirrored repository/d' + cat << EOF +additionalTrustBundle: | +${INDENTED_CERT} +EOF + fi + fi +} + +function setup_local_registry() { + + # httpd-tools provides htpasswd utility + sudo yum install -y httpd-tools + + sudo mkdir -pv ${REGISTRY_DIR}/{auth,certs,data} + sudo chown -R $USER:$USER ${REGISTRY_DIR} + + pushd $REGISTRY_DIR/certs + SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" + + if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then + SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" + SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" + else + SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}" + SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}" + fi + + # + # registry key and cert are generated if they don't exist + # + if [[ ! -s ${REGISTRY_DIR}/certs/registry.key ]]; then + openssl genrsa -out ${REGISTRY_DIR}/certs/registry.key 2048 + fi + + if [[ ! -s ${REGISTRY_DIR}/certs/registry.crt ]]; then + + if [ "${RHEL8}" = "True" ] ; then + openssl req -x509 \ + -key ${REGISTRY_DIR}/certs/registry.key \ + -out ${REGISTRY_DIR}/certs/registry.crt \ + -days 365 \ + -addext "${SSL_EXT_8}" \ + -subj "/C=US/ST=NC/L=Raleigh/O=Test Company/OU=Testing/CN=${SSL_HOST_NAME}" + else + SSL_TMP_CONF=$(mktemp 'my-ssl-conf.XXXXXX') + cat > ${SSL_TMP_CONF} <