Allow use of locally built images for CI/testing

01_install_requirements.sh

@@ -9,7 +9,7 @@ if grep -q "Red Hat Enterprise Linux release 8" /etc/redhat-release 2>/dev/null
     RHEL8="True"
 fi
 
-sudo yum install -y libselinux-utils
+sudo yum install -y libselinux-utils docker-distribution
 if selinuxenabled ; then
     # FIXME ocp-doit required this so leave permissive for now
     sudo setenforce permissive
@@ -27,6 +27,9 @@ ANSIBLE_FORCE_COLOR=true ansible-playbook \
   -b -vvv vm-setup/install-package-playbook.yml
 popd
 
+# needed if we are using locally built images
+sudo systemctl start docker-distribution
+
 # Install oc client
 oc_version=4.2
 oc_tools_dir=$HOME/oc-${oc_version}

02_configure_host.sh

@@ -125,18 +125,20 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then
     fi
 fi
 
-# Add firewall rules to ensure the image cache can be reached on the host
-if [ "${RHEL8}" = "True" ] ; then
-    sudo firewall-cmd --zone=libvirt --add-port=80/tcp
-    sudo firewall-cmd --zone=libvirt --add-port=80/tcp --permanent
-else
-    if ! sudo iptables -C INPUT -i provisioning -p tcp -m tcp --dport 80 -j ACCEPT > /dev/null 2>&1; then
-        sudo iptables -I INPUT -i provisioning -p tcp -m tcp --dport 80 -j ACCEPT
-    fi
-    if ! sudo iptables -C INPUT -i baremetal -p tcp -m tcp --dport 80 -j ACCEPT > /dev/null 2>&1; then
-        sudo iptables -I INPUT -i baremetal -p tcp -m tcp --dport 80 -j ACCEPT
+# Add firewall rules to ensure the image caches can be reached on the host
+for PORT in 80 5000 ; do
+    if [ "${RHEL8}" = "True" ] ; then
+        sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp
+        sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp --permanent
+    else
+        if ! sudo iptables -C INPUT -i provisioning -p tcp -m tcp --dport $PORT -j ACCEPT > /dev/null 2>&1; then
+            sudo iptables -I INPUT -i provisioning -p tcp -m tcp --dport $PORT -j ACCEPT
+        fi
+        if ! sudo iptables -C INPUT -i baremetal -p tcp -m tcp --dport $PORT -j ACCEPT > /dev/null 2>&1; then
+            sudo iptables -I INPUT -i baremetal -p tcp -m tcp --dport $PORT -j ACCEPT
+        fi
     fi
-fi
+done
 
 # Allow ipmi to the virtual bmc processes that we just started
 if [ "${RHEL8}" = "True" ] ; then

04_setup_ironic.sh

@@ -5,26 +5,55 @@ set -ex
 source logging.sh
 source common.sh
 source rhcos.sh
+source ocp_install_env.sh
 
-# Either pull or build the ironic images
-# To build the IRONIC image set
-# IRONIC_IMAGE=https://github.com/metalkube/metalkube-ironic
-for IMAGE_VAR in IRONIC_IMAGE IRONIC_INSPECTOR_IMAGE IPA_DOWNLOADER_IMAGE COREOS_DOWNLOADER_IMAGE VBMC_IMAGE SUSHY_TOOLS_IMAGE; do
+# To replace an image entry in the openshift releae image, set
+# <ENTRYNAME>_LOCAL_IMAGE - where ENTRYNAME matches an uppercase version of the name in the release image
+# with "-" converted to "_" e.g. to use a custom ironic-inspector
+#export IRONIC_INSPECTOR_LOCAL_IMAGE=https://github.com/metal3-io/ironic-inspector-image
+#export IRONIC_RHCOS_DOWNLOADER_LOCAL_IMAGE=https://github.com/openshift-metal3/rhcos-downloader
+#export BAREMETAL_OPERATOR_LOCAL_IMAGE=192.168.111.1:5000/localimages/bmo:latest
+rm -f assets/templates/99_local-registry.yaml $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf
+DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX")
+echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE
+for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do
     IMAGE=${!IMAGE_VAR}
+
+    export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX")
+    { echo "${PULL_SECRET}" ; } 2> /dev/null > $REGISTRY_AUTH_FILE
+    sudo -E podman pull $OPENSHIFT_RELEASE_IMAGE
+    rm -rf $REGISTRY_AUTH_FILE
+
     # Is it a git repo?
     if [[ "$IMAGE" =~ "://" ]] ; then
         REPOPATH=~/${IMAGE##*/}
         # Clone to ~ if not there already
         [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH
         cd $REPOPATH
-        export $IMAGE_VAR=localhost/${IMAGE##*/}:latest
+        export $IMAGE_VAR=${IMAGE##*/}:latest
+        export $IMAGE_VAR=192.168.111.1:5000/localimages/${!IMAGE_VAR}
         sudo podman build -t ${!IMAGE_VAR} .
         cd -
-    else
-        sudo podman pull "$IMAGE"
+        sudo podman push --tls-verify=false ${!IMAGE_VAR} ${!IMAGE_VAR}
     fi
+
+    # Update the bootstrap and master nodes to treat 192.168.111.1:5000 as insecure
+    mkdir -p $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers
+    echo -e "[registries.insecure]\nregistries = ['192.168.111.1:5000']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf
+    cp assets/templates/99_local-registry.yaml.optional assets/templates/99_local-registry.yaml
+
+    IMAGE_NAME=$(echo ${IMAGE_VAR/_LOCAL_IMAGE} | tr '[:upper:]_' '[:lower:]-')
+    OLDIMAGE=$(sudo podman run --rm $OPENSHIFT_RELEASE_IMAGE image $IMAGE_NAME)
+    echo "RUN sed -i 's%$OLDIMAGE%${!IMAGE_VAR}%g' /release-manifests/*" >> $DOCKERFILE
 done
 
+if [ -f assets/templates/99_local-registry.yaml ] ; then
+    build_installer
+    sudo podman image build -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE
+    sudo podman push --tls-verify=false $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE
+fi
+rm -f $DOCKERFILE
+
 for name in ironic ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb ipa-downloader coreos-downloader vbmc sushy-tools; do
     sudo podman ps | grep -w "$name$" && sudo podman kill $name
     sudo podman ps --all | grep -w "$name$" && sudo podman rm $name -f
@@ -39,6 +68,16 @@ fi
 sudo podman pod create -n ironic-pod 
 
 # We start the httpd and *downloader containers so that we can provide
+IRONIC_IMAGE=${IRONIC_LOCAL_IMAGE:-$IRONIC_IMAGE}
+IPA_DOWNLOADER_IMAGE=${IRONIC_IPA_DOWNLOADER_LOCAL_IMAGE:-$IPA_DOWNLOADER_IMAGE}
+COREOS_DOWNLOADER_IMAGE=${IRONIC_RHCOS_DOWNLOADER_LOCAL_IMAGE:-$COREOS_DOWNLOADER_IMAGE}
+VBMC_IMAGE=${VBMC_IMAGE_LOCAL_IMAGE:-$VBMC_IMAGE}
+SUSHY_TOOLS_IMAGE=${SUSHY_TOOLS_IMAGE_LOCAL_IMAGE:-$SUSHY_TOOLS_IMAGE}
+
+for IMAGE in ${IRONIC_IMAGE} ${IPA_DOWNLOADER_IMAGE} ${COREOS_DOWNLOADER_IMAGE} ${VBMC_IMAGE} ${SUSHY_TOOLS_IMAGE} ; do
+    sudo podman pull $([[ $IMAGE =~ 192.168.111.1:5000.* ]] && echo "--tls-verify=false" ) $IMAGE
+done
+
 # cached images to the bootstrap VM
 sudo podman run -d --net host --privileged --name httpd --pod ironic-pod \
      -v $IRONIC_DATA_DIR:/shared --entrypoint /bin/runhttpd ${IRONIC_IMAGE}

README.md

@@ -201,6 +201,20 @@ The default cpu/memory/disk resources when using virtual machines are provided
 by the [vm_setup_vars.yml](vm_setup_vars.yml) file, which sets some dev-scripts
 variables that override the defaults in metal3-dev-env
 
+### Testing a custom container images with dev-scripts
+dev-scripts uses an openshift release image that contains references to openshift
+containers, any of these containers can be overridden by setting environment
+variables of the form <NAME>_LOCAL_IMAGE to build or use copy of container
+images locally e.g. to use a custom ironic container image and build a container
+image from a git repository for the machine-config-operator you could set
+
+export IRONIC_LOCAL_IMAGE=quay.io/username/ironic
+export MACHINE_CONFIG_OPERATOR_LOCAL_IMAGE=https://github.com/openshift/machine-config-operator
+
+The value for <NAME> needs to match the name of the tags for images (found in the
+openshift release images in /release-manifests/image-references), converted to uppercase
+and with "-"'s converted to "_"'s.
+
 ### Testing a custom machine-api-operator image with this deployment
 
 The script `run-custom-mao.sh` allows the machine-api-operator pod to be re-deployed with a custom image.

assets/files/etc/sysconfig/crio-storage

@@ -0,0 +1,3 @@
+
+CRIO_STORAGE_OPTIONS="--insecure-registry=192.168.111.1:5000"
+

assets/templates/99_local-registry.yaml.optional

@@ -0,0 +1,18 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  generation: 1
+  labels:
+    machineconfiguration.openshift.io/role: master
+  name: 10-master-localimages
+spec:
+  config:
+    ignition:
+      version: 2.2.0
+    storage:
+      files:
+      - contents:
+          verification: {}
+        filesystem: root
+        mode: 0644
+        path: /etc/sysconfig/crio-storage

common.sh

@@ -39,6 +39,11 @@ source $CONFIG
 export OPENSHIFT_RELEASE_IMAGE="${OPENSHIFT_RELEASE_IMAGE:-registry.svc.ci.openshift.org/ocp/release:4.2}"
 export OPENSHIFT_INSTALL_PATH="$GOPATH/src/github.com/openshift/installer"
 
+if env | grep -q "_LOCAL_IMAGE=" ; then
+    # We need a custome installer (allows http image pulls for local images)
+    KNI_INSTALL_FROM_GIT=true
+fi
+
 if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
     export OPENSHIFT_INSTALLER=${OPENSHIFT_INSTALLER:-ocp/openshift-baremetal-install}
  else
@@ -55,6 +60,11 @@ if [ -z "$KNI_INSTALL_FROM_GIT" ]; then
     export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${OPENSHIFT_RELEASE_IMAGE}"
 fi
 
+if env | grep -q "_LOCAL_IMAGE=" ; then
+    # We're going to be using a locally modified release image
+    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="192.168.111.1:5000/localimages/local-release-image:latest" 
+fi
+
 # Set variables
 # Additional DNS
 ADDN_DNS=${ADDN_DNS:-}