diff --git a/.gitignore b/.gitignore index 0af329f7a..af4e07191 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,12 @@ openshift-install.log logs/ config_*.sh !config_example.sh + +release/release_config_*.sh +!release/release_config_example.sh +release/release-kubeconfig +release/release-pullsecret + assets/generated # Conditionally created for appropriate environments diff --git a/03_ocp_repo_sync.sh b/03_ocp_repo_sync.sh index ef1f091b9..fca441fc8 100755 --- a/03_ocp_repo_sync.sh +++ b/03_ocp_repo_sync.sh @@ -10,8 +10,6 @@ echo "$GOPATH" | lolcat # should print $HOME/go or something like that # REPO_PATH is used in sync_repo_and_patch from utils.sh export REPO_PATH="$GOPATH/src" -sync_repo_and_patch github.com/openshift-metalkube/kni-installer https://github.com/openshift-metalkube/kni-installer.git - # Build facet # FIXME(russellb) - disabled due to build failure related to metal3 rename #sync_repo_and_patch github.com/openshift-metalkube/facet https://github.com/openshift-metalkube/facet.git diff --git a/05_build_ocp_installer.sh b/05_build_ocp_installer.sh deleted file mode 100755 index f1164ad2e..000000000 --- a/05_build_ocp_installer.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -ex - -source logging.sh -source common.sh - -figlet "Building the Installer" | lolcat - -eval "$(go env)" -echo "$GOPATH" | lolcat # should print $HOME/go or something like that - -pushd "$GOPATH/src/github.com/openshift-metalkube/kni-installer" -export MODE=release -export TAGS="libvirt ironic" -./hack/build.sh -popd diff --git a/06_create_cluster.sh b/06_create_cluster.sh index 3a7bddd22..8ff538e46 100755 --- a/06_create_cluster.sh +++ b/06_create_cluster.sh @@ -8,7 +8,7 @@ source common.sh source ocp_install_env.sh # Do some PULL_SECRET sanity checking -if [[ "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}" == *"registry.svc.ci.openshift.org"* ]]; then +if [[ "${OPENSHIFT_RELEASE_IMAGE}" == *"registry.svc.ci.openshift.org"* ]]; then if [[ "${PULL_SECRET}" != *"registry.svc.ci.openshift.org"* ]]; then echo "Please get a valid pull secret for registry.svc.ci.openshift.org." exit 1 @@ -23,6 +23,9 @@ fi if [ ! -d ocp ]; then mkdir -p ocp + # Extract openshift-install from the release image + extract_installer "${OPENSHIFT_RELEASE_IMAGE}" ocp/ + # Create a master_nodes.json file jq '.nodes[0:3] | {nodes: .}' "${NODES_FILE}" | tee "${MASTER_NODES_FILE}" diff --git a/Makefile b/Makefile index 928df26be..764d43a92 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -.PHONY: default all requirements configure repo_sync ironic build ocp_run deploy_bmo register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell -default: requirements configure repo_sync ironic build ocp_run deploy_bmo register_hosts bell +.PHONY: default all requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell +default: requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts bell all: default @@ -15,9 +15,6 @@ repo_sync: ironic: ./04_setup_ironic.sh -build: - ./05_build_ocp_installer.sh - ocp_run: ./06_create_cluster.sh diff --git a/README.md b/README.md index 23675c156..acf1b89ae 100644 --- a/README.md +++ b/README.md @@ -71,14 +71,10 @@ server and download the resources it requires. The Ironic container is stored at https://quay.io/repository/metalkube/metalkube-ironic, built from https://github.com/metalkube/metalkube-ironic. -- `./05_build_ocp_installer.sh` - -These will pull and build the openshift-install and some other things from -source. - - `./06_create_cluster.sh` -This will run the kni-installer to generate ignition configs for the +This will extract openshift-install from the OCP release payload and +run `openshift-install` to generate ignition configs for the bootstrap node and the masters. The installer then launches both the bootstrap VM and master nodes using the Terraform providers for libvirt and Ironic. Once bootstrap is complete, the installer removes the @@ -133,7 +129,7 @@ e.g. to clean and re-install ocp run: ``` ./ocp_cleanup.sh rm -fr ocp -./05_run_ocp.sh +./06_create_cluster.sh ``` Or, you can run `make clean` which will run all of the cleanup steps. diff --git a/common.sh b/common.sh index 4f62c7afe..78478d151 100644 --- a/common.sh +++ b/common.sh @@ -47,7 +47,7 @@ export NUM_MASTERS=${NUM_MASTERS:-"3"} export NUM_WORKERS=${NUM_WORKERS:-"1"} export VM_EXTRADISKS=${VM_EXTRADISKS:-"false"} -export RHCOS_INSTALLER_IMAGE_URL=$(jq -r '.baseURI' $GOPATH/src/github.com/openshift-metalkube/kni-installer/data/data/rhcos.json) +export RHCOS_INSTALLER_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/ootpa/410.8.20190508.1/" export RHCOS_IMAGE_URL=${RHCOS_IMAGE_URL:-${RHCOS_INSTALLER_IMAGE_URL}} export RHCOS_IMAGE_FILENAME_OPENSTACK_GZ="$(curl ${RHCOS_IMAGE_URL}/meta.json | jq -r '.images.openstack.path')" diff --git a/docs/release-payload.md b/docs/release-payload.md new file mode 100644 index 000000000..28d4d900d --- /dev/null +++ b/docs/release-payload.md @@ -0,0 +1,112 @@ +# Publishing a KNI Release Payload + +OpenShift publishes a release payload image which includes information +about cluster operator images and their resource manifests, along with +references to installer and CLI images. The recommended method for +obtaining an installer binary is to first choose a release version and +then use the `oc adm release extract --tools` command to extract the +installer binary from the release payload. + +Since KNI has temporarily forked the installer, we build custom and +publish custom release payloads that include a reference to the forked +installer. + +## Preparation and Configuration + +We build and publish within a namespace on an OpenShift +cluster. First, prepare a `kubeconfig` with credentials to this +cluster, and with the desired namespace set as the default: + +``` +$ oc --config=release-kubeconfig login https://api.ci.openshift.org --token=... +$ oc --config=release-kubeconfig new-project kni +$ oc --config=release-kubeconfig project kni +$ oc --config=release-kubeconfig adm policy add-role-to-user admin +```` + +We need a docker registry credentials file which contains credentials +for the registry on this OpenShift cluster: + +``` +$ oc --config=release-kubeconfig registry login --to=release-pullsecret +``` + +But also, we need credentials for any registry hosting images +referenced from release payloads (e.g. ```quay.io```) + +``` +$ TOKEN=$((. ../config_$USER.sh && echo $PULL_SECRET) 2>/dev/null | jq -r '.auths["quay.io"].auth' | base64 -d) +$ podman login --authfile=release-pullsecret -u ${TOKEN%:*} -p ${TOKEN#*:} quay.io +``` + +Images are published to imagestream tags, and we need an image stream +for our installer builds and our custom release payloads: + +``` +$ oc --config=release-kubeconfig create imagestream release +$ oc --config=release-kubeconfig create imagestream installer +``` + +We need to create a ```docker-registry``` secret so the image stream +can import referenced images: + +``` +$ oc --config=release-kubeconfig \ + create secret docker-registry quay-pullsecret \ + --docker-server=quay.io \ + --docker-username=${TOKEN%:*} \ + --docker-password=${TOKEN#*:} +``` + +Finally, create a ```release_config_$USER.sh``` file with information +about all of the above: + +``` +$ cat > release_config_$USER.sh </dev/null && sudo systemctl reset-failed fix_certs.service if [ -d ocp ]; then - $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir ocp --log-level=debug destroy bootstrap - $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir ocp --log-level=debug destroy cluster + ocp/openshift-install --dir ocp --log-level=debug destroy bootstrap + ocp/openshift-install --dir ocp --log-level=debug destroy cluster rm -rf ocp fi diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 9056fc63e..13601c92c 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -13,7 +13,7 @@ export EXTERNAL_SUBNET="192.168.111.0/24" # The release we default to here is pinned and known to work with our current # version of kni-installer. # -export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="registry.svc.ci.openshift.org/kni/release:4.1.0-rc.3-kni.0" +export OPENSHIFT_RELEASE_IMAGE="registry.svc.ci.openshift.org/kni/release:4.1.0-rc.3-kni.1" function generate_ocp_install_config() { local outdir diff --git a/release/build_installer.sh b/release/build_installer.sh new file mode 100755 index 000000000..b2ce43f1a --- /dev/null +++ b/release/build_installer.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -xe + +# +# Build a new installer image +# +# See release_config_example.sh for required configuration steps +# + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +USER=`whoami` + +# Get variables from the config file +if [ -z "${RELEASE_CONFIG:-}" ]; then + # See if there's a release_config_$USER.sh in the SCRIPTDIR + if [ -f "${SCRIPTDIR}/release_config_${USER}.sh" ]; then + echo "Using RELEASE_CONFIG ${SCRIPTDIR}/release_config_${USER}.sh" + RELEASE_CONFIG="${SCRIPTDIR}/release_config_${USER}.sh" + else + echo "Please run with a configuration environment set." >&2 + echo "eg RELEASE_CONFIG=release_config_example.sh $0" >&2 + exit 1 + fi +fi +source $RELEASE_CONFIG + +INSTALLER_VERSION="$1" +if [ -z "${INSTALLER_VERSION}" ]; then + echo "usage: $0 " >&2 + echo "example: $0 4.0.0-0.9" >&2 + exit 1 +fi + +echo "Building kni-installer from ${INSTALLER_GIT_URI}:${INSTALLER_GIT_REF} to ${INSTALLER_STREAM}:${INSTALLER_VERSION}" + +# Check prerequisites +if [ $(oc --config "${RELEASE_KUBECONFIG}" project -q) != "${RELEASE_NAMESPACE}" ]; then + echo "Wrong namespace configured, run 'oc --config ${RELEASE_KUBECONFIG} project ${RELEASE_NAMESPACE}'" >&2 + exit 1 +fi + +if ! oc --config "${RELEASE_KUBECONFIG}" get imagestream "${INSTALLER_STREAM}" 2>/dev/null; then + echo "No '${INSTALLER_STREAM}' imagestream in '${RELEASE_NAMESPACE}' namespace" >&2 + exit 1 +fi + +oc --config "${RELEASE_KUBECONFIG}" apply -f - <&2 + echo "eg RELEASE_CONFIG=release_config_example.sh $0" >&2 + exit 1 + fi +fi +source $RELEASE_CONFIG + +RELEASE_NAME="$1"; shift +RELEASE_PULLSPEC="$1"; shift +if [ -z "${RELEASE_NAME}" -o -z "${RELEASE_PULLSPEC}" ]; then + echo "usage: $0 [= =]" >&2 + echo "example: $0 4.0.0-0.9-kni registry.svc.ci.openshift.org/ocp/release:4.0.0-0.9 installer=registry.svc.ci.openshift.org/kni/installer:4.0.0-0.9" >&2 + exit 1 +fi + +EXTRA_IMAGES=("$@") +for extra in "${EXTRA_IMAGES[@]}"; do + if [ -z "${extra%=*}" -o -z "${extra#*=}" ]; then + echo "Extra image parameters take the form =" >&2 + fi +done + +# Fetch the release version from payload metadata +RELEASE_VERSION=$(oc adm release info --registry-config "${RELEASE_PULLSECRET}" "${RELEASE_PULLSPEC}" -o json | jq -r .metadata.version) +if [ -z "${RELEASE_VERSION}" -o "${RELEASE_VERSION}" = "null" ]; then + echo "Could find version metadata in ${RELEASE_PULLSPEC}" >&2 + exit 1 +fi + +echo "Preparing a ${RELEASE_NAME} release based on version ${RELEASE_VERSION}" + +# Check prerequisites +if [ $(oc --config "${RELEASE_KUBECONFIG}" project -q) != "${RELEASE_NAMESPACE}" ]; then + echo "Wrong namespace configured, run 'oc --config ${RELEASE_KUBECONFIG} project ${RELEASE_NAMESPACE}'" >&2 + exit 1 +fi + +if ! oc --config "${RELEASE_KUBECONFIG}" get imagestream "${RELEASE_STREAM}" 2>/dev/null; then + echo "No '${RELEASE_STREAM}' imagestream in '${RELEASE_NAMESPACE}' namespace" >&2 + exit 1 +fi + +RELEASE_REPO=$(oc --config "${RELEASE_KUBECONFIG}" get imagestream "${RELEASE_STREAM}" -o json | jq -r .status.publicDockerImageRepository) +if [ -z "${RELEASE_REPO}" -o "${RELEASE_REPO}" = "null" ]; then + echo "No public repository URL found for ${RELEASE_NAMESPACE}/${RELEASE_STREAM}" >&2 + exit 1 +fi + +RELEASE_TMPDIR=$(mktemp --tmpdir -d "release-${RELEASE_VERSION}-XXXXXXXXXX") +trap "rm -rf ${RELEASE_TMPDIR}" EXIT + +# extract image-references +oc adm release extract --registry-config "${RELEASE_PULLSECRET}" --from "${RELEASE_PULLSPEC}" --file image-references > "${RELEASE_TMPDIR}/image-references" + +# create new image stream from image-references +oc --config "${RELEASE_KUBECONFIG}" apply -f "${RELEASE_TMPDIR}/image-references" +if ! oc --config "${RELEASE_KUBECONFIG}" get imagestream "${RELEASE_VERSION}" 2>/dev/null; then + echo "Expected '${RELEASE_VERSION}' imagestream?" >&2 + exit 1 +fi +rm -f "${RELEASE_TMPDIR}/image-references" + +function wait_for_tag() { + local is + local tag + + is="$1" + tag="$2" + + while true; do + got=$(oc --config "${RELEASE_KUBECONFIG}" get imagestream "${is}" -o json | jq -r '.status.tags[]? | select(.tag == "'"${tag}"'") | .items[0].image') + [ -n "${got}" ] && break + sleep 2 + done +} + +# Tag the extra images into the image stream +for extra in "${EXTRA_IMAGES[@]}"; do + extra_name="${extra%=*}" + extra_pullspec="${extra#*=}" + + oc --config "${RELEASE_KUBECONFIG}" tag "${extra_pullspec}" "${RELEASE_VERSION}:${extra_name}" + wait_for_tag "${RELEASE_VERSION}" "${extra_name}" +done + +# create the new release payload +oc --config "${RELEASE_KUBECONFIG}" adm release new \ + --name "${RELEASE_NAME}" \ + --registry-config "${RELEASE_PULLSECRET}" \ + --from-image-stream "${RELEASE_VERSION}" \ + --reference-mode source \ + --to-image "${RELEASE_REPO}:${RELEASE_NAME}" + +echo "New ${RELEASE_NAME} release payload available to ${RELEASE_REPO}:${RELEASE_NAME}" diff --git a/release/release_config_example.sh b/release/release_config_example.sh new file mode 100644 index 000000000..ab5376f14 --- /dev/null +++ b/release/release_config_example.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# See docs/release-payload.md for more information + +# A namespace and imagestream where the release will be published to +RELEASE_NAMESPACE=kni +RELEASE_STREAM=release + +# A kubeconfig for api.ci.openshift.org +RELEASE_KUBECONFIG=release-kubeconfig + +# Need access to wherever the payload image - and the +# images referenced by the payload - are hosted +RELEASE_PULLSECRET=release-pullsecret + +# The imagestream in $RELEASE_NAMESPACE where kni-installer will be +# published to +INSTALLER_STREAM=installer + +# The git repository and ref (e.g. branch) to build kni-installer from +INSTALLER_GIT_URI=https://github.com/openshift-metalkube/kni-installer.git +INSTALLER_GIT_REF=master diff --git a/run_ci.sh b/run_ci.sh index fe6e09e39..fe3e8557a 100755 --- a/run_ci.sh +++ b/run_ci.sh @@ -91,7 +91,7 @@ if [ -d "/home/notstack/metalkube-ironic-inspector" ] ; then fi # If directories for go projects exist, copy them to where go expects them -for PROJ in facet kni-installer ; do +for PROJ in facet ; do [ ! -d /home/notstack/$PROJ ] && continue # Set origin so that sync_repo_and_patch is rebasing against the correct source diff --git a/utils.sh b/utils.sh index 9de2d92ca..baab58408 100644 --- a/utils.sh +++ b/utils.sh @@ -2,6 +2,22 @@ set -o pipefail +function extract_installer() { + local release_image + local outdir + + release_image="$1" + outdir="$2" + + extract_dir=$(mktemp -d "installer--XXXXXXXXXX") + + echo "${PULL_SECRET}" > "${extract_dir}/pullsecret" + oc adm release extract --registry-config "${extract_dir}/pullsecret" --command=openshift-install --to "${extract_dir}" "${release_image}" + mv "${extract_dir}/openshift-install" "${outdir}" + + rm -rf "${extract_dir}" +} + function generate_assets() { rm -rf assets/generated && mkdir assets/generated for file in $(find assets/templates/ -iname '*.yaml' -type f -printf "%P\n"); do @@ -23,7 +39,7 @@ function create_cluster() { export TF_LOG=DEBUG cp ${assets_dir}/install-config.yaml{,.tmp} - $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir "${assets_dir}" --log-level=debug create manifests + "${assets_dir}/openshift-install" --dir "${assets_dir}" --log-level=debug create manifests # TODO - consider adding NTP server config to install-config.yaml instead if host clock.redhat.com ; then @@ -35,14 +51,14 @@ function create_cluster() { cp -rf assets/generated/*.yaml ${assets_dir}/openshift cp ${assets_dir}/install-config.yaml{.tmp,} - $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir "${assets_dir}" --log-level=debug create cluster + "${assets_dir}/openshift-install" --dir "${assets_dir}" --log-level=debug create cluster } function wait_for_cvo_finish() { local assets_dir assets_dir="$1" - $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir "${assets_dir}" --log-level=debug wait-for install-complete + "${assets_dir}/openshift-install" --dir "${assets_dir}" --log-level=debug wait-for install-complete } function wait_for_json() {