POC - include kubevirt's virt-operator in a custom release payload

.gitignore

@@ -4,6 +4,12 @@ openshift-install.log
 logs/
 config_*.sh
 !config_example.sh
+
+release/release_config_*.sh
+!release/release_config_example.sh
+release/release-kubeconfig
+release/release-pullsecret
+
 assets/generated
 
 # Conditionally created for appropriate environments

03_ocp_repo_sync.sh

@@ -34,8 +34,6 @@ function sync_go_repo_and_patch {
     popd
 }
 
-sync_go_repo_and_patch github.com/openshift-metalkube/kni-installer https://github.com/openshift-metalkube/kni-installer.git
-
 # Build facet
 # FIXME(russellb) - disabled due to build failure related to metal3 rename
 #sync_go_repo_and_patch github.com/openshift-metalkube/facet https://github.com/openshift-metalkube/facet.git

06_create_cluster.sh

@@ -8,7 +8,7 @@ source common.sh
 source ocp_install_env.sh
 
 # Do some PULL_SECRET sanity checking
-if [[ "${OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE}" == *"registry.svc.ci.openshift.org"* ]]; then
+if [[ "${OPENSHIFT_RELEASE_IMAGE}" == *"registry.svc.ci.openshift.org"* ]]; then
     if [[ "${PULL_SECRET}" != *"registry.svc.ci.openshift.org"* ]]; then
         echo "Please get a valid pull secret for registry.svc.ci.openshift.org."
         exit 1
@@ -18,6 +18,9 @@ fi
 if [ ! -d ocp ]; then
     mkdir -p ocp
 
+    # Extract openshift-install from the release image
+    extract_installer "${OPENSHIFT_RELEASE_IMAGE}" ocp/
+
     # Create a master_nodes.json file
     jq '.nodes[0:3] | {nodes: .}' "${NODES_FILE}" | tee "${MASTER_NODES_FILE}"
 

09_deploy_kubevirt.sh

@@ -16,10 +16,9 @@ eval "$(go env)"
 # These will ultimately be deployed via kni-installer to the bootstrap node
 # where they will then be applied to the cluster, but for now we do it
 # manually
-cd manifests
-for manifest in $(ls -1 *.yaml | sort -h); do
-  oc --as system:admin --config ../ocp/auth/kubeconfig apply -f ${manifest}
-  echo "manifests/${manifest} applied"
+for manifest in $(ls -1 clusteroperators/kubevirt/manifests/*.yaml | sort -h); do
+  oc --as system:admin --config ocp/auth/kubeconfig apply -f ${manifest}
+  echo "${manifest} applied"
 done
 
 export UIPATH="$GOPATH/src/github.com/kubevirt/web-ui-operator"

Makefile

@@ -1,5 +1,5 @@
-.PHONY: default all requirements configure repo_sync ironic build ocp_run deploy_bmo register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell
-default: requirements configure repo_sync ironic build ocp_run deploy_bmo register_hosts bell
+.PHONY: default all requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts clean ocp_cleanup ironic_cleanup host_cleanup bell
+default: requirements configure repo_sync ironic ocp_run deploy_bmo register_hosts bell
 
 all: default
 
@@ -15,9 +15,6 @@ repo_sync:
 ironic:
 	./04_setup_ironic.sh
 
-build:
-	./05_build_ocp_installer.sh
-
 ocp_run:
 	./06_create_cluster.sh
 

README.md

@@ -71,14 +71,10 @@ server and download the resources it requires.
 The Ironic container is stored at https://quay.io/repository/metalkube/metalkube-ironic, built from
 https://github.com/metalkube/metalkube-ironic.
 
-- `./05_build_ocp_installer.sh`
-
-These will pull and build the openshift-install and some other things from
-source.
-
 - `./06_create_cluster.sh`
 
-This will run the kni-installer to generate ignition configs for the
+This will extract openshift-install from the OCP release payload and
+run `openshift-install` to generate ignition configs for the
 bootstrap node and the masters.  The installer then launches both the
 bootstrap VM and master nodes using the Terraform providers for libvirt
 and Ironic.  Once bootstrap is complete, the installer removes the
@@ -133,7 +129,7 @@ e.g. to clean and re-install ocp run:
 ```
 ./ocp_cleanup.sh
 rm -fr ocp
-./05_run_ocp.sh
+./06_create_cluster.sh
 ```
 
 Or, you can run `make clean` which will run all of the cleanup steps.

manifests/100_cnv_kubevirt_op.yaml → clusteroperators/virt-operator/manifests/100_cnv_kubevirt_op.yaml

@@ -484,18 +484,29 @@ spec:
       containers:
       - command:
         - virt-operator
+        - --cluster-operator
         - --port
         - "8443"
         - -v
         - "2"
         env:
+        - name: OPERATOR_VERSION
+          value: "0.0.1-snapshot"
         - name: OPERATOR_IMAGE
-          value: index.docker.io/kubevirt/virt-operator:v0.16.3
+          value: index.docker.io/markbmc/virt-operator:v0.16.3-kni
+        - name: API_IMAGE
+          value: index.docker.io/kubevirt/virt-api:v0.16.3
+        - name: CONTROLLER_IMAGE
+          value: index.docker.io/kubevirt/virt-controller:v0.16.3
+        - name: LAUNCHER_IMAGE
+          value: index.docker.io/kubevirt/virt-launcher:v0.16.3
+        - name: HANDLER_IMAGE
+          value: index.docker.io/kubevirt/virt-handler:v0.16.3
         - name: WATCH_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.annotations['olm.targetNamespaces']
-        image: index.docker.io/kubevirt/virt-operator:v0.16.3
+        image: index.docker.io/markbmc/virt-operator:v0.16.3-kni
         imagePullPolicy: IfNotPresent
         name: virt-operator
         ports:

clusteroperators/virt-operator/manifests/130_clusteroperator.yaml

@@ -0,0 +1,9 @@
+apiVersion: config.openshift.io/v1
+kind: ClusterOperator
+metadata:
+  name: virt-operator
+spec: {}
+status:
+  versions:
+    - name: operator
+      version: "0.0.1-snapshot"

clusteroperators/virt-operator/manifests/image-references

@@ -0,0 +1,48 @@
+kind: ImageStream
+apiVersion: image.openshift.io/v1
+spec:
+  tags:
+  - name: virt-operator
+    from:
+      kind: DockerImage
+      name: index.docker.io/markbmc/virt-operator:v0.16.3-kni
+  - name: virt-api
+    from:
+      kind: DockerImage
+      name: index.docker.io/kubevirt/virt-api:v0.16.3
+  - name: virt-controller
+    from:
+      kind: DockerImage
+      name: index.docker.io/kubevirt/virt-controller:v0.16.3
+  - name: virt-launcher
+    from:
+      kind: DockerImage
+      name: index.docker.io/kubevirt/virt-launcher:v0.16.3
+  - name: virt-handler
+    from:
+      kind: DockerImage
+      name: index.docker.io/kubevirt/virt-handler:v0.16.3
+  - name: cni-plugins
+    from:
+      kind: DockerImage
+      name: quay.io/kubevirt/cni-default-plugins:latest
+  - name: sriov-device-plugin
+    from:
+      kind: DockerImage
+      name: quay.io/booxter/sriov-device-plugin:latest
+  - name: sriov-cni
+    from:
+      kind: DockerImage
+      name: docker.io/nfvpe/sriov-cni:latest
+  - name: node
+    from:
+      kind: DockerImage
+      name: docker.io/openshift/origin-node:latest
+  - name: ovs-cni-plugin
+    from:
+      kind: DockerImage
+      name: quay.io/kubevirt/ovs-cni-plugin:latest
+  - name: ovs-cni-marker
+    from:
+      kind: DockerImage
+      name: quay.io/kubevirt/ovs-cni-marker:latest

common.sh

@@ -47,7 +47,7 @@ export NUM_MASTERS=${NUM_MASTERS:-"3"}
 export NUM_WORKERS=${NUM_WORKERS:-"1"}
 export VM_EXTRADISKS=${VM_EXTRADISKS:-"false"}
 
-export RHCOS_INSTALLER_IMAGE_URL=$(jq -r '.baseURI' $GOPATH/src/github.com/openshift-metalkube/kni-installer/data/data/rhcos.json)
+export RHCOS_INSTALLER_IMAGE_URL="https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/ootpa/410.8.20190412.1/"
 export RHCOS_IMAGE_URL=${RHCOS_IMAGE_URL:-${RHCOS_INSTALLER_IMAGE_URL}}
 
 export RHCOS_IMAGE_FILENAME_OPENSTACK_GZ="$(curl ${RHCOS_IMAGE_URL}/meta.json | jq -r '.images.openstack.path')"

docs/release-payload.md

@@ -0,0 +1,144 @@
+# Publishing a KNI Release Payload
+
+OpenShift publishes a release payload image which includes information
+about cluster operator images and their resource manifests, along with
+references to installer and CLI images. The recommended method for
+obtaining an installer binary is to first choose a release version and
+then use the `oc adm release extract --tools` command to extract the
+installer binary from the release payload.
+
+Since KNI has temporarily forked the installer, we build custom and
+publish custom release payloads that include a reference to the forked
+installer.
+
+## Preparation and Configuration
+
+We build and publish within a namespace on an OpenShift
+cluster. First, prepare a `kubeconfig` with credentials to this
+cluster, and with the desired namespace set as the default:
+
+```
+$ oc --config=release-kubeconfig login https://api.ci.openshift.org --token=...
+$ oc --config=release-kubeconfig new-project kni
+$ oc --config=release-kubeconfig project kni
+$ oc --config=release-kubeconfig adm policy add-role-to-user admin <other admin>
+````
+
+We need a docker registry credentials file which contains credentials
+for the registry on this OpenShift cluster:
+
+```
+$ oc --config=release-kubeconfig registry login --to=release-pullsecret
+```
+
+But also, we need credentials for any registry hosting images
+referenced from release payloads (e.g. ```quay.io```)
+
+```
+$ TOKEN=$((. ../config_$USER.sh && echo $PULL_SECRET) 2>/dev/null | jq -r '.auths["quay.io"].auth' | base64 -d)
+$ podman login --authfile=release-pullsecret -u ${TOKEN%:*} -p ${TOKEN#*:} quay.io
+```
+
+Images are published to imagestream tags, and we need an image stream
+for our installer builds and our custom release payloads:
+
+```
+$ oc --config=release-kubeconfig create imagestream release
+$ oc --config=release-kubeconfig create imagestream installer
+```
+
+We need to create a ```docker-registry``` secret so the image stream
+can import referenced images:
+
+```
+$ oc --config=release-kubeconfig \
+    create secret docker-registry quay-pullsecret \
+    --docker-server=quay.io \
+    --docker-username=${TOKEN%:*} \
+    --docker-password=${TOKEN#*:}
+```
+
+Finally, create a ```release_config_$USER.sh``` file with information
+about all of the above:
+
+```
+$ cat > release_config_$USER.sh <<EOF
+RELEASE_NAMESPACE=kni
+RELEASE_STREAM=release
+INSTALLER_STREAM=installer
+RELEASE_KUBECONFIG=release-kubeconfig
+RELEASE_PULLSECRET=release-pullsecret
+INSTALLER_GIT_URI=https://github.com/openshift-metal3/kni-installer.git
+INSTALLER_GIT_REF=master
+EOF
+```
+
+## Building an Installer and Payload
+
+When we want to move to a newer OpenShift release, we pick a release
+payload:
+
+```
+$ oc adm release info registry.svc.ci.openshift.org/ocp/release:4.0.0-0.ci-2019-04-17-133604 -a release-pullsecret -o json | jq -r .metadata.version
+4.0.0-0.ci-2019-04-17-133604
+```
+
+Next, rebase ```openshift-metal3/kni-installer``` to the
+```openshift/installer``` commit referenced by that payload:
+
+```
+$ oc adm release info -a release-pullsecret -o json \
+    registry.svc.ci.openshift.org/ocp/release:4.0.0-0.ci-2019-04-17-133604 \
+    jq -r '.references.spec.tags[] | select(.name == "installer") | .annotations["io.openshift.build.commit.id"]'
+8c607f66662f8e35570960484612d0589c26b654
+```
+
+And then kick off a build, with the resulting image tagged into the
+installer image stream using the supplied version as the tag:
+
+```
+$ ./build_installer.sh 4.0.0-0.ci-2019-04-17-133604
+```
+
+Now, finally, we can build a new payload referencing our installer,
+and tag it into the release imagestream:
+
+```
+$ ./prep_release.sh \
+    4.0.0-0.ci-2019-04-17-133604-kni \
+    registry.svc.ci.openshift.org/ocp/release:4.0.0-0.ci-2019-04-17-133604 \
+    registry.svc.ci.openshift.org/kni/installer:4.0.0-0.ci-2019-04-17-133604
+```
+
+## Adding additional Cluster Operators
+
+In order to experiment with the possibility of adding additional,
+KNI-specific cluster operators, we first build the new cluster
+operator by taking an existing operator and building a new image layer
+containing the manifests for the release payload, and adding the
+`io.openshift.release.operator = true` label:
+
+```
+$> ./build_clusteroperator.sh virt-operator index.docker.io/markbmc/virt-operator v0.16.3-kni
+```
+
+And then we can include this cluster operator - and the dependencies
+listed in its `image-references` manifest - in our custom release
+payload:
+
+```
+$ ./prep_release.sh \
+    4.0.0-0.ci-2019-04-17-133604-kni-kubevirt \
+    registry.svc.ci.openshift.org/ocp/release:4.0.0-0.ci-2019-04-17-133604 \
+    registry.svc.ci.openshift.org/kni/installer:4.0.0-0.ci-2019-04-17-133604 \
+    virt-operator=registry.svc.ci.openshift.org/kni/virt-operator:v0.16.3-kni
+```
+
+You can test upgrading between these payloads by deploing with the
+first one and then doing:
+
+```
+$ oc --config ocp/auth/kubeconfig adm upgrade \
+    --to-image
+    registry.svc.ci.openshift.org/kni/release:4.0.0-0.ci-2019-04-17-133604-kni-kubevirt
+```

ocp_cleanup.sh

@@ -8,8 +8,8 @@ source ocp_install_env.sh
 sudo systemctl stop fix_certs.timer
 
 if [ -d ocp ]; then
-    $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir ocp --log-level=debug destroy bootstrap
-    $GOPATH/src/github.com/openshift-metalkube/kni-installer/bin/kni-install --dir ocp --log-level=debug destroy cluster
+    ocp/openshift-install --dir ocp --log-level=debug destroy bootstrap
+    ocp/openshift-install --dir ocp --log-level=debug destroy cluster
     rm -rf ocp
 fi
 

ocp_install_env.sh

@@ -13,7 +13,7 @@ export EXTERNAL_SUBNET="192.168.111.0/24"
 # The release we default to here is pinned and known to work with our current
 # version of kni-installer.
 #
-export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="registry.svc.ci.openshift.org/ocp/release:4.0.0-0.ci-2019-04-17-133604"
+export OPENSHIFT_RELEASE_IMAGE="registry.svc.ci.openshift.org/markmc/release:4.0.0-0.ci-2019-04-17-133604-kni"
 
 function generate_ocp_install_config() {
     local outdir