Add libvirt environment

tests/libvirt-environment/.gitignore

@@ -0,0 +1,2 @@
+/ignore/
+/ssh/

tests/libvirt-environment/Dockerfile

@@ -0,0 +1,21 @@
+FROM registry.fedoraproject.org/fedora
+
+WORKDIR /opt/app-root/src
+
+ENV GOPATH=/opt/app-root
+ENV GOBIN=/opt/app-root/bin
+ENV KUBECONFIG=/opt/app-root/src/github.com/openshift/installer/auth/kubeconfig
+
+COPY ./ssh /root/.ssh
+COPY ./scripts_build /scripts
+COPY ./libvirt_config ./libvirt_config
+
+RUN /scripts/build-stage0.sh
+RUN /scripts/build-stage1.sh
+RUN /scripts/build-stage2.sh
+
+COPY ./scripts_command /scripts
+COPY ./scripts_config /root
+
+CMD [ "/scripts/cmd.sh" ]
+

tests/libvirt-environment/Justfile

@@ -0,0 +1,54 @@
+# This file can be used with `just` (https://github.com/casey/just)
+
+TAG = 'smoke-test-installer'
+NET = 'bridge'
+QEMU_IMG_PATH_RHCOS = "/opt/app-root/src/qemu-img/rhcos-qemu.qcow2"
+SSH_KEY_PATH = "/root/.ssh"
+
+dl-rhcos src='' outputfile='rhcos-qemu.qcow2':
+	#!/usr/bin/env bash
+	src={{src}}
+	if [ -z {{src}} ]; then
+	    echo "SRC1: $src"
+	    build=$(curl --silent https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/builds.json | jq -r '.builds[0]')
+	    image=$(curl --silent https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/$build/meta.json | jq -r '.images["qemu"].path')
+	    src="https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/$build/$image"
+	    echo "SRC2: $src"
+	fi
+	mkdir -p ignore
+	pushd ignore
+	curl --show-error --location $src -o .{{outputfile}} --compressed
+	mv .{{outputfile}} {{outputfile}}
+
+build:
+	#!/usr/bin/env bash
+	if [ ! -f ./ssh/id_rsa ]; then
+	    mkdir -p ./ssh
+	    ssh-keygen -t rsa -b 4096 -C "[email protected]" -N '' -f ./ssh/id_rsa
+	fi
+	docker image build -t {{TAG}} .
+
+run repo_owner='openshift' branch='master': build
+	#!/usr/bin/env bash
+	docker container run --name {{TAG}} --net={{NET}} --privileged --rm -d \
+		--env REPO_OWNER={{repo_owner}} --env BRANCH={{branch}} \
+		{{TAG}}
+
+run-attach repo_owner='openshift' branch='master': build
+	#!/usr/bin/env bash
+	docker container run --name {{TAG}} --net={{NET}} --privileged --rm -it \
+		--env REPO_OWNER={{repo_owner}} --env BRANCH={{branch}} \
+		{{TAG}}
+
+exec +args='/bin/bash':
+	docker container exec -it {{TAG}} {{args}}
+
+exec-ready max-wait='10' args='':
+	#!/usr/bin/env bash
+	docker container exec -it {{TAG}} bash -c "/scripts/exec_when_ready.sh {{max-wait}} {{args}}"
+
+stop:
+	docker container stop {{TAG}}
+
+cleanup:
+	docker image rm {{TAG}} $(docker image ls -a | grep '^<none>' | awk '{print $3}')

tests/libvirt-environment/README.md

@@ -0,0 +1,34 @@
+# Fedora libvirt environment
+This Libvirt container is designed to be a general purpose and disposable environment where users are freed from the process of setting up a Libvirt environment to install an `Openshift` cluster.
+
+## Requirements
+* `just` (https://github.com/casey/just)
+* `jq` (https://stedolan.github.io/jq)
+
+## Download images
+`just dl-rhcos`
+
+## Build & Run
+* `just run [repo_owner] [branch]`        - Creates a background running container after the image is built. It must be stopped using `just stop`.
+* `just run-attach [repo_owner] [branch]` - Creates a foreground running container. It will be terminated if the terminal is closed or the process stopped in any way.
+
+`repo_owner` is an optional parameter which specifies the owner of the git repository to clone for the installer.
+`branch` is an optional parameter which specifies the git branch to clone for the installer.
+
+## Stop
+`just stop`
+
+## Enter the container from a different terminal
+`just exec` [args]                        - Defaults to `/bin/bash`
+
+`args` is an optional parameter which contains instructions to execute within the container.
+
+## Execute a command as soon as the cluster is available
+`just exec-ready` [max-wait] [args]        - Executes the specified command once the cluster is available. Defaults to the cluster's `bootstrap` journal follow.
+
+`max-wait` is an optional parameter which specifies the maximum amount of seconds to wait for the cluster to be available. Defaults to 300 seconds.
+`args` is an optional parameter which contains instructions to execute within the container. 
+
+## Status
+  1. The cluster completes but takes *a long time* on my machine, so the smoke tests time out first.
+  2. Wait a long time (30 minutes or so) and execute the smoke tests manually.

tests/libvirt-environment/libvirt_config/libvirt-storage-pool.xml

@@ -0,0 +1,12 @@
+<pool type='dir'>
+  <name>default</name>
+  <target>
+    <path>/var/lib/libvirt/images</path>
+    <permissions>
+      <mode>0777</mode>
+      <owner>0</owner>
+      <group>0</group>
+      <label>system_u:object_r:virt_image_t:s0</label>
+    </permissions>
+  </target>
+</pool>

tests/libvirt-environment/scripts_build/build-stage0.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -xe
+
+dnf update -y
+dnf install -y \
+    w3m \
+    unzip \
+    deltarpm pki-ca \
+    kubernetes-client \
+    git gcc gcc-c++ libtool golang \
+    qemu-img libvirt libvirt-python libvirt-client libvirt-devel @virtualization \
+    dnsmasq kubernetes-client
+
+dnf clean all
+rm -rf /var/cache/dnf/*
+
+curl -OL https://github.com/openshift/origin/releases/download/v3.10.0/openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz
+tar -zxf openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz
+mv -f ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit/oc /usr/local/bin
+
+curl -OL https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip
+unzip terraform_0.11.8_linux_amd64.zip
+mv -f ./terraform /usr/local/bin
+
+rm -rf ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz \
+  ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit \
+  ./terraform_0.11.8_linux_amd64.zip

tests/libvirt-environment/scripts_build/build-stage1.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -xe
+
+# LIBVIRT
+cat <<EOF >> /etc/polkit-1/rules.d/80-libvirt.rules
+polkit.addRule(function(action, subject) {
+  if (action.id == "org.libvirt.unix.manage" && subject.local && subject.active && subject.isInGroup("wheel")) {
+      return polkit.Result.YES;
+  }
+});
+EOF
+
+sed -i 's/#user = "root"/user = "root"/; s/#group = "root"/group = "root"/' /etc/libvirt/qemu.conf
+
+cat <<EOF >>/etc/libvirt/libvirtd.conf
+listen_tls = 0
+listen_tcp = 1
+auth_tcp="none"
+tcp_port = "16509"
+log_level = 4
+EOF
+
+cat <<EOF >>/etc/sysconfig/libvirtd
+LIBVIRTD_ARGS="--listen"
+EOF
+
+libvirtd -d
+virsh --connect qemu:///system pool-create --file=/opt/app-root/src/libvirt_config/libvirt-storage-pool.xml
+
+# TERRAFORM
+cat <<EOF > "${HOME}/.terraformrc"
+plugin_cache_dir = "${HOME}/.terraform.d/plugin-cache"
+EOF

tests/libvirt-environment/scripts_build/build-stage2.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -xe
+
+export REVISION=1
+
+# GIT
+mkdir /opt/app-root/bin
+mkdir /opt/app-root/pkg
+
+GOBIN=~/.terraform.d/plugins go get github.com/dmacvicar/terraform-provider-libvirt

tests/libvirt-environment/scripts_command/cmd.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -x
+
+chmod 700 /root/.ssh
+chmod 600 /root/.ssh/*
+
+# shellcheck disable=SC1091
+source /root/.bash_profile
+
+# DNSMASQ setup
+cat <<EOF > /etc/dnsmasq.conf
+bind-interfaces
+interface=lo
+strict-order
+user=root
+domain-needed
+bogus-priv
+filterwin2k
+localise-queries
+no-negcache
+no-resolv
+$(grep -oE 'nameserver.*' /etc/resolv.conf | sed -E 's/^nameserver (.*)/server=\1/')
+# server=$(ip route get 1.1.1.1 | grep -oE 'via ([^ ]+)' | sed -E 's/via //')
+ server=/tt.testing/192.168.126.1
+EOF
+
+cp /etc/resolv.conf{,.bkp}
+cat <<EOF > /etc/resolv.conf
+nameserver 127.0.0.1
+EOF
+
+dnsmasq
+
+# Start LIBVIRT
+libvirtd -d --listen -f /etc/libvirt/libvirtd.conf
+virtlockd -d
+virtlogd -d
+
+mkdir -p "/opt/app-root/src/github.com/openshift"
+cd "/opt/app-root/src/github.com/openshift" || exit 1
+git clone "https://github.com/${REPO_OWNER}/installer.git" || exit 1
+
+cd "/opt/app-root/src/github.com/openshift/installer" || exit 1
+git checkout "$BRANCH" || exit 1
+./hack/build.sh
+bash -i

tests/libvirt-environment/scripts_command/exec_when_ready.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+start_time=$(date -u +%s)
+sleep_time=2s
+elapsed=0
+MAX_WAIT=300
+BOOTSTRAPIP=''
+EXEC_CMD=''
+
+await_trail=''
+
+eval "$(ssh-agent -s)" > /dev/null || exit 1
+ssh-add "${HOME}/.ssh/id_rsa" > /dev/null 2>&1 || exit 1
+
+# First argument is assigned to MAX_WAIT
+if [[ -n $1 ]]; then
+    MAX_WAIT=$1
+fi
+
+# Second argument is assigned to EXEC_CMD
+if [[ -n $2 ]]; then
+    EXEC_CMD=$2
+fi
+
+while [[ elapsed -lt MAX_WAIT ]]
+do
+    if [[ -n $EXEC_CMD ]]; then
+        await_trail="${await_trail}."
+        if [[ ${#await_trail} -gt 3 ]]; then
+            await_trail=''
+        fi
+        echo -ne "\\rAwaiting cluster availability${await_trail}    \\r"
+    fi
+
+    if [[ -z $BOOTSTRAPIP ]]; then
+
+        TEMPBOOTSTRAPIP=$(virsh --connect qemu+tcp://192.168.122.1/system domifaddr bootstrap 2> /dev/null | awk '/192/{print $4}')
+        if ! [[ -z $TEMPBOOTSTRAPIP ]]; then
+            BOOTSTRAPIP=${TEMPBOOTSTRAPIP::${#TEMPBOOTSTRAPIP}-3}
+        fi
+    fi
+
+    if [[ -n $BOOTSTRAPIP ]]; then
+        msg=$(ssh -oStrictHostKeyChecking=no core@"${BOOTSTRAPIP}" journalctl -n 1 -u bootkube.service -u tectonic 2> /dev/null)
+        if echo "$msg" | grep 'Tectonic installation is done'; then
+            echo ''
+            if ! [[ -z $EXEC_CMD ]]; then
+                bash -c "${EXEC_CMD}"
+                exit 0
+            fi
+            bash -c "ssh -oStrictHostKeyChecking=no core@${BOOTSTRAPIP} journalctl -f -u bootkube -u tectonic"
+            exit 0
+        fi
+        if [[ -z $EXEC_CMD ]]; then
+            echo "${msg}"
+        fi
+    fi
+    elapsed=$(($(date -u +%s) - start_time))
+    sleep $sleep_time
+done
+echo -e "\\nWatch stopped after elapsed time: ${elapsed}"
+exit 1

tests/libvirt-environment/scripts_config/.bash_profile

@@ -0,0 +1,5 @@
+
+export GOBIN='/opt/app-root/bin'
+export KUBECONFIG=/opt/app-root/src/github.com/openshift/installer/auth/kubeconfig
+export PATH="${PATH}:${GOBIN}"
+export TAGS='libvirt'