diff --git a/tests/libvirt-environment/.gitignore b/tests/libvirt-environment/.gitignore new file mode 100644 index 00000000000..7b2f73e5c9d --- /dev/null +++ b/tests/libvirt-environment/.gitignore @@ -0,0 +1,2 @@ +/ignore/ +/ssh/ diff --git a/tests/libvirt-environment/Dockerfile b/tests/libvirt-environment/Dockerfile new file mode 100644 index 00000000000..2fad09eb42a --- /dev/null +++ b/tests/libvirt-environment/Dockerfile @@ -0,0 +1,21 @@ +FROM registry.fedoraproject.org/fedora + +WORKDIR /opt/app-root/src + +ENV GOPATH=/opt/app-root +ENV GOBIN=/opt/app-root/bin +ENV KUBECONFIG=/opt/app-root/src/github.com/openshift/installer/auth/kubeconfig + +COPY ./ssh /root/.ssh +COPY ./scripts_build /scripts +COPY ./libvirt_config ./libvirt_config + +RUN /scripts/build-stage0.sh +RUN /scripts/build-stage1.sh +RUN /scripts/build-stage2.sh + +COPY ./scripts_command /scripts +COPY ./scripts_config /root + +CMD [ "/scripts/cmd.sh" ] + diff --git a/tests/libvirt-environment/Justfile b/tests/libvirt-environment/Justfile new file mode 100644 index 00000000000..951ace14759 --- /dev/null +++ b/tests/libvirt-environment/Justfile @@ -0,0 +1,54 @@ +# This file can be used with `just` (https://github.com/casey/just) + +TAG = 'smoke-test-installer' +NET = 'bridge' +QEMU_IMG_PATH_RHCOS = "/opt/app-root/src/qemu-img/rhcos-qemu.qcow2" +SSH_KEY_PATH = "/root/.ssh" + +dl-rhcos src='' outputfile='rhcos-qemu.qcow2': + #!/usr/bin/env bash + src={{src}} + if [ -z {{src}} ]; then + echo "SRC1: $src" + build=$(curl --silent https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/builds.json | jq -r '.builds[0]') + image=$(curl --silent https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/$build/meta.json | jq -r '.images["qemu"].path') + src="https://releases-rhcos.svc.ci.openshift.org/storage/releases/maipo/$build/$image" + echo "SRC2: $src" + fi + mkdir -p ignore + pushd ignore + curl --show-error --location $src -o .{{outputfile}} --compressed + mv .{{outputfile}} {{outputfile}} + +build: + #!/usr/bin/env bash + if [ ! -f ./ssh/id_rsa ]; then + mkdir -p ./ssh + ssh-keygen -t rsa -b 4096 -C "admin@openshiftdemo.org" -N '' -f ./ssh/id_rsa + fi + docker image build -t {{TAG}} . + +run repo_owner='openshift' branch='master': build + #!/usr/bin/env bash + docker container run --name {{TAG}} --net={{NET}} --privileged --rm -d \ + --env REPO_OWNER={{repo_owner}} --env BRANCH={{branch}} \ + {{TAG}} + +run-attach repo_owner='openshift' branch='master': build + #!/usr/bin/env bash + docker container run --name {{TAG}} --net={{NET}} --privileged --rm -it \ + --env REPO_OWNER={{repo_owner}} --env BRANCH={{branch}} \ + {{TAG}} + +exec +args='/bin/bash': + docker container exec -it {{TAG}} {{args}} + +exec-ready max-wait='10' args='': + #!/usr/bin/env bash + docker container exec -it {{TAG}} bash -c "/scripts/exec_when_ready.sh {{max-wait}} {{args}}" + +stop: + docker container stop {{TAG}} + +cleanup: + docker image rm {{TAG}} $(docker image ls -a | grep '^' | awk '{print $3}') diff --git a/tests/libvirt-environment/README.md b/tests/libvirt-environment/README.md new file mode 100644 index 00000000000..4dbec4648be --- /dev/null +++ b/tests/libvirt-environment/README.md @@ -0,0 +1,34 @@ +# Fedora libvirt environment +This Libvirt container is designed to be a general purpose and disposable environment where users are freed from the process of setting up a Libvirt environment to install an `Openshift` cluster. + +## Requirements +* `just` (https://github.com/casey/just) +* `jq` (https://stedolan.github.io/jq) + +## Download images +`just dl-rhcos` + +## Build & Run +* `just run [repo_owner] [branch]` - Creates a background running container after the image is built. It must be stopped using `just stop`. +* `just run-attach [repo_owner] [branch]` - Creates a foreground running container. It will be terminated if the terminal is closed or the process stopped in any way. + +`repo_owner` is an optional parameter which specifies the owner of the git repository to clone for the installer. +`branch` is an optional parameter which specifies the git branch to clone for the installer. + +## Stop +`just stop` + +## Enter the container from a different terminal +`just exec` [args] - Defaults to `/bin/bash` + +`args` is an optional parameter which contains instructions to execute within the container. + +## Execute a command as soon as the cluster is available +`just exec-ready` [max-wait] [args] - Executes the specified command once the cluster is available. Defaults to the cluster's `bootstrap` journal follow. + +`max-wait` is an optional parameter which specifies the maximum amount of seconds to wait for the cluster to be available. Defaults to 300 seconds. +`args` is an optional parameter which contains instructions to execute within the container. + +## Status + 1. The cluster completes but takes *a long time* on my machine, so the smoke tests time out first. + 2. Wait a long time (30 minutes or so) and execute the smoke tests manually. diff --git a/tests/libvirt-environment/libvirt_config/._libvirt-storage-pool.xml b/tests/libvirt-environment/libvirt_config/._libvirt-storage-pool.xml new file mode 100644 index 00000000000..06ebd206fa5 Binary files /dev/null and b/tests/libvirt-environment/libvirt_config/._libvirt-storage-pool.xml differ diff --git a/tests/libvirt-environment/libvirt_config/libvirt-storage-pool.xml b/tests/libvirt-environment/libvirt_config/libvirt-storage-pool.xml new file mode 100644 index 00000000000..6e843196008 --- /dev/null +++ b/tests/libvirt-environment/libvirt_config/libvirt-storage-pool.xml @@ -0,0 +1,12 @@ + + default + + /var/lib/libvirt/images + + 0777 + 0 + 0 + + + + diff --git a/tests/libvirt-environment/scripts_build/build-stage0.sh b/tests/libvirt-environment/scripts_build/build-stage0.sh new file mode 100755 index 00000000000..0b7cb3715bd --- /dev/null +++ b/tests/libvirt-environment/scripts_build/build-stage0.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -xe + +dnf update -y +dnf install -y \ + w3m \ + unzip \ + deltarpm pki-ca \ + kubernetes-client \ + git gcc gcc-c++ libtool golang \ + qemu-img libvirt libvirt-python libvirt-client libvirt-devel @virtualization \ + dnsmasq kubernetes-client + +dnf clean all +rm -rf /var/cache/dnf/* + +curl -OL https://github.com/openshift/origin/releases/download/v3.10.0/openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz +tar -zxf openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz +mv -f ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit/oc /usr/local/bin + +curl -OL https://releases.hashicorp.com/terraform/0.11.8/terraform_0.11.8_linux_amd64.zip +unzip terraform_0.11.8_linux_amd64.zip +mv -f ./terraform /usr/local/bin + +rm -rf ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit.tar.gz \ + ./openshift-origin-client-tools-v3.10.0-dd10d17-linux-64bit \ + ./terraform_0.11.8_linux_amd64.zip diff --git a/tests/libvirt-environment/scripts_build/build-stage1.sh b/tests/libvirt-environment/scripts_build/build-stage1.sh new file mode 100755 index 00000000000..a8cfba6199b --- /dev/null +++ b/tests/libvirt-environment/scripts_build/build-stage1.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -xe + +# LIBVIRT +cat <> /etc/polkit-1/rules.d/80-libvirt.rules +polkit.addRule(function(action, subject) { + if (action.id == "org.libvirt.unix.manage" && subject.local && subject.active && subject.isInGroup("wheel")) { + return polkit.Result.YES; + } +}); +EOF + +sed -i 's/#user = "root"/user = "root"/; s/#group = "root"/group = "root"/' /etc/libvirt/qemu.conf + +cat <>/etc/libvirt/libvirtd.conf +listen_tls = 0 +listen_tcp = 1 +auth_tcp="none" +tcp_port = "16509" +log_level = 4 +EOF + +cat <>/etc/sysconfig/libvirtd +LIBVIRTD_ARGS="--listen" +EOF + +libvirtd -d +virsh --connect qemu:///system pool-create --file=/opt/app-root/src/libvirt_config/libvirt-storage-pool.xml + +# TERRAFORM +cat < "${HOME}/.terraformrc" +plugin_cache_dir = "${HOME}/.terraform.d/plugin-cache" +EOF diff --git a/tests/libvirt-environment/scripts_build/build-stage2.sh b/tests/libvirt-environment/scripts_build/build-stage2.sh new file mode 100755 index 00000000000..cddba6909af --- /dev/null +++ b/tests/libvirt-environment/scripts_build/build-stage2.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -xe + +export REVISION=1 + +# GIT +mkdir /opt/app-root/bin +mkdir /opt/app-root/pkg + +GOBIN=~/.terraform.d/plugins go get github.com/dmacvicar/terraform-provider-libvirt diff --git a/tests/libvirt-environment/scripts_command/cmd.sh b/tests/libvirt-environment/scripts_command/cmd.sh new file mode 100755 index 00000000000..ee7f1fe5b90 --- /dev/null +++ b/tests/libvirt-environment/scripts_command/cmd.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -x + +chmod 700 /root/.ssh +chmod 600 /root/.ssh/* + +# shellcheck disable=SC1091 +source /root/.bash_profile + +# DNSMASQ setup +cat < /etc/dnsmasq.conf +bind-interfaces +interface=lo +strict-order +user=root +domain-needed +bogus-priv +filterwin2k +localise-queries +no-negcache +no-resolv +$(grep -oE 'nameserver.*' /etc/resolv.conf | sed -E 's/^nameserver (.*)/server=\1/') +# server=$(ip route get 1.1.1.1 | grep -oE 'via ([^ ]+)' | sed -E 's/via //') + server=/tt.testing/192.168.126.1 +EOF + +cp /etc/resolv.conf{,.bkp} +cat < /etc/resolv.conf +nameserver 127.0.0.1 +EOF + +dnsmasq + +# Start LIBVIRT +libvirtd -d --listen -f /etc/libvirt/libvirtd.conf +virtlockd -d +virtlogd -d + +mkdir -p "/opt/app-root/src/github.com/openshift" +cd "/opt/app-root/src/github.com/openshift" || exit 1 +git clone "https://github.com/${REPO_OWNER}/installer.git" || exit 1 + +cd "/opt/app-root/src/github.com/openshift/installer" || exit 1 +git checkout "$BRANCH" || exit 1 +./hack/build.sh +bash -i diff --git a/tests/libvirt-environment/scripts_command/exec_when_ready.sh b/tests/libvirt-environment/scripts_command/exec_when_ready.sh new file mode 100755 index 00000000000..10a5ad85827 --- /dev/null +++ b/tests/libvirt-environment/scripts_command/exec_when_ready.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +start_time=$(date -u +%s) +sleep_time=2s +elapsed=0 +MAX_WAIT=300 +BOOTSTRAPIP='' +EXEC_CMD='' + +await_trail='' + +eval "$(ssh-agent -s)" > /dev/null || exit 1 +ssh-add "${HOME}/.ssh/id_rsa" > /dev/null 2>&1 || exit 1 + +# First argument is assigned to MAX_WAIT +if [[ -n $1 ]]; then + MAX_WAIT=$1 +fi + +# Second argument is assigned to EXEC_CMD +if [[ -n $2 ]]; then + EXEC_CMD=$2 +fi + +while [[ elapsed -lt MAX_WAIT ]] +do + if [[ -n $EXEC_CMD ]]; then + await_trail="${await_trail}." + if [[ ${#await_trail} -gt 3 ]]; then + await_trail='' + fi + echo -ne "\\rAwaiting cluster availability${await_trail} \\r" + fi + + if [[ -z $BOOTSTRAPIP ]]; then + + TEMPBOOTSTRAPIP=$(virsh --connect qemu+tcp://192.168.122.1/system domifaddr bootstrap 2> /dev/null | awk '/192/{print $4}') + if ! [[ -z $TEMPBOOTSTRAPIP ]]; then + BOOTSTRAPIP=${TEMPBOOTSTRAPIP::${#TEMPBOOTSTRAPIP}-3} + fi + fi + + if [[ -n $BOOTSTRAPIP ]]; then + msg=$(ssh -oStrictHostKeyChecking=no core@"${BOOTSTRAPIP}" journalctl -n 1 -u bootkube.service -u tectonic 2> /dev/null) + if echo "$msg" | grep 'Tectonic installation is done'; then + echo '' + if ! [[ -z $EXEC_CMD ]]; then + bash -c "${EXEC_CMD}" + exit 0 + fi + bash -c "ssh -oStrictHostKeyChecking=no core@${BOOTSTRAPIP} journalctl -f -u bootkube -u tectonic" + exit 0 + fi + if [[ -z $EXEC_CMD ]]; then + echo "${msg}" + fi + fi + elapsed=$(($(date -u +%s) - start_time)) + sleep $sleep_time +done +echo -e "\\nWatch stopped after elapsed time: ${elapsed}" +exit 1 diff --git a/tests/libvirt-environment/scripts_config/.bash_profile b/tests/libvirt-environment/scripts_config/.bash_profile new file mode 100644 index 00000000000..b5eaa7442c5 --- /dev/null +++ b/tests/libvirt-environment/scripts_config/.bash_profile @@ -0,0 +1,5 @@ + +export GOBIN='/opt/app-root/bin' +export KUBECONFIG=/opt/app-root/src/github.com/openshift/installer/auth/kubeconfig +export PATH="${PATH}:${GOBIN}" +export TAGS='libvirt'