diff --git a/.gitignore b/.gitignore index 19dd29f25..6ed8c9ca8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,7 @@ build/operator/_test build/_output # Temporary local cluster files -cluster/.kubeconfig -cluster/.kubectl +_kubevirtci/ # Temporary test files deploy/test/ diff --git a/Makefile b/Makefile index 69ef57dc4..781b74a4f 100644 --- a/Makefile +++ b/Makefile @@ -108,13 +108,7 @@ cluster-operator-install: ./cluster/operator-install.sh $(E2E_SUITES): $(OPERATOR_SDK) - $(OPERATOR_SDK) test \ - local \ - ./$@ \ - --namespace cluster-network-addons \ - --no-setup \ - --kubeconfig ./cluster/.kubeconfig \ - --go-test-flags "$(E2E_TEST_ARGS)" + OPERATOR_SDK=$(OPERATOR_SDK) TEST_SUITE=$@ TEST_ARGS="$(E2E_TEST_ARGS)" ./hack/functest.sh cluster-clean: ./cluster/clean.sh diff --git a/automation/check-patch.packages b/automation/check-patch.packages index 41c876eb7..78e544280 100644 --- a/automation/check-patch.packages +++ b/automation/check-patch.packages @@ -1,2 +1,3 @@ docker +git make diff --git a/cluster/clean.sh b/cluster/clean.sh index bf15b8c91..5c5b02a81 100755 --- a/cluster/clean.sh +++ b/cluster/clean.sh @@ -1,11 +1,23 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex echo 'Cleaning up ...' ./cluster/kubectl.sh delete --ignore-not-found -f _out/cluster-network-addons/${VERSION}/namespace.yaml ./cluster/kubectl.sh delete --ignore-not-found -f _out/cluster-network-addons/${VERSION}/network-addons-config.crd.yaml ./cluster/kubectl.sh delete --ignore-not-found -f _out/cluster-network-addons/${VERSION}/operator.yaml - -sleep 2 - -echo 'Done' diff --git a/cluster/cli.sh b/cluster/cli.sh index 4f4bb2652..89cd949ca 100755 --- a/cluster/cli.sh +++ b/cluster/cli.sh @@ -1,11 +1,20 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. +source ./cluster/kubevirtci.sh +kubevirtci::install -source ${SCRIPT_ROOT}/cluster/gocli.sh - -if [[ -t 1 ]]; then - $gocli_interactive "$@" -else - $gocli "$@" -fi +$(kubevirtci::path)/cluster-up/cli.sh "$@" diff --git a/cluster/down.sh b/cluster/down.sh index 39f47a245..de02e7cab 100755 --- a/cluster/down.sh +++ b/cluster/down.sh @@ -1,13 +1,22 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -source ./cluster/gocli.sh +set -ex -echo 'Bringing down cluster and client ...' -$gocli rm +source ./cluster/kubevirtci.sh +kubevirtci::install -# clean up unused docker volumes -dangling_volumes="$(docker volume ls -qf dangling=true)" -if [[ $? == 0 && -n "$dangling_volumes" ]]; then - echo 'Cleaning up docker and dangling volumes ...' - docker volume rm $dangling_volumes -fi +$(kubevirtci::path)/cluster-up/down.sh diff --git a/cluster/gocli.sh b/cluster/gocli.sh deleted file mode 100755 index 9c375b8f5..000000000 --- a/cluster/gocli.sh +++ /dev/null @@ -1,3 +0,0 @@ -gocli_image="kubevirtci/gocli@sha256:2ff1e9cddfa2cfdf268301a52d1a5ec252ace6908196609e001844e5458b746a" -gocli="docker run --net=host --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock $gocli_image" -gocli_interactive="docker run --net=host --privileged --rm -it -v /var/run/docker.sock:/var/run/docker.sock $gocli_image" diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh index dd7d915d1..3ae099467 100755 --- a/cluster/kubectl.sh +++ b/cluster/kubectl.sh @@ -1,8 +1,20 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -function _kubectl() { - export KUBECONFIG=./cluster/.kubeconfig - ./cluster/.kubectl "$@" -} +source ./cluster/kubevirtci.sh +kubevirtci::install -_kubectl "$@" +$(kubevirtci::path)/cluster-up/kubectl.sh "$@" diff --git a/cluster/kubevirtci.sh b/cluster/kubevirtci.sh new file mode 100644 index 000000000..2ba5b18e3 --- /dev/null +++ b/cluster/kubevirtci.sh @@ -0,0 +1,36 @@ +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-'k8s-1.14.6'} + +KUBEVIRTCI_VERSION='0e5b027098796137a9b95aed57943061e185bfcd' +KUBEVIRTCI_PATH="${PWD}/_kubevirtci" + +function kubevirtci::install() { + if [ ! -d ${KUBEVIRTCI_PATH} ]; then + git clone https://github.com/kubevirt/kubevirtci.git ${KUBEVIRTCI_PATH} + ( + cd ${KUBEVIRTCI_PATH} + git checkout ${KUBEVIRTCI_VERSION} + ) + fi +} + +function kubevirtci::path() { + echo -n ${KUBEVIRTCI_PATH} +} + +function kubevirtci::kubeconfig() { + echo -n ${KUBEVIRTCI_PATH}/_ci-configs/${KUBEVIRT_PROVIDER}/.kubeconfig +} diff --git a/cluster/operator-install.sh b/cluster/operator-install.sh index 15dcbbc5b..6ebeb57a8 100755 --- a/cluster/operator-install.sh +++ b/cluster/operator-install.sh @@ -1,4 +1,20 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex ./cluster/kubectl.sh create -f _out/cluster-network-addons/${VERSION}/operator.yaml ./cluster/kubectl.sh -n cluster-network-addons wait deployment cluster-network-addons-operator --for condition=Available --timeout=600s diff --git a/cluster/operator-push.sh b/cluster/operator-push.sh index cf6e222ab..c6478caa9 100755 --- a/cluster/operator-push.sh +++ b/cluster/operator-push.sh @@ -1,4 +1,20 @@ -#!/bin/bash -e +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex registry_port=$(./cluster/cli.sh ports registry | tr -d '\r') registry=localhost:$registry_port diff --git a/cluster/up.sh b/cluster/up.sh index 45a6a9327..d1d57b906 100755 --- a/cluster/up.sh +++ b/cluster/up.sh @@ -1,77 +1,44 @@ -#!/bin/bash -e - -KUBERNETES_IMAGE="k8s-1.13.3@sha256:afbdd9b4208e5ce2ec327f302c336cea3ed3c22488603eab63b92c3bfd36d6cd" -OPENSHIFT_IMAGE="os-3.11.0-crio@sha256:3f11a6f437fcdf2d70de4fcc31e0383656f994d0d05f9a83face114ea7254bc0" - -CLUSTER_PROVIDER=${CLUSTER_PROVIDER:-k8s-1.13.3} -CLUSTER_MEMORY_SIZE=${CLUSTER_MEMORY_SIZE:-5120M} -CLUSTER_NUM_NODES=${CLUSTER_NUM_NODES:-1} - -SECONDARY_NICS_NUM=${SECONDARY_NICS_NUM:-1} - -if ! [[ $CLUSTER_NUM_NODES =~ '^-?[0-9]+$' ]] || [[ $CLUSTER_NUM_NODES -lt 1 ]] ; then - CLUSTER_NUM_NODES=1 +#!/bin/bash +# +# Copyright 2018-2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +source ./cluster/kubevirtci.sh +kubevirtci::install + +$(kubevirtci::path)/cluster-up/up.sh + +if [[ "$KUBEVIRT_PROVIDER" =~ k8s- ]]; then + echo 'Install NetworkManager on nodes' + for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do + ./cluster/cli.sh ssh ${node} sudo -- yum install -y yum-plugin-copr + ./cluster/cli.sh ssh ${node} sudo -- yum copr enable -y networkmanager/NetworkManager-1.20 + ./cluster/cli.sh ssh ${node} sudo -- yum install -y NetworkManager NetworkManager-ovs + ./cluster/cli.sh ssh ${node} sudo -- systemctl daemon-reload + ./cluster/cli.sh ssh ${node} sudo -- systemctl restart NetworkManager + echo "Check NetworkManager is working fine on node $node" + ./cluster/cli.sh ssh ${node} -- nmcli device show > /dev/null + done fi -case "${CLUSTER_PROVIDER}" in - 'k8s-1.13.3') - image=$KUBERNETES_IMAGE - ;; - 'os-3.11.0') - image=$OPENSHIFT_IMAGE - ;; -esac - -CREATE_SECONDARY_NICS="" -for i in $(seq 1 ${SECONDARY_NICS_NUM}); do - CREATE_SECONDARY_NICS+=" -device virtio-net-pci,netdev=secondarynet$i -netdev tap,id=secondarynet$i,ifname=stap$i,script=no,downscript=no" -done - -echo "Install cluster from image: ${image}" -if [[ $image == $KUBERNETES_IMAGE ]]; then - # Run Kubernetes cluster image - ./cluster/cli.sh run --random-ports --nodes ${CLUSTER_NUM_NODES} --memory ${CLUSTER_MEMORY_SIZE} --background --qemu-args "'${CREATE_SECONDARY_NICS}'" kubevirtci/${image} - - # Copy kubectl tool and configuration file - ./cluster/cli.sh scp /usr/bin/kubectl - > ./cluster/.kubectl - chmod u+x ./cluster/.kubectl - ./cluster/cli.sh scp /etc/kubernetes/admin.conf - > ./cluster/.kubeconfig - - # Configure insecure access to Kubernetes cluster - cluster_port=$(./cluster/cli.sh ports k8s | tr -d '\r') - ./cluster/kubectl.sh config set-cluster kubernetes --server=https://127.0.0.1:$cluster_port - ./cluster/kubectl.sh config set-cluster kubernetes --insecure-skip-tls-verify=true -elif [[ $image == $OPENSHIFT_IMAGE ]]; then - # If on a developer setup, expose ocp on 8443, so that the openshift web console can be used (the port is important because of auth redirects) - if [ -z "${JOB_NAME}" ]; then - CLUSTER_PROVIDER_EXTRA_ARGS="${CLUSTER_PROVIDER_EXTRA_ARGS} --ocp-port 8443" - fi - - # Run OpenShift cluster image - ./cluster/cli.sh run --random-ports --reverse --nodes ${CLUSTER_NUM_NODES} --memory ${CLUSTER_MEMORY_SIZE} --background --qemu-args "'${CREATE_SECONDARY_NICS}'" kubevirtci/${image} ${CLUSTER_PROVIDER_EXTRA_ARGS} - ./cluster/cli.sh scp /etc/origin/master/admin.kubeconfig - > ./cluster/.kubeconfig - ./cluster/cli.sh ssh node01 -- sudo cp /etc/origin/master/admin.kubeconfig ~vagrant/ - ./cluster/cli.sh ssh node01 -- sudo chown vagrant:vagrant ~vagrant/admin.kubeconfig - - # Copy oc tool and configuration file - ./cluster/cli.sh scp /usr/bin/oc - > ./cluster/.kubectl - chmod u+x ./cluster/.kubectl - ./cluster/cli.sh scp /etc/origin/master/admin.kubeconfig - > ./cluster/.kubeconfig - - # Update Kube config to support unsecured connection - cluster_port=$(./cluster/cli.sh ports ocp | tr -d '\r') - ./cluster/kubectl.sh config set-cluster node01:8443 --server=https://127.0.0.1:$cluster_port - ./cluster/kubectl.sh config set-cluster node01:8443 --insecure-skip-tls-verify=true +if [[ "$KUBEVIRT_PROVIDER" =~ k8s- ]]; then + echo 'Installing Open vSwitch on nodes' + for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do + ./cluster/cli.sh ssh ${node} -- sudo yum install -y http://cbs.centos.org/kojifiles/packages/openvswitch/2.9.2/1.el7/x86_64/openvswitch-2.9.2-1.el7.x86_64.rpm http://cbs.centos.org/kojifiles/packages/openvswitch/2.9.2/1.el7/x86_64/openvswitch-devel-2.9.2-1.el7.x86_64.rpm http://cbs.centos.org/kojifiles/packages/dpdk/17.11/3.el7/x86_64/dpdk-17.11-3.el7.x86_64.rpm + ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload + ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch + done fi - -echo 'Wait until all nodes are ready' -until [[ $(./cluster/kubectl.sh get nodes --no-headers | wc -l) -eq $(./cluster/kubectl.sh get nodes --no-headers | grep ' Ready' | wc -l) ]]; do - sleep 1 -done - -echo 'Install NetworkManager on nodes' -for i in $(seq 1 ${KUBEVIRT_NUM_NODES}); do - ./cluster/cli.sh ssh "node$(printf "%02d" ${i})" -- sudo yum install -y NetworkManager - ./cluster/cli.sh ssh "node$(printf "%02d" ${i})" -- sudo systemctl daemon-reload - ./cluster/cli.sh ssh "node$(printf "%02d" ${i})" -- sudo systemctl restart NetworkManager -done diff --git a/hack/functest.sh b/hack/functest.sh new file mode 100755 index 000000000..08060ccb5 --- /dev/null +++ b/hack/functest.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2017-2019 Red Hat, Inc. + +set -ex + +source ./cluster/kubevirtci.sh + +${OPERATOR_SDK} test \ + local \ + ./${TEST_SUITE} \ + --namespace cluster-network-addons \ + --no-setup \ + --kubeconfig $(kubevirtci::kubeconfig) \ + --go-test-flags "${TEST_ARGS}"