Skip to content

Commit

Permalink
Merge pull request #4948 from medyagh/paralell_integration_tests4
Browse files Browse the repository at this point in the history
refactor integration tests to run in parallel
  • Loading branch information
medyagh authored Aug 2, 2019
2 parents ce98918 + e6079dc commit f872767
Show file tree
Hide file tree
Showing 34 changed files with 1,331 additions and 890 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,5 @@ deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/VERSION
/.idea

/.vscode

test/integration/testdata/minikube-linux-amd64-latest-stable
6 changes: 2 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ require (
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/blang/semver v3.5.0+incompatible
github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
github.com/docker/docker v1.13.1 // indirect
github.com/docker/go-units v0.3.3
github.com/docker/machine v0.16.1-0.20190718054102-a555e4f7a8f5
github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
Expand All @@ -24,9 +24,9 @@ require (
github.com/google/go-github/v25 v25.0.2
github.com/gorilla/mux v1.7.1 // indirect
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
github.com/hashicorp/go-getter v1.3.0
github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
github.com/hashicorp/go-retryablehttp v0.5.4
github.com/hashicorp/go-version v1.1.0 // indirect
github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
Expand All @@ -36,7 +36,6 @@ require (
github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
github.com/mattn/go-colorable v0.1.1 // indirect
github.com/mattn/go-isatty v0.0.5
github.com/mattn/go-runewidth v0.0.0-20161012013512-737072b4e32b // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd
Expand All @@ -62,7 +61,6 @@ require (
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f
golang.org/x/text v0.3.2
gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.6 // indirect
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
k8s.io/api v0.0.0
k8s.io/apimachinery v0.0.0
Expand Down
168 changes: 145 additions & 23 deletions go.sum

Large diffs are not rendered by default.

22 changes: 17 additions & 5 deletions hack/jenkins/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
# EXTRA_START_ARGS: additional flags to pass into minikube start
# EXTRA_ARGS: additional flags to pass into minikube
# JOB_NAME: the name of the logfile and check name to update on github
#
# PARALLEL_COUNT: number of tests to run in parallel


readonly TEST_ROOT="${HOME}/minikube-integration"
Expand Down Expand Up @@ -75,14 +75,15 @@ gsutil -qm cp \

gsutil -qm cp "gs://minikube-builds/${MINIKUBE_LOCATION}/testdata"/* testdata/


# Set the executable bit on the e2e binary and out binary
export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
export E2E_BIN="out/e2e-${OS_ARCH}"
chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}" out/docker-machine-driver-*

procs=$(pgrep "minikube-${OS_ARCH}|e2e-${OS_ARCH}" || true)
if [[ "${procs}" != "" ]]; then
echo "ERROR: found stale test processes to kill:"
echo "Warning: found stale test processes to kill:"
ps -f -p ${procs} || true
kill ${procs} || true
kill -9 ${procs} || true
Expand Down Expand Up @@ -130,8 +131,13 @@ if type -P virsh; then
| awk '{ print $2 }' \
| xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}" \
|| true
# list again after clean up
virsh -c qemu:///system list --all
virsh -c qemu:///system list --all \
| grep Test \
| awk '{ print $2 }' \
| xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}" \
|| true
echo ">> Virsh VM list after clean up (should be empty) :"
virsh -c qemu:///system list --all || true
fi

if type -P vboxmanage; then
Expand All @@ -141,13 +147,19 @@ if type -P vboxmanage; then
| cut -d'"' -f2 \
| xargs -I {} sh -c "vboxmanage startvm {} --type emergencystop; vboxmanage unregistervm {} --delete" \
|| true
vboxmanage list vms \
| grep Test \
| cut -d'"' -f2 \
| xargs -I {} sh -c "vboxmanage startvm {} --type emergencystop; vboxmanage unregistervm {} --delete" \
|| true

# remove inaccessible stale VMs https://github.com/kubernetes/minikube/issues/4872
vboxmanage list vms \
| grep inaccessible \
| cut -d'"' -f3 \
| xargs -I {} sh -c "vboxmanage startvm {} --type emergencystop; vboxmanage unregistervm {} --delete" \
|| true

# list them again after clean up
vboxmanage list vms || true
fi
Expand Down Expand Up @@ -238,7 +250,7 @@ echo ">> Starting ${E2E_BIN} at $(date)"
${SUDO_PREFIX}${E2E_BIN} \
-minikube-start-args="--vm-driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
-minikube-args="--v=10 --logtostderr ${EXTRA_ARGS}" \
-test.v -test.timeout=100m -binary="${MINIKUBE_BIN}" && result=$? || result=$?
-test.v -test.timeout=100m -test.parallel=${PARALLEL_COUNT} -binary="${MINIKUBE_BIN}" && result=$? || result=$?
echo ">> ${E2E_BIN} exited with ${result} at $(date)"
echo ""

Expand Down
1 change: 1 addition & 0 deletions hack/jenkins/linux_integration_tests_kvm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ set -e
OS_ARCH="linux-amd64"
VM_DRIVER="kvm2"
JOB_NAME="Linux-KVM"
PARALLEL_COUNT=4

# Download files and set permissions
source ./common.sh
1 change: 1 addition & 0 deletions hack/jenkins/linux_integration_tests_none.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ OS_ARCH="linux-amd64"
VM_DRIVER="none"
JOB_NAME="Linux-None"
EXTRA_ARGS="--bootstrapper=kubeadm"
PARALLEL_COUNT=1

SUDO_PREFIX="sudo -E "
export KUBECONFIG="/root/.kube/config"
Expand Down
1 change: 1 addition & 0 deletions hack/jenkins/linux_integration_tests_virtualbox.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ set -e
OS_ARCH="linux-amd64"
VM_DRIVER="virtualbox"
JOB_NAME="Linux-VirtualBox"
PARALLEL_COUNT=4

# Download files and set permissions
source ./common.sh
2 changes: 1 addition & 1 deletion hack/jenkins/osx_integration_tests_hyperkit.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ VM_DRIVER="hyperkit"
JOB_NAME="OSX-Hyperkit"
EXTRA_ARGS="--bootstrapper=kubeadm"
EXTRA_START_ARGS=""

PARALLEL_COUNT=3

# Download files and set permissions
source common.sh
1 change: 1 addition & 0 deletions hack/jenkins/osx_integration_tests_virtualbox.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ OS_ARCH="darwin-amd64"
VM_DRIVER="virtualbox"
JOB_NAME="OSX-Virtualbox"
EXTRA_ARGS="--bootstrapper=kubeadm"
PARALLEL_COUNT=3

# Download files and set permissions
source common.sh
7 changes: 6 additions & 1 deletion pkg/util/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,14 @@ func (s *PodStore) Stop() {
}

// GetClient gets the client from config
func GetClient() (kubernetes.Interface, error) {
func GetClient(kubectlContext ...string) (kubernetes.Interface, error) {
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
configOverrides := &clientcmd.ConfigOverrides{}
if kubectlContext != nil {
configOverrides = &clientcmd.ConfigOverrides{
CurrentContext: kubectlContext[0],
}
}
kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
config, err := kubeConfig.ClientConfig()
if err != nil {
Expand Down
90 changes: 90 additions & 0 deletions test/integration/a_download_only_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// a_download_only_test.go filename starts with a, for the purpose that it runs before all parallel tests and downloads the images and caches them.
package integration

import (
"fmt"
"os"
"path/filepath"
"runtime"
"testing"
"time"

"github.com/hashicorp/go-getter"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/minikube/constants"
pkgutil "k8s.io/minikube/pkg/util"
"k8s.io/minikube/test/integration/util"
)

// Note this test runs before all because filename is alphabetically first
// is used to cache images and binaries used by other parallel tests to avoid redownloading.
// TestDownloadOnly tests the --download-only option
func TestDownloadOnly(t *testing.T) {
p := profileName(t)
mk := NewMinikubeRunner(t, p)
if !isTestNoneDriver(t) { // none driver doesnt need to be deleted
defer mk.TearDown(t)
}

t.Run("Oldest", func(t *testing.T) {
stdout, stderr, err := mk.Start("--download-only", fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion))
if err != nil {
t.Errorf("%s minikube --download-only failed : %v\nstdout: %s\nstderr: %s", p, err, stdout, stderr)
}
})

t.Run("Newest", func(t *testing.T) {
stdout, stderr, err := mk.Start("--download-only", fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion))
if err != nil {
t.Errorf("%s minikube --download-only failed : %v\nstdout: %s\nstderr: %s", p, err, stdout, stderr)
}
// TODO: add test to check if files are downloaded
})

t.Run("DownloadLatestRelease", func(t *testing.T) {
dest := filepath.Join(*testdataDir, fmt.Sprintf("minikube-%s-%s-latest-stable", runtime.GOOS, runtime.GOARCH))
err := downloadMinikubeBinary(t, dest, "latest")
if err != nil {
t.Errorf("erorr downloading the latest minikube release %v", err)
}
})
}

// downloadMinikubeBinary downloads the minikube binary from github used by TestVersionUpgrade
// acts as a test setup for TestVersionUpgrade
func downloadMinikubeBinary(t *testing.T, dest string, version string) error {
t.Helper()
// Grab latest release binary
url := pkgutil.GetBinaryDownloadURL(version, runtime.GOOS)
download := func() error {
return getter.GetFile(dest, url)
}

if err := util.RetryX(download, 13*time.Second, 5*time.Minute); err != nil {
return errors.Wrap(err, "Failed to get latest release binary")
}
if runtime.GOOS != "windows" {
if err := os.Chmod(dest, 0700); err != nil {
return err
}
}
return nil
}
Loading

0 comments on commit f872767

Please sign in to comment.