E2E using gce first pass

.travis.yml

@@ -4,4 +4,4 @@ go:
 script:
 - hack/verify-gofmt.sh
 - make build
-- make test
+- make test-unit

Makefile

@@ -41,5 +41,8 @@ image:
 clean:
 	rm -rf _output
 
-test:
+test-unit:
 	./test/run-unit-tests.sh
+
+test-e2e:
+	./test/run-e2e-tests.sh

hack/e2e-gce/gcloud_create_cluster.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Make sure that uuid package is installed"
+
+master_uuid=$(uuid)
+node1_uuid=$(uuid)
+node2_uuid=$(uuid)
+kube_apiserver_port=6443
+kube_version=1.9.4
+
+DESCHEDULER_ROOT=$(dirname "${BASH_SOURCE}")/../../
+E2E_GCE_HOME=$DESCHEDULER_ROOT/hack/e2e-gce
+
+
+create_cluster() {
+	echo "#################### Creating instances ##########################"
+        gcloud compute instances create descheduler-$master_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b
+	# Keeping the --zone here so as to make sure that e2e's can run locally.
+	echo "gcloud compute instances delete descheduler-$master_uuid --zone=us-east1-b --quiet" > $E2E_GCE_HOME/delete_cluster.sh
+
+	gcloud compute instances create descheduler-$node1_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b
+	echo "gcloud compute instances delete descheduler-$node1_uuid --zone=us-east1-b --quiet" >> $E2E_GCE_HOME/delete_cluster.sh
+
+	gcloud compute instances create descheduler-$node2_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b
+	echo "gcloud compute instances delete descheduler-$node2_uuid --zone=us-east1-b --quiet" >> $E2E_GCE_HOME/delete_cluster.sh
+
+	# Delete the firewall port created for master.
+	echo "gcloud compute firewall-rules delete kubeapiserver-$master_uuid --quiet" >> $E2E_GCE_HOME/delete_cluster.sh
+	chmod 755 $E2E_GCE_HOME/delete_cluster.sh
+}
+
+
+generate_kubeadm_instance_files() {
+	# TODO: Check if they have come up. awk $6 contains the state(RUNNING or not).
+	master_public_ip=$(gcloud compute instances list | grep $master_uuid|awk '{print $5}')
+	node1_public_ip=$(gcloud compute instances list | grep $node1_uuid|awk '{print $5}')
+	node2_public_ip=$(gcloud compute instances list | grep $node2_uuid|awk '{print $5}')
+	echo "kubeadm init --kubernetes-version=${kube_version} --apiserver-advertise-address=${master_public_ip}" --skip-preflight-checks --pod-network-cidr=10.96.0.0/12 > $E2E_GCE_HOME/kubeadm_install.sh
+}
+
+
+transfer_install_files() {
+	gcloud compute scp  $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$master_uuid:/tmp --zone=us-east1-b
+	gcloud compute scp $E2E_GCE_HOME/kubeadm_install.sh descheduler-$master_uuid:/tmp --zone=us-east1-b
+	gcloud compute scp  $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$node1_uuid:/tmp --zone=us-east1-b
+	gcloud compute scp  $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$node2_uuid:/tmp --zone=us-east1-b
+}
+
+
+install_kube() {
+	# Docker installation.
+	gcloud compute ssh descheduler-$master_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b
+	gcloud compute ssh descheduler-$node1_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b
+	gcloud compute ssh descheduler-$node2_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b
+	# kubeadm installation.
+	# 1. Transfer files to master, nodes.
+	transfer_install_files
+	# 2. Install kubeadm.
+	#TODO: Add rm /tmp/kubeadm_install.sh
+	# Open port for kube API server	
+	gcloud compute firewall-rules create kubeapiserver-$master_uuid --allow tcp:6443 --source-tags=descheduler-$master_uuid  --source-ranges=0.0.0.0/0 --description="Opening api server port" 
+
+	gcloud compute ssh descheduler-$master_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b
+	kubeadm_join_command=$(gcloud compute ssh descheduler-$master_uuid --command "sudo chmod 755 /tmp/kubeadm_install.sh; sudo /tmp/kubeadm_install.sh" --zone=us-east1-b|grep 'kubeadm join')
+
+	# Copy the kubeconfig file onto /tmp for e2e tests. 
+	gcloud compute ssh descheduler-$master_uuid --command "sudo cp /etc/kubernetes/admin.conf /tmp; sudo chmod 777 /tmp/admin.conf" --zone=us-east1-b
+	gcloud compute scp descheduler-$master_uuid:/tmp/admin.conf /tmp/admin.conf --zone=us-east1-b
+
+	# Postinstall on master, need to add a network plugin for kube-dns to come to running state.
+	gcloud compute ssh descheduler-$master_uuid --command "sudo kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml --kubeconfig /etc/kubernetes/admin.conf" --zone=us-east1-b
+	echo $kubeadm_join_command > $E2E_GCE_HOME/kubeadm_join.sh
+
+	# Copy kubeadm_join to every node. 
+	#TODO: Put these in a loop, so that extension becomes possible.	
+	gcloud compute ssh descheduler-$node1_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b
+	gcloud compute scp $E2E_GCE_HOME/kubeadm_join.sh descheduler-$node1_uuid:/tmp --zone=us-east1-b
+	gcloud compute ssh descheduler-$node1_uuid --command "sudo chmod 755 /tmp/kubeadm_join.sh; sudo /tmp/kubeadm_join.sh" --zone=us-east1-b
+
+	gcloud compute ssh descheduler-$node2_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b
+	gcloud compute scp $E2E_GCE_HOME/kubeadm_join.sh descheduler-$node2_uuid:/tmp --zone=us-east1-b
+	gcloud compute ssh descheduler-$node2_uuid --command "sudo chmod 755 /tmp/kubeadm_join.sh; sudo /tmp/kubeadm_join.sh" --zone=us-east1-b
+
+}
+
+
+create_cluster
+
+generate_kubeadm_instance_files
+
+install_kube

hack/e2e-gce/gcloud_sdk_configure.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+
+gcloud auth activate-service-account --key-file "${GCE_SA_CREDS}"
+gcloud config set project $GCE_PROJECT_ID
+gcloud config set compute/zone $GCE_ZONE

hack/e2e-gce/install_gcloud.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-176.0.0-linux-x86_64.tar.gz
+
+tar -xvzf google-cloud-sdk-176.0.0-linux-x86_64.tar.gz
+
+./google-cloud-sdk/install.sh -q

hack/e2e-gce/kubeadm_preinstall.sh

@@ -0,0 +1,11 @@
+apt-get update
+apt-get install -y docker.io
+
+apt-get update && apt-get install -y apt-transport-https
+curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF
+apt-get update
+apt-get install -y kubelet kubeadm kubectl
+exit 0

test/e2e/e2e_test.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"github.com/golang/glog"
+	"testing"
+	"time"
+
+	"github.com/kubernetes-incubator/descheduler/cmd/descheduler/app/options"
+	deschedulerapi "github.com/kubernetes-incubator/descheduler/pkg/api"
+	"github.com/kubernetes-incubator/descheduler/pkg/descheduler/client"
+	eutils "github.com/kubernetes-incubator/descheduler/pkg/descheduler/evictions/utils"
+	nodeutil "github.com/kubernetes-incubator/descheduler/pkg/descheduler/node"
+	podutil "github.com/kubernetes-incubator/descheduler/pkg/descheduler/pod"
+	"github.com/kubernetes-incubator/descheduler/pkg/descheduler/strategies"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/api/testapi"
+)
+
+func MakePodSpec() v1.PodSpec {
+	return v1.PodSpec{
+		Containers: []v1.Container{{
+			Name:  "pause",
+			Image: "kubernetes/pause",
+			Ports: []v1.ContainerPort{{ContainerPort: 80}},
+			Resources: v1.ResourceRequirements{
+				Limits: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("100m"),
+					v1.ResourceMemory: resource.MustParse("500Mi"),
+				},
+				Requests: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("100m"),
+					v1.ResourceMemory: resource.MustParse("500Mi"),
+				},
+			},
+		}},
+	}
+}
+
+// RcByNameContainer returns a ReplicationControoler with specified name and container
+func RcByNameContainer(name string, replicas int32, labels map[string]string, gracePeriod *int64) *v1.ReplicationController {
+
+	zeroGracePeriod := int64(0)
+
+	// Add "name": name to the labels, overwriting if it exists.
+	labels["name"] = name
+	if gracePeriod == nil {
+		gracePeriod = &zeroGracePeriod
+	}
+	return &v1.ReplicationController{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ReplicationController",
+			APIVersion: testapi.Groups[v1.GroupName].GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.ReplicationControllerSpec{
+			Replicas: func(i int32) *int32 { return &i }(replicas),
+			Selector: map[string]string{
+				"name": name,
+			},
+			Template: &v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: MakePodSpec(),
+			},
+		},
+	}
+}
+
+// startEndToEndForLowNodeUtilization tests the lownode utilization strategy.
+func startEndToEndForLowNodeUtilization(clientset clientset.Interface) {
+	var thresholds = make(deschedulerapi.ResourceThresholds)
+	var targetThresholds = make(deschedulerapi.ResourceThresholds)
+	thresholds[v1.ResourceMemory] = 20
+	thresholds[v1.ResourcePods] = 20
+	thresholds[v1.ResourceCPU] = 85
+	targetThresholds[v1.ResourceMemory] = 20
+	targetThresholds[v1.ResourcePods] = 20
+	targetThresholds[v1.ResourceCPU] = 90
+	// Run descheduler.
+	evictionPolicyGroupVersion, err := eutils.SupportEviction(clientset)
+	if err != nil || len(evictionPolicyGroupVersion) == 0 {
+		glog.Fatalf("%v", err)
+	}
+	stopChannel := make(chan struct{})
+	nodes, err := nodeutil.ReadyNodes(clientset, "", stopChannel)
+	if err != nil {
+		glog.Fatalf("%v", err)
+	}
+	nodeUtilizationThresholds := deschedulerapi.NodeResourceUtilizationThresholds{Thresholds: thresholds, TargetThresholds: targetThresholds}
+	nodeUtilizationStrategyParams := deschedulerapi.StrategyParameters{NodeResourceUtilizationThresholds: nodeUtilizationThresholds}
+	lowNodeUtilizationStrategy := deschedulerapi.DeschedulerStrategy{Enabled: true, Params: nodeUtilizationStrategyParams}
+	ds := &options.DeschedulerServer{Client: clientset}
+	strategies.LowNodeUtilization(ds, lowNodeUtilizationStrategy, evictionPolicyGroupVersion, nodes)
+	time.Sleep(10 * time.Second)
+
+	return
+}
+
+func TestE2E(t *testing.T) {
+	// If we have reached here, it means cluster would have been already setup and the kubeconfig file should
+	// be in /tmp directory.
+	clientSet, err := client.CreateClient("/tmp/admin.conf")
+	if err != nil {
+		t.Errorf("Error during client creation with %v", err)
+	}
+	nodeList, err := clientSet.Core().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		t.Errorf("Error listing node with %v", err)
+	}
+	// Assumption: We would have 3 node cluster by now. Kubeadm brings all the master components onto master node.
+	// So, the last node would have least utilization.
+	leastLoadedNode := nodeList.Items[2]
+	rc := RcByNameContainer("test-rc", int32(15), map[string]string{"test": "app"}, nil)
+	_, err = clientSet.CoreV1().ReplicationControllers("default").Create(rc)
+	if err != nil {
+		t.Errorf("Error creating deployment %v", err)
+	}
+	podsOnleastUtilizedNode, err := podutil.ListPodsOnANode(clientSet, &leastLoadedNode)
+	if err != nil {
+		t.Errorf("Error listing pods on a node %v", err)
+	}
+	podsBefore := len(podsOnleastUtilizedNode)
+	t.Log("Eviction of pods starting")
+	startEndToEndForLowNodeUtilization(clientSet)
+	podsOnleastUtilizedNode, err = podutil.ListPodsOnANode(clientSet, &leastLoadedNode)
+	if err != nil {
+		t.Errorf("Error listing pods on a node %v", err)
+	}
+	podsAfter := len(podsOnleastUtilizedNode)
+	if podsBefore > podsAfter {
+		t.Fatalf("We should have see more pods on this node as per kubeadm's way of installing %v, %v", podsBefore, podsAfter)
+	}
+}

test/run-e2e-tests.sh

@@ -0,0 +1,19 @@
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+# This just run e2e tests.
+go test github.com/kubernetes-incubator/descheduler/test/e2e/ -v
+

test/run-unit-tests.sh

@@ -14,6 +14,6 @@
 
 #!/bin/bash
 
-# run unit tests
-go test $(go list github.com/kubernetes-incubator/descheduler/... | grep -v github.com/kubernetes-incubator/descheduler/vendor/)
+# This just run unit-tests. Ignoring the current directory so as to avoid running e2e tests.
+go test $(go list github.com/kubernetes-incubator/descheduler/... | grep -v github.com/kubernetes-incubator/descheduler/vendor/| grep -v github.com/kubernetes-incubator/descheduler/test/)