Merge pull request #272 from neiljerram/fix-tests-for-1.10

Bump to k8s 1.10.4 and fix FVs to work with that

Makefile

@@ -43,7 +43,7 @@ OS?=$(shell uname -s | tr A-Z a-z)
 ###############################################################################
 GO_BUILD_VER ?= v0.15
 
-K8S_VERSION=v1.8.1
+K8S_VERSION?=v1.10.4
 HYPERKUBE_IMAGE?=gcr.io/google_containers/hyperkube-$(ARCH):$(K8S_VERSION)
 ETCD_VERSION=v3.2.5
 ETCD_IMAGE ?= quay.io/coreos/etcd:$(ETCD_VERSION)-$(BUILDARCH)
@@ -212,7 +212,7 @@ ut: vendor
 GINKGO_FOCUS?=.*
 fv: tests/fv/fv.test image
 	@echo Running Go FVs.
-	cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) CONTAINER_NAME=$(CONTAINER_NAME):latest-$(ARCH) ./fv.test -ginkgo.slowSpecThreshold 30 -ginkgo.focus $(GINKGO_FOCUS)
+	cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) CONTAINER_NAME=$(CONTAINER_NAME):latest-$(ARCH) PRIVATE_KEY=`pwd`/private.key ./fv.test -ginkgo.slowSpecThreshold 30 -ginkgo.focus $(GINKGO_FOCUS)
 
 tests/fv/fv.test: $(shell find ./tests -type f -name '*.go' -print)
 	# We pre-build the test binary so that we can run it outside a container and allow it
@@ -345,4 +345,4 @@ help: # Some kind of magic from https://gist.github.com/rcmachado/af3db315e31383
 	}                                                                   \
 	{ helpMsg = $$0 }'                                                  \
 	width=20                                                            \
-	$(MAKEFILE_LIST)
+	$(MAKEFILE_LIST)

tests/fv/fv_test.go

@@ -44,11 +44,12 @@ import (
 
 var _ = Describe("kube-controllers FV tests", func() {
 	var (
-		etcd             *containers.Container
-		policyController *containers.Container
-		apiserver        *containers.Container
-		calicoClient     client.Interface
-		k8sClient        *kubernetes.Clientset
+		etcd              *containers.Container
+		policyController  *containers.Container
+		apiserver         *containers.Container
+		calicoClient      client.Interface
+		k8sClient         *kubernetes.Clientset
+		controllerManager *containers.Container
 	)
 
 	const kNodeName = "k8snodename"
@@ -79,12 +80,20 @@ var _ = Describe("kube-controllers FV tests", func() {
 			_, err := k8sClient.CoreV1().Namespaces().List(metav1.ListOptions{})
 			return err
 		}, 15*time.Second, 500*time.Millisecond).Should(BeNil())
+
+		// Run controller manager.  Empirically it can take around 10s until the
+		// controller manager is ready to create default service accounts, even
+		// when the hyperkube image has already been downloaded to run the API
+		// server.  We use Eventually to allow for possible delay when doing
+		// initial pod creation below.
+		controllerManager = testutils.RunK8sControllerManager(apiserver.IP)
 	})
 
 	AfterEach(func() {
-		etcd.Stop()
+		controllerManager.Stop()
 		policyController.Stop()
 		apiserver.Stop()
+		etcd.Stop()
 	})
 
 	It("should initialize the datastore at start-of-day", func() {
@@ -675,8 +684,10 @@ var _ = Describe("kube-controllers FV tests", func() {
 			}
 
 			By("creating a Pod in the k8s API", func() {
-				_, err := k8sClient.CoreV1().Pods("default").Create(&pod)
-				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() error {
+					_, err := k8sClient.CoreV1().Pods("default").Create(&pod)
+					return err
+				}, "20s", "2s").ShouldNot(HaveOccurred())
 			})
 
 			By("updating the pod's status to be running", func() {
@@ -713,8 +724,11 @@ var _ = Describe("kube-controllers FV tests", func() {
 			By("updating the pod's labels to trigger a cache update", func() {
 				// Definitively trigger a pod controller cache update by updating the pod's labels
 				// in the Kubernetes API. This ensures the controller has the cached WEP with container-id-1.
+				podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				pod = *podNow
 				pod.Labels["foo"] = "label2"
-				_, err := k8sClient.CoreV1().Pods("default").Update(&pod)
+				_, err = k8sClient.CoreV1().Pods("default").Update(&pod)
 				Expect(err).NotTo(HaveOccurred())
 			})
 
@@ -756,8 +770,11 @@ var _ = Describe("kube-controllers FV tests", func() {
 
 			By("updating the pod's labels a second time to trigger a datastore sync", func() {
 				// Trigger a pod 'update' in the pod controller by updating the pod's labels.
+				podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				pod = *podNow
 				pod.Labels["foo"] = "label3"
-				_, err := k8sClient.CoreV1().Pods(podNamespace).Update(&pod)
+				_, err = k8sClient.CoreV1().Pods(podNamespace).Update(&pod)
 				Expect(err).NotTo(HaveOccurred())
 			})
 
@@ -806,13 +823,18 @@ var _ = Describe("kube-controllers FV tests", func() {
 		}
 
 		By("creating a Pod in the k8s API", func() {
-			_, err := k8sClient.CoreV1().Pods("default").Create(&pod)
-			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() error {
+				_, err := k8sClient.CoreV1().Pods("default").Create(&pod)
+				return err
+			}, "20s", "2s").ShouldNot(HaveOccurred())
 		})
 
 		By("updating that pod's labels", func() {
+			podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			pod = *podNow
 			pod.Labels["foo"] = "label2"
-			_, err := k8sClient.CoreV1().Pods("default").Update(&pod)
+			_, err = k8sClient.CoreV1().Pods("default").Update(&pod)
 			Expect(err).NotTo(HaveOccurred())
 		})
 

tests/fv/private.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAwBXj4nTY7UhlAnwMZUisHxOIEkeZjeRfX/efdlT+0/9K0MUy
+++hm6N526CqRmm75ynlbkBx1qtVgKbID5gK+v3wyALS0J9oZRYdx1lWN6S5Ix4Rp
+vU00NBVkbJLRGfUrJRFgpALgUeCy3LC9r+9ZAVucikFmbBSYS6tcej2OALXuV7DI
+PYr7PvJ+JVNGVw/5wNW5FNoGKxhj10Lb4uAC/0YB5p70JM/x7vAbA9PaPljcDh0L
+JC2AWOla1EVG1QCE/pOo2bM+PoUX7EuKX17iISw46yIh38OBcxNqXnskMLm1rKkp
+lW7x4039JoBlu/A4pVYFecc68+GFSTAum4TtPci2x5wewG6FeiAx3g8BKA/BRgRF
+WVbP4bIIOLQ53ecdPw7hG9/7B4mZjR1C879pV6tHni/oa1RtpZ9ArOtIiB+8k6np
+sY9kBzg4RmTadPlQOmhHi+93rxKeu1PlBuGGHIH0f2RszCYJ/x9HiDVvO3KEgkth
+2jkOHMPGbqs05mRdntn2VXXhM/q4LnpmVeL45NeCMjqVZpTbE64CviuDhihU9QrU
+ThXTNFuz+QmNSuPSCcUe4PtJOJlUNsmqCA4KP/llam4OMI772ymRua7AIKgfhsn2
+zC7Mk8GMq6M+QYNQvsRnpG5GwuTulJaDWd93J2dHb4yHQScC+3Qwp4g05GUCAwEA
+AQKCAgEAoOZ1Qm9gerrGqeMhOwI587oXpEBHtxzSa3/wGl4uK4TQtbpJ78uH0I7x
+CZ8G4lcNdHZdpYTBZ/1elO6h51zrTVkKRZ8Yvb0qqBxB1ilKO0Zi/oXDUSfHj81P
+lscQ0u6UWnOJATnLj6pqCQUfj60tYD7ODN2lnulNfS0oxNilB6Fd6iSSR0MRHHRi
+CIAODsW6Dra4V2gd/KFsmJdPu2ar/As3cmg52FA7t0t3RaiDSvCCPGxFteNLfK7U
+P7VcFPZfcSRdLya+//3awTuxyBuhzenOL+Aigd6xo6Ri2IvMD+iv7tawl5Ucl6vv
+Xv30ug4KVSyOX6jsj/eb/2eV0v3w/veivYoxUR7X3eFJ70tB1oukpuPHr7Aps2Z2
+UdCMd60EBHfAHd7wXDuMbiKrmO7xCcn+pFXuAODNMTyc2mphGFgBTFY+usqiJKRF
+wVQDLoWNAaHIRjPfe8I45rR9PvfvlbG5P+XNz5lArd/SutMBjGEP9qGGOlrRtBRf
+jCKHP0bBb3JaWPiojZDmGv7IEVHiYJMLiWOfSbEFYk4vkr/FAkbCXrGd1sD67jT6
+BEg3uGzaAISxhDuoKipbbDD3qaaiXAoxQeAjV17ytv+9DmuvzH6uTLY2lwlCtIog
+Ambp6oeFTvDVo75GDleM81balAyVel03rv9R9uFYWk7D9IfA60ECggEBAPV2N8ZG
+fTcO2M+tr1Z3yu4m80vhrrauy6+zV7M+7CQrjmKr5iWYSKG9tZxpYhc+c9VsCs6m
+1NxytW6hxHT8fKPS8F5qnAleL/eogXvy6/7cnMaLW17PEnowexhsBLYUDc+f2/DT
+yM/hSIlCC7+bAESfV1HJGizao4L/yNzUpl7/hWRWZCCIlZShqc5RgeYBPBipOoCf
+4siuesnc5Ari+q/QVzuDKCEqZrk0SnHe4L9jzoY6H0WGVtImhyRaoEebSMJaoA2U
+TAIs9DfJoToVpoPnK+cCNPZvmbvOeWbc8tCcRnC7iOH6ilw7ffr1cPQ99zr1+v5W
+DQiLCl7BIgbpy5ECggEBAMhVCGebBK062lTqKAc0ma1RghoRfiETj1GhxTrebprD
+FnWKy/qIX3er0iKquTjpQqAtPMx+/EJ20DqNl9oBQHe9nWvaxyfK0QGRdFka7mWE
+ngF9xsXzuY94LLlYGPtLa1E0QPevudoxnWW2MrY1YSPi3ZqwtyBoIvMKdW3Pyr1P
+21rbx8dkvgJQYXXWesVBs1fZY9gBjJ55RiSxX3x9fG7Kxfq4ZWx72mFDW0vCKk8F
+4lncdNXF1tvfELqNVeKqNtL/3Ae6cIMnop7jDZ6VRunDIeKIyd0zs/GHnWE0XFTa
+QlNOTGRBTk8EorUZpAlrmmY8Yra2K992PcLvtE6cWZUCggEBAMdSgiQN5WwqMEEk
+hvcuO6l8E6EQ2HEcK+a3OYYimFsey3awpCjvKju885bZtH+vhRxPqc75hmyk8VOv
+JPridMwLZIknBSJOS6FOQidGsgC5YM2RKFcqy5RRfgPs4z5/tl9dLpq3P6AfDmn3
+KdcjpOoUgjG1nf3CZXjhAFjijV8jG0zJInGQt/aD+IHDijod0pEsCoecTeosVSVK
+rBtzCaJvCgcXkCSua9AE+FbTZ6nDAFfbJr2sQyBPjPomZXUAUesQ8FtWiwpROtd6
+cePa5ObDd9EU57EeGrZ5CfhPfxWLcV/lyVfHvUb8QGbjnB+ZmgNU78ZMVjUTww4C
+wwDeJSECggEAKMyAiLEec7Uvr+l8O3Nv9MdLyslTOaNFJeffRgz005n1ofyhbCWH
+49RfWGYSkM+YnQx5fGBEObj+0V3y1hg5K+m2/Lcyphz9EeewnVEJyi0HDOCzsa7g
+VKKx4RFbDRqgAYq1zHvCSvidyJAhpdAbN3bFkPJ6aSQsMfkoTzPpY0K3NrrZ5fbT
+Z589KgWDPVcTEkoYtm4iWEf0jBZ/jtfzlpC/Xz9ckIpswDA6kKQ7f5qALbLjLII9
+LswwGP4dxqrJvZEUG5LRxTQ2P0uYg3p9VbBC32LiUaVs+TyYRaqnPzSwG00v658N
+NrSH+n/9U1nRuQ6FQBa7uqfXDjNJMlVPPQKCAQEAxsluGdXNIKHwkQK5Mx2LS63V
+WO6X6nDAxWI2BM8AeAwcnSwpZStZzmHQaj6qm+RNE/TAILZSOs/BWijlhxM/jBh6
+gbKEF8+cwWS/WaSztYaUIl8fA2GD7Z4o0gY0bK1wxJYM4AfB9F7e7mu0p5yiRO8X
+e0nmCCAra4u8T+W1X1hYVntEj0KHGszR2S4CICsdFwheinTfbz2Voe9OoPrhkmbZ
+lxrxEy0q9D0kSbX2KekeLWWq6I7Ae8gIp1T+MNTU2UKICXPQTM2+F6hivCg7UX/g
+RWUnv8T+2Smsm5BfG9riUeSKzmnwkOI0oKBzB9nkt6prT5dW90endOHlJhJOuQ==
+-----END RSA PRIVATE KEY-----

tests/testutils/utils.go

@@ -53,16 +53,33 @@ current-context: test-context`
 
 func RunK8sApiserver(etcdIp string) *containers.Container {
 	return containers.Run("st-apiserver",
+		"-v", os.Getenv("PRIVATE_KEY")+":/private.key",
 		fmt.Sprintf("%s", os.Getenv("HYPERKUBE_IMAGE")),
 		"/hyperkube", "apiserver",
 		"--service-cluster-ip-range=10.101.0.0/16",
 		"--authorization-mode=AlwaysAllow",
 		"--insecure-port=8080",
 		"--insecure-bind-address=0.0.0.0",
 		fmt.Sprintf("--etcd-servers=http://%s:2379", etcdIp),
+		"--service-account-key-file=/private.key",
 	)
 }
 
+func RunK8sControllerManager(apiserverIp string) *containers.Container {
+	c := containers.Run("st-controller-manager",
+		"-v", os.Getenv("PRIVATE_KEY")+":/private.key",
+		fmt.Sprintf("%s", os.Getenv("HYPERKUBE_IMAGE")),
+		"/hyperkube", "controller-manager",
+		fmt.Sprintf("--master=%v:8080", apiserverIp),
+		"--min-resync-period=3m",
+		"--allocate-node-cidrs=true",
+		"--cluster-cidr=192.168.0.0/16",
+		"--v=5",
+		"--service-account-private-key-file=/private.key",
+	)
+	return c
+}
+
 func RunEtcd() *containers.Container {
 	return containers.Run("etcd-fv",
 		fmt.Sprintf("%s", os.Getenv("ETCD_IMAGE")),