From b7a61a45a5f32f0e9a216294b3e1f7952b69492b Mon Sep 17 00:00:00 2001 From: Neil Jerram Date: Fri, 22 Jun 2018 13:52:03 +0100 Subject: [PATCH] Get tests running with k8s 1.10.4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are several steps needed here. If we just bump K8S_VERSION to 1.10.4, simple pod creation fails with an error indicating that the default service account is missing: • Failure [22.916 seconds] kube-controllers FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:45 Pod FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:651 should not overwrite a workload endpoint's container ID [It] /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:652 Expected error: <*errors.StatusError | 0xc42001eab0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: {SelfLink: "", ResourceVersion: "", Continue: ""}, Status: "Failure", Message: "pods \"testpod\" is forbidden: error looking up service account default/default: serviceaccount \"default\" not found", Reason: "Forbidden", Details: {Name: "testpod", Group: "", Kind: "pods", UID: "", Causes: nil, RetryAfterSeconds: 0}, Code: 403, }, } pods "testpod" is forbidden: error looking up service account default/default: serviceaccount "default" not found not to have occurred /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:679 Fix for that is to run the k8s controller manager, which in 1.10 is responsible for creating default service accounts. However, we then see another failure, because we haven't configured a key for the controller manager to use to sign an API token for the service account: • Failure [37.584 seconds] kube-controllers FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:45 Pod FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:656 should not overwrite a workload endpoint's container ID [It] /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:657 Expected error: <*errors.StatusError | 0xc420101560>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: {SelfLink: "", ResourceVersion: "", Continue: ""}, Status: "Failure", Message: "No API token found for service account \"default\", retry after the token is automatically created and added to the service account", Reason: "ServerTimeout", Details: { Name: "create pod", Group: "", Kind: "serviceaccounts", UID: "", Causes: nil, RetryAfterSeconds: 1, }, Code: 500, }, } No API token found for service account "default", retry after the token is automatically created and added to the service account not to have occurred /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:684 Fix for that is to create a token signing key and configure it to the API server and controller manager. (Ref/thanks: https://jvns.ca/blog/2017/08/05/how-kubernetes-certificates-work/) Then we move on to: • Failure [29.626 seconds] kube-controllers FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:45 Pod FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:656 should not overwrite a workload endpoint's container ID [It] /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:657 Expected error: <*errors.StatusError | 0xc4205dafc0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: {SelfLink: "", ResourceVersion: "", Continue: ""}, Status: "Failure", Message: "Pod \"testpod\" is invalid: spec: Forbidden: pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds` or `spec.tolerations` (only additions to existing tolerations)\n{\"Volumes\":\n\nA: null,\"InitContainers\":null,\"Containers\":[{\"Name\":\"container1\",\"Image\":\"busybox\",\"Command\":[\"sleep\",\"3600\"],\"Args\":null,\"WorkingDir\":\"\",\"Ports\":null,\"EnvFrom\":null,\"Env\":null,\"Resources\":{\"Limits\":null,\"Requests\":null},\"VolumeMounts\":null,\"VolumeDevices\":null,\"LivenessProbe\":null,\"ReadinessProbe\":null,\"Lifecycle\":null,\"TerminationMessagePath\":\"/dev/termination-log\",\"TerminationMessagePolicy\":\"File\",\"ImagePullPolicy\":\"Always\",\"SecurityContext\":null,\"Stdin\":false,\"StdinOnce\":false,\"TTY\":false}],\"RestartPolicy\":\"Always\",\"TerminationGracePeriodSeconds\":30,\"ActiveDeadlineSeconds\":null,\"DNSPolicy\":\"ClusterFirst\",\"NodeSelector\":null,\"ServiceAccountName\":\"\",\"AutomountServiceAccountToken\":null,\"NodeName\":\"127.0.0.1\",\"SecurityContext\":{\"HostNetwork\":false,\"HostPID\":false,\"HostIPC\":false,\"ShareProcessNamespace\":null,\"SELinuxOptions\":null,\"RunAsUser\":null,\"RunAsGroup\":null,\"RunAsNonRoot\":null,\"SupplementalGroups\":null,\"FSGroup\":null},\"ImagePullSecrets\":null,\"Hostname\":\"\",\"Subdomain\":\"\",\"Affinity\":null,\"SchedulerName\":\"default-scheduler\",\"Tolerations\":[{\"Key\":\"node.kubernetes.io/not-ready\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300},{\"Key\":\"node.kubernetes.io/unreachable\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300}],\"HostAliases\":null,\"PriorityClassName\":\"\",\"Priority\":null,\"DNSConfig\":null}\n\nB: [{\"Name\":\"default-token-4f8z6\",\"HostPath\":null,\"EmptyDir\":null,\"GCEPersistentDisk\":null,\"AWSElasticBlockStore\":null,\"GitRepo\":null,\"Secret\":{\"SecretName\":\"default-token-4f8z6\",\"Items\":null,\"DefaultMode\":420,\"Optional\":null},\"NFS\":null,\"ISCSI\":null,\"Glusterfs\":null,\"PersistentVolumeClaim\":null,\"RBD\":null,\"Quobyte\":null,\"FlexVolume\":null,\"Cinder\":null,\"CephFS\":null,\"Flocker\":null,\"DownwardAPI\":null,\"FC\":null,\"AzureFile\":null,\"ConfigMap\":null,\"VsphereVolume\":null,\"AzureDisk\":null,\"PhotonPersistentDisk\":null,\"Projected\":null,\"PortworxVolume\":null,\"ScaleIO\":null,\"StorageOS\":null}],\"InitContainers\":null,\"Containers\":[{\"Name\":\"container1\",\"Image\":\"busybox\",\"Command\":[\"sleep\",\"3600\"],\"Args\":null,\"WorkingDir\":\"\",\"Ports\":null,\"EnvFrom\":null,\"Env\":null,\"Resources\":{\"Limits\":null,\"Requests\":null},\"VolumeMounts\":[{\"Name\":\"default-token-4f8z6\",\"ReadOnly\":true,\"MountPath\":\"/var/run/secrets/kubernetes.io/serviceaccount\",\"SubPath\":\"\",\"MountPropagation\":null}],\"VolumeDevices\":null,\"LivenessProbe\":null,\"ReadinessProbe\":null,\"Lifecycle\":null,\"TerminationMessagePath\":\"/dev/termination-log\",\"TerminationMessagePolicy\":\"File\",\"ImagePullPolicy\":\"Always\",\"SecurityContext\":null,\"Stdin\":false,\"StdinOnce\":false,\"TTY\":false}],\"RestartPolicy\":\"Always\",\"TerminationGracePeriodSeconds\":30,\"ActiveDeadlineSeconds\":null,\"DNSPolicy\":\"ClusterFirst\",\"NodeSelector\":null,\"ServiceAccountName\":\"default\",\"AutomountServiceAccountToken\":null,\"NodeName\":\"127.0.0.1\",\"SecurityContext\":{\"HostNetwork\":false,\"HostPID\":false,\"HostIPC\":false,\"ShareProcessNamespace\":null,\"SELinuxOptions\":null,\"RunAsUser\":null,\"RunAsGroup\":null,\"RunAsNonRoot\":null,\"SupplementalGroups\":null,\"FSGroup\":null},\"ImagePullSecrets\":null,\"Hostname\":\"\",\"Subdomain\":\"\",\"Affinity\":null,\"SchedulerName\":\"default-scheduler\",\"Tolerations\":[{\"Key\":\"node.kubernetes.io/not-ready\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300},{\"Key\":\"node.kubernetes.io/unreachable\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300}],\"HostAliases\":null,\"PriorityClassName\":\"\",\"Priority\":null,\"DNSConfig\":null}\n\n", Reason: "Invalid", Details: { Name: "testpod", Group: "", Kind: "Pod", UID: "", Causes: [ { Type: "FieldValueForbidden", Message: "Forbidden: pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds` or `spec.tolerations` (only additions to existing tolerations)\n{\"Volumes\":\n\nA: null,\"InitContainers\":null,\"Containers\":[{\"Name\":\"container1\",\"Image\":\"busybox\",\"Command\":[\"sleep\",\"3600\"],\"Args\":null,\"WorkingDir\":\"\",\"Ports\":null,\"EnvFrom\":null,\"Env\":null,\"Resources\":{\"Limits\":null,\"Requests\":null},\"VolumeMounts\":null,\"VolumeDevices\":null,\"LivenessProbe\":null,\"ReadinessProbe\":null,\"Lifecycle\":null,\"TerminationMessagePath\":\"/dev/termination-log\",\"TerminationMessagePolicy\":\"File\",\"ImagePullPolicy\":\"Always\",\"SecurityContext\":null,\"Stdin\":false,\"StdinOnce\":false,\"TTY\":false}],\"RestartPolicy\":\"Always\",\"TerminationGracePeriodSeconds\":30,\"ActiveDeadlineSeconds\":null,\"DNSPolicy\":\"ClusterFirst\",\"NodeSelector\":null,\"ServiceAccountName\":\"\",\"AutomountServiceAccountToken\":null,\"NodeName\":\"127.0.0.1\",\"SecurityContext\":{\"HostNetwork\":false,\"HostPID\":false,\"HostIPC\":false,\"ShareProcessNamespace\":null,\"SELinuxOptions\":null,\"RunAsUser\":null,\"RunAsGroup\":null,\"RunAsNonRoot\":null,\"SupplementalGroups\":null,\"FSGroup\":null},\"ImagePullSecrets\":null,\"Hostname\":\"\",\"Subdomain\":\"\",\"Affinity\":null,\"SchedulerName\":\"default-scheduler\",\"Tolerations\":[{\"Key\":\"node.kubernetes.io/not-ready\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300},{\"Key\":\"node.kubernetes.io/unreachable\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300}],\"HostAliases\":null,\"PriorityClassName\":\"\",\"Priority\":null,\"DNSConfig\":null}\n\nB: [{\"Name\":\"default-token-4f8z6\",\"HostPath\":null,\"EmptyDir\":null,\"GCEPersistentDisk\":null,\"AWSElasticBlockStore\":null,\"GitRepo\":null,\"Secret\":{\"SecretName\":\"default-token-4f8z6\",\"Items\":null,\"DefaultMode\":420,\"Optional\":null},\"NFS\":null,\"ISCSI\":null,\"Glusterfs\":null,\"PersistentVolumeClaim\":null,\"RBD\":null,\"Quobyte\":null,\"FlexVolume\":null,\"Cinder\":null,\"CephFS\":null,\"Flocker\":null,\"DownwardAPI\":null,\"FC\":null,\"AzureFile\":null,\"ConfigMap\":null,\"VsphereVolume\":null,\"AzureDisk\":null,\"PhotonPersistentDisk\":null,\"Projected\":null,\"PortworxVolume\":null,\"ScaleIO\":null,\"StorageOS\":null}],\"InitContainers\":null,\"Containers\":[{\"Name\":\"container1\",\"Image\":\"busybox\",\"Command\":[\"sleep\",\"3600\"],\"Args\":null,\"WorkingDir\":\"\",\"Ports\":null,\"EnvFrom\":null,\"Env\":null,\"Resources\":{\"Limits\":null,\"Requests\":null},\"VolumeMounts\":[{\"Name\":\"default-token-4f8z6\",\"ReadOnly\":true,\"MountPath\":\"/var/run/secrets/kubernetes.io/serviceaccount\",\"SubPath\":\"\",\"MountPropagation\":null}],\"VolumeDevices\":null,\"LivenessProbe\":null,\"ReadinessProbe\":null,\"Lifecycle\":null,\"TerminationMessagePath\":\"/dev/termination-log\",\"TerminationMessagePolicy\":\"File\",\"ImagePullPolicy\":\"Always\",\"SecurityContext\":null,\"Stdin\":false,\"StdinOnce\":false,\"TTY\":false}],\"RestartPolicy\":\"Always\",\"TerminationGracePeriodSeconds\":30,\"ActiveDeadlineSeconds\":null,\"DNSPolicy\":\"ClusterFirst\",\"NodeSelector\":null,\"ServiceAccountName\":\"default\",\"AutomountServiceAccountToken\":null,\"NodeName\":\"127.0.0.1\",\"SecurityContext\":{\"HostNetwork\":false,\"HostPID\":false,\"HostIPC\":false,\"ShareProcessNamespace\":null,\"SELinuxOptions\":null,\"RunAsUser\":null,\"RunAsGroup\":null,\"RunAsNonRoot\":null,\"SupplementalGroups\":null,\"FSGroup\":null},\"ImagePullSecrets\":null,\"Hostname\":\"\",\"Subdomain\":\"\",\"Affinity\":null,\"SchedulerName\":\"default-scheduler\",\"Tolerations\":[{\"Key\":\"node.kubernetes.io/not-ready\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300},{\"Key\":\"node.kubernetes.io/unreachable\",\"Operator\":\"Exists\",\"Value\":\"\",\"Effect\":\"NoExecute\",\"TolerationSeconds\":300}],\"HostAliases\":null,\"PriorityClassName\":\"\",\"Priority\":null,\"DNSConfig\":null}\n\n", Field: "spec", }, ], RetryAfterSeconds: 0, }, Code: 422, }, } Pod "testpod" is invalid: spec: Forbidden: pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds` or `spec.tolerations` (only additions to existing tolerations) {"Volumes": A: null,"InitContainers":null,"Containers":[{"Name":"container1","Image":"busybox","Command":["sleep","3600"],"Args":null,"WorkingDir":"","Ports":null,"EnvFrom":null,"Env":null,"Resources":{"Limits":null,"Requests":null},"VolumeMounts":null,"VolumeDevices":null,"LivenessProbe":null,"ReadinessProbe":null,"Lifecycle":null,"TerminationMessagePath":"/dev/termination-log","TerminationMessagePolicy":"File","ImagePullPolicy":"Always","SecurityContext":null,"Stdin":false,"StdinOnce":false,"TTY":false}],"RestartPolicy":"Always","TerminationGracePeriodSeconds":30,"ActiveDeadlineSeconds":null,"DNSPolicy":"ClusterFirst","NodeSelector":null,"ServiceAccountName":"","AutomountServiceAccountToken":null,"NodeName":"127.0.0.1","SecurityContext":{"HostNetwork":false,"HostPID":false,"HostIPC":false,"ShareProcessNamespace":null,"SELinuxOptions":null,"RunAsUser":null,"RunAsGroup":null,"RunAsNonRoot":null,"SupplementalGroups":null,"FSGroup":null},"ImagePullSecrets":null,"Hostname":"","Subdomain":"","Affinity":null,"SchedulerName":"default-scheduler","Tolerations":[{"Key":"node.kubernetes.io/not-ready","Operator":"Exists","Value":"","Effect":"NoExecute","TolerationSeconds":300},{"Key":"node.kubernetes.io/unreachable","Operator":"Exists","Value":"","Effect":"NoExecute","TolerationSeconds":300}],"HostAliases":null,"PriorityClassName":"","Priority":null,"DNSConfig":null} B: [{"Name":"default-token-4f8z6","HostPath":null,"EmptyDir":null,"GCEPersistentDisk":null,"AWSElasticBlockStore":null,"GitRepo":null,"Secret":{"SecretName":"default-token-4f8z6","Items":null,"DefaultMode":420,"Optional":null},"NFS":null,"ISCSI":null,"Glusterfs":null,"PersistentVolumeClaim":null,"RBD":null,"Quobyte":null,"FlexVolume":null,"Cinder":null,"CephFS":null,"Flocker":null,"DownwardAPI":null,"FC":null,"AzureFile":null,"ConfigMap":null,"VsphereVolume":null,"AzureDisk":null,"PhotonPersistentDisk":null,"Projected":null,"PortworxVolume":null,"ScaleIO":null,"StorageOS":null}],"InitContainers":null,"Containers":[{"Name":"container1","Image":"busybox","Command":["sleep","3600"],"Args":null,"WorkingDir":"","Ports":null,"EnvFrom":null,"Env":null,"Resources":{"Limits":null,"Requests":null},"VolumeMounts":[{"Name":"default-token-4f8z6","ReadOnly":true,"MountPath":"/var/run/secrets/kubernetes.io/serviceaccount","SubPath":"","MountPropagation":null}],"VolumeDevices":null,"LivenessProbe":null,"ReadinessProbe":null,"Lifecycle":null,"TerminationMessagePath":"/dev/termination-log","TerminationMessagePolicy":"File","ImagePullPolicy":"Always","SecurityContext":null,"Stdin":false,"StdinOnce":false,"TTY":false}],"RestartPolicy":"Always","TerminationGracePeriodSeconds":30,"ActiveDeadlineSeconds":null,"DNSPolicy":"ClusterFirst","NodeSelector":null,"ServiceAccountName":"default","AutomountServiceAccountToken":null,"NodeName":"127.0.0.1","SecurityContext":{"HostNetwork":false,"HostPID":false,"HostIPC":false,"ShareProcessNamespace":null,"SELinuxOptions":null,"RunAsUser":null,"RunAsGroup":null,"RunAsNonRoot":null,"SupplementalGroups":null,"FSGroup":null},"ImagePullSecrets":null,"Hostname":"","Subdomain":"","Affinity":null,"SchedulerName":"default-scheduler","Tolerations":[{"Key":"node.kubernetes.io/not-ready","Operator":"Exists","Value":"","Effect":"NoExecute","TolerationSeconds":300},{"Key":"node.kubernetes.io/unreachable","Operator":"Exists","Value":"","Effect":"NoExecute","TolerationSeconds":300}],"HostAliases":null,"PriorityClassName":"","Priority":null,"DNSConfig":null} not to have occurred /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:723 This is because Pod content has changed since first creation, and our Update is invalid because it appears to be trying to change back bits that it isn't allowed to. Fix that by getting the current Pod content before making the intended label change. Then we see a second similar problem: • Failure [28.069 seconds] kube-controllers FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:45 Pod FV tests /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:656 should not overwrite a workload endpoint's container ID [It] /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:657 Expected error: <*errors.StatusError | 0xc42069a6c0>: { ErrStatus: { TypeMeta: {Kind: "", APIVersion: ""}, ListMeta: {SelfLink: "", ResourceVersion: "", Continue: ""}, Status: "Failure", Message: "Operation cannot be fulfilled on pods \"testpod\": the object has been modified; please apply your changes to the latest version and try again", Reason: "Conflict", Details: {Name: "testpod", Group: "", Kind: "pods", UID: "", Causes: nil, RetryAfterSeconds: 0}, Code: 409, }, } Operation cannot be fulfilled on pods "testpod": the object has been modified; please apply your changes to the latest version and try again not to have occurred /go/src/github.com/projectcalico/kube-controllers/tests/fv/fv_test.go:769 Similar fix here, and the "should not overwrite a workload endpoint's container ID" FV now passes. Finally a similar pod update fix is needed for the "should not create a workload endpoint when one does not already exist" FV. --- Makefile | 6 ++--- tests/fv/fv_test.go | 48 +++++++++++++++++++++++++++---------- tests/fv/private.key | 51 ++++++++++++++++++++++++++++++++++++++++ tests/testutils/utils.go | 17 ++++++++++++++ 4 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 tests/fv/private.key diff --git a/Makefile b/Makefile index 54a9a530..9cafda55 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ OS?=$(shell uname -s | tr A-Z a-z) ############################################################################### GO_BUILD_VER ?= v0.15 -K8S_VERSION=v1.8.1 +K8S_VERSION?=v1.10.4 HYPERKUBE_IMAGE?=gcr.io/google_containers/hyperkube-$(ARCH):$(K8S_VERSION) ETCD_VERSION=v3.2.5 ETCD_IMAGE ?= quay.io/coreos/etcd:$(ETCD_VERSION)-$(BUILDARCH) @@ -212,7 +212,7 @@ ut: vendor GINKGO_FOCUS?=.* fv: tests/fv/fv.test image @echo Running Go FVs. - cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) CONTAINER_NAME=$(CONTAINER_NAME):latest-$(ARCH) ./fv.test -ginkgo.slowSpecThreshold 30 -ginkgo.focus $(GINKGO_FOCUS) + cd tests/fv && ETCD_IMAGE=$(ETCD_IMAGE) HYPERKUBE_IMAGE=$(HYPERKUBE_IMAGE) CONTAINER_NAME=$(CONTAINER_NAME):latest-$(ARCH) PRIVATE_KEY=`pwd`/private.key ./fv.test -ginkgo.slowSpecThreshold 30 -ginkgo.focus $(GINKGO_FOCUS) tests/fv/fv.test: $(shell find ./tests -type f -name '*.go' -print) # We pre-build the test binary so that we can run it outside a container and allow it @@ -345,4 +345,4 @@ help: # Some kind of magic from https://gist.github.com/rcmachado/af3db315e31383 } \ { helpMsg = $$0 }' \ width=20 \ - $(MAKEFILE_LIST) \ No newline at end of file + $(MAKEFILE_LIST) diff --git a/tests/fv/fv_test.go b/tests/fv/fv_test.go index 21beaaa7..b5aaaaa0 100644 --- a/tests/fv/fv_test.go +++ b/tests/fv/fv_test.go @@ -44,11 +44,12 @@ import ( var _ = Describe("kube-controllers FV tests", func() { var ( - etcd *containers.Container - policyController *containers.Container - apiserver *containers.Container - calicoClient client.Interface - k8sClient *kubernetes.Clientset + etcd *containers.Container + policyController *containers.Container + apiserver *containers.Container + calicoClient client.Interface + k8sClient *kubernetes.Clientset + controllerManager *containers.Container ) const kNodeName = "k8snodename" @@ -79,12 +80,20 @@ var _ = Describe("kube-controllers FV tests", func() { _, err := k8sClient.CoreV1().Namespaces().List(metav1.ListOptions{}) return err }, 15*time.Second, 500*time.Millisecond).Should(BeNil()) + + // Run controller manager. Empirically it can take around 10s until the + // controller manager is ready to create default service accounts, even + // when the hyperkube image has already been downloaded to run the API + // server. We use Eventually to allow for possible delay when doing + // initial pod creation below. + controllerManager = testutils.RunK8sControllerManager(apiserver.IP) }) AfterEach(func() { - etcd.Stop() + controllerManager.Stop() policyController.Stop() apiserver.Stop() + etcd.Stop() }) It("should initialize the datastore at start-of-day", func() { @@ -675,8 +684,10 @@ var _ = Describe("kube-controllers FV tests", func() { } By("creating a Pod in the k8s API", func() { - _, err := k8sClient.CoreV1().Pods("default").Create(&pod) - Expect(err).NotTo(HaveOccurred()) + Eventually(func() error { + _, err := k8sClient.CoreV1().Pods("default").Create(&pod) + return err + }, "20s", "2s").ShouldNot(HaveOccurred()) }) By("updating the pod's status to be running", func() { @@ -713,8 +724,11 @@ var _ = Describe("kube-controllers FV tests", func() { By("updating the pod's labels to trigger a cache update", func() { // Definitively trigger a pod controller cache update by updating the pod's labels // in the Kubernetes API. This ensures the controller has the cached WEP with container-id-1. + podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pod = *podNow pod.Labels["foo"] = "label2" - _, err := k8sClient.CoreV1().Pods("default").Update(&pod) + _, err = k8sClient.CoreV1().Pods("default").Update(&pod) Expect(err).NotTo(HaveOccurred()) }) @@ -756,8 +770,11 @@ var _ = Describe("kube-controllers FV tests", func() { By("updating the pod's labels a second time to trigger a datastore sync", func() { // Trigger a pod 'update' in the pod controller by updating the pod's labels. + podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pod = *podNow pod.Labels["foo"] = "label3" - _, err := k8sClient.CoreV1().Pods(podNamespace).Update(&pod) + _, err = k8sClient.CoreV1().Pods(podNamespace).Update(&pod) Expect(err).NotTo(HaveOccurred()) }) @@ -806,13 +823,18 @@ var _ = Describe("kube-controllers FV tests", func() { } By("creating a Pod in the k8s API", func() { - _, err := k8sClient.CoreV1().Pods("default").Create(&pod) - Expect(err).NotTo(HaveOccurred()) + Eventually(func() error { + _, err := k8sClient.CoreV1().Pods("default").Create(&pod) + return err + }, "20s", "2s").ShouldNot(HaveOccurred()) }) By("updating that pod's labels", func() { + podNow, err := k8sClient.CoreV1().Pods("default").Get(podName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + pod = *podNow pod.Labels["foo"] = "label2" - _, err := k8sClient.CoreV1().Pods("default").Update(&pod) + _, err = k8sClient.CoreV1().Pods("default").Update(&pod) Expect(err).NotTo(HaveOccurred()) }) diff --git a/tests/fv/private.key b/tests/fv/private.key new file mode 100644 index 00000000..76f13b7f --- /dev/null +++ b/tests/fv/private.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAwBXj4nTY7UhlAnwMZUisHxOIEkeZjeRfX/efdlT+0/9K0MUy +++hm6N526CqRmm75ynlbkBx1qtVgKbID5gK+v3wyALS0J9oZRYdx1lWN6S5Ix4Rp +vU00NBVkbJLRGfUrJRFgpALgUeCy3LC9r+9ZAVucikFmbBSYS6tcej2OALXuV7DI +PYr7PvJ+JVNGVw/5wNW5FNoGKxhj10Lb4uAC/0YB5p70JM/x7vAbA9PaPljcDh0L +JC2AWOla1EVG1QCE/pOo2bM+PoUX7EuKX17iISw46yIh38OBcxNqXnskMLm1rKkp +lW7x4039JoBlu/A4pVYFecc68+GFSTAum4TtPci2x5wewG6FeiAx3g8BKA/BRgRF +WVbP4bIIOLQ53ecdPw7hG9/7B4mZjR1C879pV6tHni/oa1RtpZ9ArOtIiB+8k6np +sY9kBzg4RmTadPlQOmhHi+93rxKeu1PlBuGGHIH0f2RszCYJ/x9HiDVvO3KEgkth +2jkOHMPGbqs05mRdntn2VXXhM/q4LnpmVeL45NeCMjqVZpTbE64CviuDhihU9QrU +ThXTNFuz+QmNSuPSCcUe4PtJOJlUNsmqCA4KP/llam4OMI772ymRua7AIKgfhsn2 +zC7Mk8GMq6M+QYNQvsRnpG5GwuTulJaDWd93J2dHb4yHQScC+3Qwp4g05GUCAwEA +AQKCAgEAoOZ1Qm9gerrGqeMhOwI587oXpEBHtxzSa3/wGl4uK4TQtbpJ78uH0I7x +CZ8G4lcNdHZdpYTBZ/1elO6h51zrTVkKRZ8Yvb0qqBxB1ilKO0Zi/oXDUSfHj81P +lscQ0u6UWnOJATnLj6pqCQUfj60tYD7ODN2lnulNfS0oxNilB6Fd6iSSR0MRHHRi +CIAODsW6Dra4V2gd/KFsmJdPu2ar/As3cmg52FA7t0t3RaiDSvCCPGxFteNLfK7U +P7VcFPZfcSRdLya+//3awTuxyBuhzenOL+Aigd6xo6Ri2IvMD+iv7tawl5Ucl6vv +Xv30ug4KVSyOX6jsj/eb/2eV0v3w/veivYoxUR7X3eFJ70tB1oukpuPHr7Aps2Z2 +UdCMd60EBHfAHd7wXDuMbiKrmO7xCcn+pFXuAODNMTyc2mphGFgBTFY+usqiJKRF +wVQDLoWNAaHIRjPfe8I45rR9PvfvlbG5P+XNz5lArd/SutMBjGEP9qGGOlrRtBRf +jCKHP0bBb3JaWPiojZDmGv7IEVHiYJMLiWOfSbEFYk4vkr/FAkbCXrGd1sD67jT6 +BEg3uGzaAISxhDuoKipbbDD3qaaiXAoxQeAjV17ytv+9DmuvzH6uTLY2lwlCtIog +Ambp6oeFTvDVo75GDleM81balAyVel03rv9R9uFYWk7D9IfA60ECggEBAPV2N8ZG +fTcO2M+tr1Z3yu4m80vhrrauy6+zV7M+7CQrjmKr5iWYSKG9tZxpYhc+c9VsCs6m +1NxytW6hxHT8fKPS8F5qnAleL/eogXvy6/7cnMaLW17PEnowexhsBLYUDc+f2/DT +yM/hSIlCC7+bAESfV1HJGizao4L/yNzUpl7/hWRWZCCIlZShqc5RgeYBPBipOoCf +4siuesnc5Ari+q/QVzuDKCEqZrk0SnHe4L9jzoY6H0WGVtImhyRaoEebSMJaoA2U +TAIs9DfJoToVpoPnK+cCNPZvmbvOeWbc8tCcRnC7iOH6ilw7ffr1cPQ99zr1+v5W +DQiLCl7BIgbpy5ECggEBAMhVCGebBK062lTqKAc0ma1RghoRfiETj1GhxTrebprD +FnWKy/qIX3er0iKquTjpQqAtPMx+/EJ20DqNl9oBQHe9nWvaxyfK0QGRdFka7mWE +ngF9xsXzuY94LLlYGPtLa1E0QPevudoxnWW2MrY1YSPi3ZqwtyBoIvMKdW3Pyr1P +21rbx8dkvgJQYXXWesVBs1fZY9gBjJ55RiSxX3x9fG7Kxfq4ZWx72mFDW0vCKk8F +4lncdNXF1tvfELqNVeKqNtL/3Ae6cIMnop7jDZ6VRunDIeKIyd0zs/GHnWE0XFTa +QlNOTGRBTk8EorUZpAlrmmY8Yra2K992PcLvtE6cWZUCggEBAMdSgiQN5WwqMEEk +hvcuO6l8E6EQ2HEcK+a3OYYimFsey3awpCjvKju885bZtH+vhRxPqc75hmyk8VOv +JPridMwLZIknBSJOS6FOQidGsgC5YM2RKFcqy5RRfgPs4z5/tl9dLpq3P6AfDmn3 +KdcjpOoUgjG1nf3CZXjhAFjijV8jG0zJInGQt/aD+IHDijod0pEsCoecTeosVSVK +rBtzCaJvCgcXkCSua9AE+FbTZ6nDAFfbJr2sQyBPjPomZXUAUesQ8FtWiwpROtd6 +cePa5ObDd9EU57EeGrZ5CfhPfxWLcV/lyVfHvUb8QGbjnB+ZmgNU78ZMVjUTww4C +wwDeJSECggEAKMyAiLEec7Uvr+l8O3Nv9MdLyslTOaNFJeffRgz005n1ofyhbCWH +49RfWGYSkM+YnQx5fGBEObj+0V3y1hg5K+m2/Lcyphz9EeewnVEJyi0HDOCzsa7g +VKKx4RFbDRqgAYq1zHvCSvidyJAhpdAbN3bFkPJ6aSQsMfkoTzPpY0K3NrrZ5fbT +Z589KgWDPVcTEkoYtm4iWEf0jBZ/jtfzlpC/Xz9ckIpswDA6kKQ7f5qALbLjLII9 +LswwGP4dxqrJvZEUG5LRxTQ2P0uYg3p9VbBC32LiUaVs+TyYRaqnPzSwG00v658N +NrSH+n/9U1nRuQ6FQBa7uqfXDjNJMlVPPQKCAQEAxsluGdXNIKHwkQK5Mx2LS63V +WO6X6nDAxWI2BM8AeAwcnSwpZStZzmHQaj6qm+RNE/TAILZSOs/BWijlhxM/jBh6 +gbKEF8+cwWS/WaSztYaUIl8fA2GD7Z4o0gY0bK1wxJYM4AfB9F7e7mu0p5yiRO8X +e0nmCCAra4u8T+W1X1hYVntEj0KHGszR2S4CICsdFwheinTfbz2Voe9OoPrhkmbZ +lxrxEy0q9D0kSbX2KekeLWWq6I7Ae8gIp1T+MNTU2UKICXPQTM2+F6hivCg7UX/g +RWUnv8T+2Smsm5BfG9riUeSKzmnwkOI0oKBzB9nkt6prT5dW90endOHlJhJOuQ== +-----END RSA PRIVATE KEY----- diff --git a/tests/testutils/utils.go b/tests/testutils/utils.go index 738d7981..e919c7c1 100644 --- a/tests/testutils/utils.go +++ b/tests/testutils/utils.go @@ -53,6 +53,7 @@ current-context: test-context` func RunK8sApiserver(etcdIp string) *containers.Container { return containers.Run("st-apiserver", + "-v", os.Getenv("PRIVATE_KEY")+":/private.key", fmt.Sprintf("%s", os.Getenv("HYPERKUBE_IMAGE")), "/hyperkube", "apiserver", "--service-cluster-ip-range=10.101.0.0/16", @@ -60,9 +61,25 @@ func RunK8sApiserver(etcdIp string) *containers.Container { "--insecure-port=8080", "--insecure-bind-address=0.0.0.0", fmt.Sprintf("--etcd-servers=http://%s:2379", etcdIp), + "--service-account-key-file=/private.key", ) } +func RunK8sControllerManager(apiserverIp string) *containers.Container { + c := containers.Run("st-controller-manager", + "-v", os.Getenv("PRIVATE_KEY")+":/private.key", + fmt.Sprintf("%s", os.Getenv("HYPERKUBE_IMAGE")), + "/hyperkube", "controller-manager", + fmt.Sprintf("--master=%v:8080", apiserverIp), + "--min-resync-period=3m", + "--allocate-node-cidrs=true", + "--cluster-cidr=192.168.0.0/16", + "--v=5", + "--service-account-private-key-file=/private.key", + ) + return c +} + func RunEtcd() *containers.Container { return containers.Run("etcd-fv", fmt.Sprintf("%s", os.Getenv("ETCD_IMAGE")),