diff --git a/tests/smoke/cluster_test.go b/tests/smoke/cluster_test.go index 67df371d8c..4ea278bca8 100644 --- a/tests/smoke/cluster_test.go +++ b/tests/smoke/cluster_test.go @@ -12,18 +12,11 @@ import ( "path/filepath" - "github.com/coreos/ktestutil/testworkload" - apierrors "k8s.io/apimachinery/pkg/api/errors" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/pkg/api" "k8s.io/client-go/pkg/api/v1" - "k8s.io/client-go/pkg/apis/extensions/v1beta1" - rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1" "k8s.io/client-go/tools/clientcmd" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/kubectl/resource" @@ -68,15 +61,6 @@ func testCluster(t *testing.T) { t.Run("AllNodesRunning", testAllNodesRunning) t.Run("AllResourcesCreated", testAllResourcesCreated) t.Run("AllPodsRunning", testAllPodsRunning) - // TODO: temporary disabling this for OpenTonic - // t.Run("GetIdentityLogs", testGetIdentityLogs) - - // TODO: temporary disabling this for OpenTonic - //ne := os.Getenv(networkingEnv) - //if ne == "canal" || ne == "calico-ipip" { - // t.Run("NetworkPolicy", testNetworkPolicy) - //} - t.Run("KillAPIServer", testKillAPIServer) } @@ -165,71 +149,6 @@ func testAllNodesRunning(t *testing.T) { t.Logf("Successfully found %d ready nodes.", nodeCount) } -func getIdentityLogs(t *testing.T) error { - c, _ := newClient(t) - podPrefix := "tectonic-identity" - _, err := validatePodLogging(c, tectonicSystemNamespace, podPrefix) - if err != nil { - return fmt.Errorf("failed to gather logs for %s/%s, %v", tectonicSystemNamespace, podPrefix, err) - } - return nil -} - -func testGetIdentityLogs(t *testing.T) { - max := 10 * time.Minute - err := retry(getIdentityLogs, t, 15*time.Second, max) - if err != nil { - t.Fatalf("Failed to gather identity logs in %v.", max) - } - t.Log("Successfully gathered identity logs.") -} - -// validatePodLogging verifies that logs can be retrieved for a container in Pod. -func validatePodLogging(c *kubernetes.Clientset, namespace, podPrefix string) ([]byte, error) { - var logs []byte - pods, err := c.Pods(namespace).List(meta_v1.ListOptions{}) - if err != nil { - return logs, fmt.Errorf("could not list pods: %v", err) - } - - var names string - for _, p := range pods.Items { - if len(names) != 0 { - names += ", " - } - names += p.Name - - if !strings.HasPrefix(p.Name, podPrefix) { - continue - } - if len(p.Spec.Containers) == 0 { - return logs, fmt.Errorf("%s pod has no containers", p.Name) - } - - opt := v1.PodLogOptions{ - Container: p.Spec.Containers[0].Name, - } - - result := c.Core().Pods(namespace).GetLogs(p.Name, &opt).Do() - if err := result.Error(); err != nil { - return logs, fmt.Errorf("failed to get pod logs: %v", err) - } - - var statusCode int - result.StatusCode(&statusCode) - if statusCode/100 != 2 { - return logs, fmt.Errorf("expected 200 from log response, got %d", statusCode) - } - logs, err := result.Raw() - if err != nil { - return logs, fmt.Errorf("failed to read logs: %v", err) - } - return logs, nil - } - - return logs, fmt.Errorf("failed to find pods with prefix %q (found pods in %s: %s)", podPrefix, namespace, names) -} - func testKillAPIServer(t *testing.T) { c, _ := newClient(t) pods, err := getAPIServers(c) @@ -280,8 +199,8 @@ func testKillAPIServer(t *testing.T) { return fmt.Errorf("API server has not yet been running for more than one check") } - max := 6 * time.Minute - err = retry(apiServerUp, t, 10*time.Second, max) + max := 9 * time.Minute + err = retry(apiServerUp, t, 15*time.Second, max) if err != nil { t.Fatalf("Failed waiting for API server pods to be ready in %v.", max) } @@ -373,224 +292,6 @@ func testAllResourcesCreated(t *testing.T) { } } -var networkPolicyTestNamespacePSPRoleBinding = []byte(`kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: permissive-psp-access - namespace: network-policy-test -subjects: -- kind: ServiceAccount - name: replicaset-controller - namespace: kube-system -- kind: ServiceAccount - name: replication-controller - namespace: kube-system -- kind: ServiceAccount - name: job-controller - namespace: kube-system -- kind: ServiceAccount - name: daemon-set-controller - namespace: kube-system -- kind: ServiceAccount - name: statefulset-controller - namespace: kube-system -roleRef: - kind: ClusterRole - name: permissive-psp-access - apiGroup: rbac.authorization.k8s.io -`) - -// testNetworkPolicy permforms 3 tests: -// * first ping test to check if network is setup correctly and reachable. -// * second ping test after setting `default-deny` policy on `network-policy-test` namespace -// to ensure nothing can talk to each other. -// * third ping test after setting `access-nginx` policy to ensure now nginx workload is reachable. -func testNetworkPolicy(t *testing.T) { - var ( - namespace = "network-policy-test" - nginx *testworkload.Nginx - client kubernetes.Interface - ) - client, _ = newClient(t) - - ns := func(t *testing.T) error { - _, err := client.CoreV1().Namespaces().Create(&v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - }, - }) - if apierrors.IsAlreadyExists(err) { - t.Logf("ns already exists") - } else if err != nil { - return fmt.Errorf("failed to create namespace with name %v", namespace) - } - - rbi, _, err := api.Codecs.UniversalDecoder().Decode(networkPolicyTestNamespacePSPRoleBinding, nil, &rbacv1beta1.RoleBinding{}) - if err != nil { - return fmt.Errorf("unable to decode network policy manifest: %v", err) - } - rb, ok := rbi.(*rbacv1beta1.RoleBinding) - if !ok { - return fmt.Errorf("expected manifest to decode into *rbacv1beta1.RoleBinding, got %T", rbi) - } - _, err = client.RbacV1beta1().RoleBindings(namespace).Create(rb) - if apierrors.IsAlreadyExists(err) { - t.Logf("rolebinding already exists") - } else if err != nil { - return fmt.Errorf("failed to create rolebinding %v", rb) - } - return nil - } - - max := 10 * time.Minute - if err := retry(ns, t, 30*time.Second, max); err != nil { - t.Fatalf("timed out waiting for namespace to be setup") - } - - if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { - var err error - if nginx, err = testworkload.NewNginx(client, namespace, testworkload.WithNginxPingJobLabels(map[string]string{"allow": "access"})); err != nil { - t.Logf("failed to create test nginx: %v", err) - return false, nil - } - return true, nil - }); err != nil { - t.Fatalf("failed to create an testworkload: %v", err) - } - defer nginx.Delete() - - if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { - if err := nginx.IsReachable(); err != nil { - t.Logf("error not reachable %s: %v", nginx.Name, err) - return false, nil - } - return true, nil - }); err != nil { - t.Fatalf("network not set up correctly: %v", err) - } - - t.Run("DefaultDeny", func(t *testing.T) { testDefaultDenyNetworkPolicy(t, client, namespace, nginx) }) - t.Run("NetworkPolicy", func(t *testing.T) { testAllowNetworkPolicy(t, client, namespace, nginx) }) -} - -func testDefaultDenyNetworkPolicy(t *testing.T, client kubernetes.Interface, namespace string, nginx *testworkload.Nginx) { - var defaultDenyNetworkPolicy = []byte(`kind: NetworkPolicy -apiVersion: extensions/v1beta1 -metadata: - name: default-deny -spec: - podSelector: -`) - - npi, _, err := api.Codecs.UniversalDecoder().Decode(defaultDenyNetworkPolicy, nil, &v1beta1.NetworkPolicy{}) - if err != nil { - t.Fatalf("unable to decode network policy manifest: %v", err) - } - np, ok := npi.(*v1beta1.NetworkPolicy) - if !ok { - t.Fatalf("expected manifest to decode into *api.networkpolicy, got %T", npi) - } - - httpRestClient := client.ExtensionsV1beta1().RESTClient() - uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s", - strings.ToLower("extensions"), - strings.ToLower("v1beta1"), - strings.ToLower(namespace), - strings.ToLower("NetworkPolicies")) - - result := httpRestClient.Post().RequestURI(uri).Body(np).Do() - if result.Error() != nil { - t.Fatal(result.Error()) - } - defer func() { - uri = fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s", - strings.ToLower("extensions"), - strings.ToLower("v1beta1"), - strings.ToLower(namespace), - strings.ToLower("NetworkPolicies"), - strings.ToLower(np.ObjectMeta.Name)) - - result = httpRestClient.Delete().RequestURI(uri).Do() - if result.Error() != nil { - t.Fatal(result.Error()) - } - - }() - - if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { - if err := nginx.IsUnReachable(); err != nil { - t.Logf("error still reachable %s: %v", nginx.Name, err) - return false, nil - } - return true, nil - }); err != nil { - t.Fatalf("default deny failed: %v", err) - } -} - -func testAllowNetworkPolicy(t *testing.T, client kubernetes.Interface, namespace string, nginx *testworkload.Nginx) { - var netPolicyTpl = []byte(`kind: NetworkPolicy -apiVersion: extensions/v1beta1 -metadata: - name: access-nginx -spec: - podSelector: - matchLabels: - app: %s - ingress: - - from: - - podSelector: - matchLabels: - allow: access -`) - - netPolicy := fmt.Sprintf(string(netPolicyTpl), nginx.Name) - npi, _, err := api.Codecs.UniversalDecoder().Decode([]byte(netPolicy), nil, &v1beta1.NetworkPolicy{}) - if err != nil { - t.Fatalf("unable to decode network policy manifest: %v", err) - } - np, ok := npi.(*v1beta1.NetworkPolicy) - if !ok { - t.Fatalf("expected manifest to decode into *api.networkpolicy, got %T", npi) - } - - httpRestClient := client.ExtensionsV1beta1().RESTClient() - uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s", - strings.ToLower("extensions"), - strings.ToLower("v1beta1"), - strings.ToLower(namespace), - strings.ToLower("NetworkPolicies")) - - result := httpRestClient.Post().RequestURI(uri).Body(np).Do() - if result.Error() != nil { - t.Fatal(result.Error()) - } - defer func() { - uri = fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s", - strings.ToLower("extensions"), - strings.ToLower("v1beta1"), - strings.ToLower(namespace), - strings.ToLower("NetworkPolicies"), - strings.ToLower(np.ObjectMeta.Name)) - - result = httpRestClient.Delete().RequestURI(uri).Do() - if result.Error() != nil { - t.Fatal(result.Error()) - } - - }() - - if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { - if err := nginx.IsReachable(); err != nil { - t.Logf("error not reachable %s: %v", nginx.Name, err) - return false, nil - } - return true, nil - }); err != nil { - t.Fatalf("allow nginx network policy failed: %v", err) - } -} - func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) { pods, err := client.Core().Pods(kubeSystemNamespace).List(meta_v1.ListOptions{LabelSelector: apiServerSelector}) if err != nil { @@ -602,18 +303,6 @@ func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) { return pods, nil } -// podsStr prints a comma separated list of namespaced Pod names -func podsStr(pods []v1.Pod) (out string) { - for n, p := range pods { - // add comma to all entries except first - if n != 0 { - out += ", " - } - out += fmt.Sprintf("%s/%s", p.GetNamespace(), p.GetName()) - } - return -} - func nodeReady(node v1.Node) (ok bool) { for _, cond := range node.Status.Conditions { if cond.Type == v1.NodeReady {