Skip to content
This repository was archived by the owner on Feb 5, 2020. It is now read-only.

Cleanup smoke tests #3298

Merged
squat merged 5 commits into from
Jun 18, 2018
Merged

Cleanup smoke tests #3298

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
31eb165
remove unused code
paulfantom Jun 18, 2018
6d089f5
cleanup imports
paulfantom Jun 18, 2018
3b244ff
remove unused network policy tests
paulfantom Jun 18, 2018
32d0bc1
extend run time for killAPI test to reduce flakiness
paulfantom Jun 18, 2018
073ef37
remove unneeded imports
paulfantom Jun 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
315 changes: 2 additions & 313 deletions tests/smoke/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,11 @@ import (

"path/filepath"

"github.com/coreos/ktestutil/testworkload"
apierrors "k8s.io/apimachinery/pkg/api/errors"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/pkg/apis/extensions/v1beta1"
rbacv1beta1 "k8s.io/client-go/pkg/apis/rbac/v1beta1"
"k8s.io/client-go/tools/clientcmd"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
"k8s.io/kubernetes/pkg/kubectl/resource"
Expand Down Expand Up @@ -68,15 +61,6 @@ func testCluster(t *testing.T) {
t.Run("AllNodesRunning", testAllNodesRunning)
t.Run("AllResourcesCreated", testAllResourcesCreated)
t.Run("AllPodsRunning", testAllPodsRunning)
// TODO: temporary disabling this for OpenTonic
// t.Run("GetIdentityLogs", testGetIdentityLogs)

// TODO: temporary disabling this for OpenTonic
//ne := os.Getenv(networkingEnv)
//if ne == "canal" || ne == "calico-ipip" {
// t.Run("NetworkPolicy", testNetworkPolicy)
//}

t.Run("KillAPIServer", testKillAPIServer)
}

Expand Down Expand Up @@ -165,71 +149,6 @@ func testAllNodesRunning(t *testing.T) {
t.Logf("Successfully found %d ready nodes.", nodeCount)
}

func getIdentityLogs(t *testing.T) error {
c, _ := newClient(t)
podPrefix := "tectonic-identity"
_, err := validatePodLogging(c, tectonicSystemNamespace, podPrefix)
if err != nil {
return fmt.Errorf("failed to gather logs for %s/%s, %v", tectonicSystemNamespace, podPrefix, err)
}
return nil
}

func testGetIdentityLogs(t *testing.T) {
max := 10 * time.Minute
err := retry(getIdentityLogs, t, 15*time.Second, max)
if err != nil {
t.Fatalf("Failed to gather identity logs in %v.", max)
}
t.Log("Successfully gathered identity logs.")
}

// validatePodLogging verifies that logs can be retrieved for a container in Pod.
func validatePodLogging(c *kubernetes.Clientset, namespace, podPrefix string) ([]byte, error) {
var logs []byte
pods, err := c.Pods(namespace).List(meta_v1.ListOptions{})
if err != nil {
return logs, fmt.Errorf("could not list pods: %v", err)
}

var names string
for _, p := range pods.Items {
if len(names) != 0 {
names += ", "
}
names += p.Name

if !strings.HasPrefix(p.Name, podPrefix) {
continue
}
if len(p.Spec.Containers) == 0 {
return logs, fmt.Errorf("%s pod has no containers", p.Name)
}

opt := v1.PodLogOptions{
Container: p.Spec.Containers[0].Name,
}

result := c.Core().Pods(namespace).GetLogs(p.Name, &opt).Do()
if err := result.Error(); err != nil {
return logs, fmt.Errorf("failed to get pod logs: %v", err)
}

var statusCode int
result.StatusCode(&statusCode)
if statusCode/100 != 2 {
return logs, fmt.Errorf("expected 200 from log response, got %d", statusCode)
}
logs, err := result.Raw()
if err != nil {
return logs, fmt.Errorf("failed to read logs: %v", err)
}
return logs, nil
}

return logs, fmt.Errorf("failed to find pods with prefix %q (found pods in %s: %s)", podPrefix, namespace, names)
}

func testKillAPIServer(t *testing.T) {
c, _ := newClient(t)
pods, err := getAPIServers(c)
Expand Down Expand Up @@ -280,8 +199,8 @@ func testKillAPIServer(t *testing.T) {
return fmt.Errorf("API server has not yet been running for more than one check")
}

max := 6 * time.Minute
err = retry(apiServerUp, t, 10*time.Second, max)
max := 9 * time.Minute
err = retry(apiServerUp, t, 15*time.Second, max)
if err != nil {
t.Fatalf("Failed waiting for API server pods to be ready in %v.", max)
}
Expand Down Expand Up @@ -373,224 +292,6 @@ func testAllResourcesCreated(t *testing.T) {
}
}

var networkPolicyTestNamespacePSPRoleBinding = []byte(`kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: permissive-psp-access
namespace: network-policy-test
subjects:
- kind: ServiceAccount
name: replicaset-controller
namespace: kube-system
- kind: ServiceAccount
name: replication-controller
namespace: kube-system
- kind: ServiceAccount
name: job-controller
namespace: kube-system
- kind: ServiceAccount
name: daemon-set-controller
namespace: kube-system
- kind: ServiceAccount
name: statefulset-controller
namespace: kube-system
roleRef:
kind: ClusterRole
name: permissive-psp-access
apiGroup: rbac.authorization.k8s.io
`)

// testNetworkPolicy permforms 3 tests:
// * first ping test to check if network is setup correctly and reachable.
// * second ping test after setting `default-deny` policy on `network-policy-test` namespace
// to ensure nothing can talk to each other.
// * third ping test after setting `access-nginx` policy to ensure now nginx workload is reachable.
func testNetworkPolicy(t *testing.T) {
var (
namespace = "network-policy-test"
nginx *testworkload.Nginx
client kubernetes.Interface
)
client, _ = newClient(t)

ns := func(t *testing.T) error {
_, err := client.CoreV1().Namespaces().Create(&v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
},
})
if apierrors.IsAlreadyExists(err) {
t.Logf("ns already exists")
} else if err != nil {
return fmt.Errorf("failed to create namespace with name %v", namespace)
}

rbi, _, err := api.Codecs.UniversalDecoder().Decode(networkPolicyTestNamespacePSPRoleBinding, nil, &rbacv1beta1.RoleBinding{})
if err != nil {
return fmt.Errorf("unable to decode network policy manifest: %v", err)
}
rb, ok := rbi.(*rbacv1beta1.RoleBinding)
if !ok {
return fmt.Errorf("expected manifest to decode into *rbacv1beta1.RoleBinding, got %T", rbi)
}
_, err = client.RbacV1beta1().RoleBindings(namespace).Create(rb)
if apierrors.IsAlreadyExists(err) {
t.Logf("rolebinding already exists")
} else if err != nil {
return fmt.Errorf("failed to create rolebinding %v", rb)
}
return nil
}

max := 10 * time.Minute
if err := retry(ns, t, 30*time.Second, max); err != nil {
t.Fatalf("timed out waiting for namespace to be setup")
}

if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) {
var err error
if nginx, err = testworkload.NewNginx(client, namespace, testworkload.WithNginxPingJobLabels(map[string]string{"allow": "access"})); err != nil {
t.Logf("failed to create test nginx: %v", err)
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("failed to create an testworkload: %v", err)
}
defer nginx.Delete()

if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) {
if err := nginx.IsReachable(); err != nil {
t.Logf("error not reachable %s: %v", nginx.Name, err)
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("network not set up correctly: %v", err)
}

t.Run("DefaultDeny", func(t *testing.T) { testDefaultDenyNetworkPolicy(t, client, namespace, nginx) })
t.Run("NetworkPolicy", func(t *testing.T) { testAllowNetworkPolicy(t, client, namespace, nginx) })
}

func testDefaultDenyNetworkPolicy(t *testing.T, client kubernetes.Interface, namespace string, nginx *testworkload.Nginx) {
var defaultDenyNetworkPolicy = []byte(`kind: NetworkPolicy
apiVersion: extensions/v1beta1
metadata:
name: default-deny
spec:
podSelector:
`)

npi, _, err := api.Codecs.UniversalDecoder().Decode(defaultDenyNetworkPolicy, nil, &v1beta1.NetworkPolicy{})
if err != nil {
t.Fatalf("unable to decode network policy manifest: %v", err)
}
np, ok := npi.(*v1beta1.NetworkPolicy)
if !ok {
t.Fatalf("expected manifest to decode into *api.networkpolicy, got %T", npi)
}

httpRestClient := client.ExtensionsV1beta1().RESTClient()
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s",
strings.ToLower("extensions"),
strings.ToLower("v1beta1"),
strings.ToLower(namespace),
strings.ToLower("NetworkPolicies"))

result := httpRestClient.Post().RequestURI(uri).Body(np).Do()
if result.Error() != nil {
t.Fatal(result.Error())
}
defer func() {
uri = fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s",
strings.ToLower("extensions"),
strings.ToLower("v1beta1"),
strings.ToLower(namespace),
strings.ToLower("NetworkPolicies"),
strings.ToLower(np.ObjectMeta.Name))

result = httpRestClient.Delete().RequestURI(uri).Do()
if result.Error() != nil {
t.Fatal(result.Error())
}

}()

if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) {
if err := nginx.IsUnReachable(); err != nil {
t.Logf("error still reachable %s: %v", nginx.Name, err)
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("default deny failed: %v", err)
}
}

func testAllowNetworkPolicy(t *testing.T, client kubernetes.Interface, namespace string, nginx *testworkload.Nginx) {
var netPolicyTpl = []byte(`kind: NetworkPolicy
apiVersion: extensions/v1beta1
metadata:
name: access-nginx
spec:
podSelector:
matchLabels:
app: %s
ingress:
- from:
- podSelector:
matchLabels:
allow: access
`)

netPolicy := fmt.Sprintf(string(netPolicyTpl), nginx.Name)
npi, _, err := api.Codecs.UniversalDecoder().Decode([]byte(netPolicy), nil, &v1beta1.NetworkPolicy{})
if err != nil {
t.Fatalf("unable to decode network policy manifest: %v", err)
}
np, ok := npi.(*v1beta1.NetworkPolicy)
if !ok {
t.Fatalf("expected manifest to decode into *api.networkpolicy, got %T", npi)
}

httpRestClient := client.ExtensionsV1beta1().RESTClient()
uri := fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s",
strings.ToLower("extensions"),
strings.ToLower("v1beta1"),
strings.ToLower(namespace),
strings.ToLower("NetworkPolicies"))

result := httpRestClient.Post().RequestURI(uri).Body(np).Do()
if result.Error() != nil {
t.Fatal(result.Error())
}
defer func() {
uri = fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s/%s",
strings.ToLower("extensions"),
strings.ToLower("v1beta1"),
strings.ToLower(namespace),
strings.ToLower("NetworkPolicies"),
strings.ToLower(np.ObjectMeta.Name))

result = httpRestClient.Delete().RequestURI(uri).Do()
if result.Error() != nil {
t.Fatal(result.Error())
}

}()

if err := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) {
if err := nginx.IsReachable(); err != nil {
t.Logf("error not reachable %s: %v", nginx.Name, err)
return false, nil
}
return true, nil
}); err != nil {
t.Fatalf("allow nginx network policy failed: %v", err)
}
}

func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) {
pods, err := client.Core().Pods(kubeSystemNamespace).List(meta_v1.ListOptions{LabelSelector: apiServerSelector})
if err != nil {
Expand All @@ -602,18 +303,6 @@ func getAPIServers(client *kubernetes.Clientset) (*v1.PodList, error) {
return pods, nil
}

// podsStr prints a comma separated list of namespaced Pod names
func podsStr(pods []v1.Pod) (out string) {
for n, p := range pods {
// add comma to all entries except first
if n != 0 {
out += ", "
}
out += fmt.Sprintf("%s/%s", p.GetNamespace(), p.GetName())
}
return
}

func nodeReady(node v1.Node) (ok bool) {
for _, cond := range node.Status.Conditions {
if cond.Type == v1.NodeReady {
Expand Down