Skip to content

Commit

Permalink
Merge pull request #7956 from govargo/split-integration-tunnelcmd
Browse files Browse the repository at this point in the history 
TestFunctional/parallel/TunnelCmd into serial subtest
  • Loading branch information
@medyagh
medyagh authored May 6, 2020
2 parents a3e9967 + 884f0e2 commit f3d5c57
Show file tree
Hide file tree
Showing 2 changed files with 203 additions and 20 deletions.
2 changes: 2 additions & 0 deletions site/content/en/docs/handbook/accessing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ Each service will get its own external ip.

If you are on macOS, the tunnel command also allows DNS resolution for Kubernetes services from the host.

NOTE: docker driver doesn't suport DNS resolution

### Cleaning up orphaned routes

If the `minikube tunnel` shuts down in an abrupt manner, it may leave orphaned network routes on your system. If this happens, the ~/.minikube/tunnels.json file will contain an entry for that tunnel. To remove orphaned routes, run:
Expand Down
221 changes: 201 additions & 20 deletions test/integration/fn_tunnel_cmd.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build integration

/*
Copyright 2018 The Kubernetes Authors All rights reserved.
Expand All @@ -20,6 +22,7 @@ import (
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"os/exec"
"path/filepath"
Expand All @@ -33,37 +36,104 @@ import (
"github.com/pkg/errors"

"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/tunnel"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/util"
"k8s.io/minikube/pkg/util/retry"
)

var tunnelSession StartSession

var (
hostname = ""
domain = "nginx-svc.default.svc.cluster.local."
)

func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
ctx, cancel := context.WithTimeout(ctx, Minutes(20))
type validateFunc func(context.Context, *testing.T, string)
defer cancel()

// Serial tests
t.Run("serial", func(t *testing.T) {
tests := []struct {
name string
validator validateFunc
}{
{"StartTunnel", validateTunnelStart}, // Start tunnel
{"WaitService", validateServiceStable}, // Wait for service is stable
{"AccessDirect", validateAccessDirect}, // Access test for loadbalancer IP
{"DNSResolutionByDig", validateDNSDig}, // DNS forwarding test by dig
{"DNSResolutionByDscacheutil", validateDNSDscacheutil}, // DNS forwarding test by dscacheutil
{"AccessThroughDNS", validateAccessDNS}, // Access test for absolute dns name
{"DeleteTunnel", validateTunnelDelete}, // Stop tunnel and delete cluster
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
tc.validator(ctx, t, profile)
})
}
})
}

// checkRoutePassword skips tunnel test if sudo password required for route
func checkRoutePassword(t *testing.T) {
if !KicDriver() && runtime.GOOS != "windows" {
if err := exec.Command("sudo", "-n", "ifconfig").Run(); err != nil {
t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
}
}
}

client, err := kapi.Client(profile)
if err != nil {
t.Fatalf("failed to get kubernetes client for %q: %v", profile, err)
// checkDNSForward skips DNS forwarding test if runtime is not supported
func checkDNSForward(t *testing.T) {
// Not all platforms support DNS forwarding
if runtime.GOOS != "darwin" {
t.Skip("DNS forwarding is supported for darwin only now, skipping test DNS forwarding")
}
}

// Pre-Cleanup
if err := tunnel.NewManager().CleanupNotRunningTunnels(); err != nil {
t.Errorf("CleanupNotRunningTunnels: %v", err)
// getKubeDNSIP returns kube-dns ClusterIP
func getKubeDNSIP(t *testing.T, profile string) string {
// Load ClusterConfig
c, err := config.Load(profile)
if err != nil {
t.Errorf("failed to load cluster config: %v", err)
}
// Get ipNet
_, ipNet, err := net.ParseCIDR(c.KubernetesConfig.ServiceCIDR)
if err != nil {
t.Errorf("failed to parse service CIDR: %v", err)
}
// Get kube-dns ClusterIP
ip, err := util.GetDNSIP(ipNet.String())
if err != nil {
t.Errorf("failed to get kube-dns IP: %v", err)
}

return ip.String()
}

// validateTunnelStart starts `minikube tunnel`
func validateTunnelStart(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)

// Start the tunnel
args := []string{"-p", profile, "tunnel", "--alsologtostderr"}
ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("failed to start a tunnel: args %q: %v", args, err)
}
defer ss.Stop(t)
tunnelSession = *ss
}

// validateServiceStable starts nginx pod, nginx service and waits nginx having loadbalancer ingress IP
func validateServiceStable(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)

client, err := kapi.Client(profile)
if err != nil {
t.Fatalf("failed to get kubernetes client for %q: %v", profile, err)
}

// Start the "nginx" pod.
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "testsvc.yaml")))
Expand All @@ -79,7 +149,6 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
}

// Wait until the nginx-svc has a loadbalancer ingress IP
hostname := ""
err = wait.PollImmediate(5*time.Second, Minutes(3), func() (bool, error) {
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc", "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}"))
if err != nil {
Expand All @@ -100,6 +169,11 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
}
t.Logf("failed to kubectl get svc nginx-svc:\n%s", rr.Stdout)
}
}

// validateAccessDirect validates if the test service can be accessed with LoadBalancer IP from host
func validateAccessDirect(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)

got := []byte{}
url := fmt.Sprintf("http://%s", hostname)
Expand All @@ -120,8 +194,16 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
}
return nil
}
if err = retry.Expo(fetch, 3*time.Second, Minutes(2), 13); err != nil {

// Check if the nginx service can be accessed
if err := retry.Expo(fetch, 3*time.Second, Minutes(2), 13); err != nil {
t.Errorf("failed to hit nginx at %q: %v", url, err)

rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc"))
if err != nil {
t.Errorf("%s failed: %v", rr.Command(), err)
}
t.Logf("failed to kubectl get svc nginx-svc:\n%s", rr.Stdout)
}

want := "Welcome to nginx!"
Expand All @@ -130,25 +212,124 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
} else {
t.Errorf("expected body to contain %q, but got *%q*", want, got)
}
}

// Not all platforms support DNS forwarding
if runtime.GOOS != "darwin" {
return
// validateDNSDig validates if the DNS forwarding works by dig command DNS lookup
// NOTE: DNS forwarding is experimental: https://minikube.sigs.k8s.io/docs/handbook/accessing/#dns-resolution-experimental
func validateDNSDig(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)
checkDNSForward(t)

ip := getKubeDNSIP(t, profile)
dnsIP := fmt.Sprintf("@%s", ip)

// Check if the dig DNS lookup works toward kube-dns IP
rr, err := Run(t, exec.CommandContext(ctx, "dig", "+time=5", "+tries=3", dnsIP, domain, "A"))
// dig command returns its output for stdout only. So we don't check stderr output.
if err != nil {
t.Errorf("failed to resolve DNS name: %v", err)
}

want := "ANSWER: 1"
if strings.Contains(rr.Stdout.String(), want) {
t.Logf("DNS resolution by dig for %s is working!", domain)
} else {
t.Errorf("expected body to contain %q, but got *%q*", want, rr.Stdout.String())

// debug DNS configuration
rr, err := Run(t, exec.CommandContext(ctx, "scutil", "--dns"))
if err != nil {
t.Errorf("%s failed: %v", rr.Command(), err)
}
t.Logf("debug for DNS configuration:\n%s", rr.Stdout.String())
}
}

// validateDNSDscacheutil validates if the DNS forwarding works by dscacheutil command DNS lookup
// NOTE: DNS forwarding is experimental: https://minikube.sigs.k8s.io/docs/handbook/accessing/#dns-resolution-experimental
func validateDNSDscacheutil(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)
checkDNSForward(t)

// Check if the dscacheutil DNS lookup works toward target domain
rr, err := Run(t, exec.CommandContext(ctx, "dscacheutil", "-q", "host", "-a", "name", domain))
// If dscacheutil cannot lookup dns record, it returns no output. So we don't check stderr output.
if err != nil {
t.Errorf("failed to resolve DNS name: %v", err)
}

want := hostname
if strings.Contains(rr.Stdout.String(), want) {
t.Logf("DNS resolution by dscacheutil for %s is working!", domain)
} else {
t.Errorf("expected body to contain %q, but got *%q*", want, rr.Stdout.String())
}
}

// validateAccessDNS validates if the test service can be accessed with DNS forwarding from host
// NOTE: DNS forwarding is experimental: https://minikube.sigs.k8s.io/docs/handbook/accessing/#dns-resolution-experimental
func validateAccessDNS(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)
checkDNSForward(t)

got := []byte{}
url := fmt.Sprintf("http://%s", domain)

ip := getKubeDNSIP(t, profile)
dnsIP := fmt.Sprintf("%s:53", ip)

// Set kube-dns dial
kubeDNSDial := func(ctx context.Context, network, address string) (net.Conn, error) {
d := net.Dialer{}
return d.DialContext(ctx, "udp", dnsIP)
}

// use FQDN to avoid extra DNS query lookup
url = "http://nginx-svc.default.svc.cluster.local."
if err = retry.Expo(fetch, 3*time.Second, Seconds(30), 10); err != nil {
// Set kube-dns resolver
r := net.Resolver{
PreferGo: true,
Dial: kubeDNSDial,
}
dialer := net.Dialer{Resolver: &r}

// Use kube-dns resolver
transport := &http.Transport{
Dial: dialer.Dial,
DialContext: dialer.DialContext,
}

fetch := func() error {
h := &http.Client{Timeout: time.Second * 10, Transport: transport}
resp, err := h.Get(url)
if err != nil {
return &retry.RetriableError{Err: err}
}
if resp.Body == nil {
return &retry.RetriableError{Err: fmt.Errorf("no body")}
}
defer resp.Body.Close()
got, err = ioutil.ReadAll(resp.Body)
if err != nil {
return &retry.RetriableError{Err: err}
}
return nil
}

// Access nginx-svc through DNS resolution
if err := retry.Expo(fetch, 3*time.Second, Seconds(30), 10); err != nil {
t.Errorf("failed to hit nginx with DNS forwarded %q: %v", url, err)
// debug more information for: https://github.com/kubernetes/minikube/issues/7809
clusterLogs(t, profile)
}

want = "Welcome to nginx!"
want := "Welcome to nginx!"
if strings.Contains(string(got), want) {
t.Logf("tunnel at %s is working!", url)
} else {
t.Errorf("expected body to contain %q, but got *%q*", want, got)
}
}

// validateTunnelDelete stops `minikube tunnel`
func validateTunnelDelete(ctx context.Context, t *testing.T, profile string) {
checkRoutePassword(t)
// Stop tunnel
tunnelSession.Stop(t)
}

0 comments on commit f3d5c57

Please sign in to comment.