Service to service troubleshooting

acceptance/tests/cli/cli_install_test.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"testing"
@@ -12,6 +13,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const ipv4RegEx = "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
@@ -21,9 +23,11 @@ const ipv4RegEx = "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.(25[0-5]|2[0-4][0-9]
 func TestInstall(t *testing.T) {
 	cases := map[string]struct {
 		secure bool
+		tproxy bool
 	}{
-		"not-secure": {secure: false},
-		"secure":     {secure: true},
+		"not-secure":        {secure: false, tproxy: false},
+		"secure":            {secure: true, tproxy: false},
+		"not-secure-tproxy": {secure: false, tproxy: true},
 	}
 
 	for name, c := range cases {
@@ -32,6 +36,7 @@ func TestInstall(t *testing.T) {
 			require.NoError(t, err)
 
 			cfg := suite.Config()
+			cfg.EnableTransparentProxy = c.tproxy
 			ctx := suite.Environment().DefaultContext(t)
 
 			connHelper := connhelper.ConnectHelper{
@@ -83,6 +88,39 @@ func TestInstall(t *testing.T) {
 				}
 			})
 
+			// Troubleshoot: Get the client pod so we can portForward to it and get the 'troubleshoot upstreams' output
+			clientPod, err := connHelper.Ctx.KubernetesClient(t).CoreV1().Pods(connHelper.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{
+				LabelSelector: "app=static-client",
+			})
+			require.NoError(t, err)
+
+			clientPodName := clientPod.Items[0].Name
+			upstreamsOut, err := cli.Run(t, ctx.KubectlOptions(t), "troubleshoot", "upstreams", "-pod", clientPodName)
+			logger.Log(t, string(upstreamsOut))
+			require.NoError(t, err)
+
+			if c.tproxy {
+				// If tproxy is enabled we are looking for the upstream ip which is the ClusterIP of the Kubernetes Service
+				serverService, err := connHelper.Ctx.KubernetesClient(t).CoreV1().Services(connHelper.Ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{
+					FieldSelector: "metadata.name=static-server",
+				})
+				require.NoError(t, err)
+				serverIP := serverService.Items[0].Spec.ClusterIP
+
+				proxyOut, err := cli.Run(t, ctx.KubectlOptions(t), "troubleshoot", "proxy", "-pod", clientPodName, "-upstream-ip", serverIP)
+				require.NoError(t, err)
+				require.Regexp(t, "upstream resources are valid", string(proxyOut))
+				logger.Log(t, string(proxyOut))
+			} else {
+				// With tproxy disabled and explicit upstreams we need the envoy-id of the server
+				require.Regexp(t, "static-server", string(upstreamsOut))
+
+				proxyOut, err := cli.Run(t, ctx.KubectlOptions(t), "troubleshoot", "proxy", "-pod", clientPodName, "-upstream-envoy-id", "static-server")
+				require.NoError(t, err)
+				require.Regexp(t, "upstream resources are valid", string(proxyOut))
+				logger.Log(t, string(proxyOut))
+			}
+
 			connHelper.TestConnectionSuccess(t)
 			connHelper.TestConnectionFailureWhenUnhealthy(t)
 		})

cli/cmd/troubleshoot/command.go

@@ -0,0 +1,26 @@
+package troubleshoot
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul-k8s/cli/common"
+	"github.com/mitchellh/cli"
+)
+
+// TroubleshootCommand  provides a synopsis for the troubleshoot subcommands (e.g. proxy, upstreams).
+type TroubleshootCommand struct {
+	*common.BaseCommand
+}
+
+// Run prints out information about the subcommands.
+func (c *TroubleshootCommand) Run([]string) int {
+	return cli.RunResultHelp
+}
+
+func (c *TroubleshootCommand) Help() string {
+	return fmt.Sprintf("%s\n\nUsage: consul-k8s troubleshoot <subcommand>", c.Synopsis())
+}
+
+func (c *TroubleshootCommand) Synopsis() string {
+	return "Troubleshoot network and security configurations."
+}

cli/cmd/troubleshoot/proxy/proxy.go

@@ -0,0 +1,282 @@
+package proxy
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+
+	"github.com/hashicorp/consul-k8s/cli/common"
+	"github.com/hashicorp/consul-k8s/cli/common/flag"
+	"github.com/hashicorp/consul-k8s/cli/common/terminal"
+	troubleshoot "github.com/hashicorp/consul/troubleshoot/proxy"
+	"github.com/posener/complete"
+	helmCLI "helm.sh/helm/v3/pkg/cli"
+	"k8s.io/apimachinery/pkg/api/validation"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+const (
+	defaultAdminPort        int = 19000
+	flagNameKubeConfig          = "kubeconfig"
+	flagNameKubeContext         = "context"
+	flagNameNamespace           = "namespace"
+	flagNamePod                 = "pod"
+	flagNameUpstreamEnvoyID     = "upstream-envoy-id"
+	flagNameUpstreamIP          = "upstream-ip"
+	DebugColor                  = "\033[0;36m%s\033[0m"
+)
+
+type ProxyCommand struct {
+	*common.BaseCommand
+
+	kubernetes kubernetes.Interface
+
+	set *flag.Sets
+
+	flagKubeConfig  string
+	flagKubeContext string
+	flagNamespace   string
+
+	flagPod             string
+	flagUpstreamEnvoyID string
+	flagUpstreamIP      string
+
+	restConfig *rest.Config
+
+	once sync.Once
+	help string
+}
+
+// init sets up flags and help text for the command.
+func (c *ProxyCommand) init() {
+	c.set = flag.NewSets()
+	f := c.set.NewSet("Command Options")
+
+	f.StringVar(&flag.StringVar{
+		Name:    flagNamePod,
+		Target:  &c.flagPod,
+		Usage:   "The pod to port-forward to.",
+		Aliases: []string{"p"},
+	})
+
+	f.StringVar(&flag.StringVar{
+		Name:    flagNameUpstreamEnvoyID,
+		Target:  &c.flagUpstreamEnvoyID,
+		Usage:   "The envoy identifier of the upstream service that receives the communication. (explicit upstreams only)",
+		Aliases: []string{"id"},
+	})
+
+	f.StringVar(&flag.StringVar{
+		Name:    flagNameUpstreamIP,
+		Target:  &c.flagUpstreamIP,
+		Usage:   "The IP address of the upstream service that receives the communication. (transparent proxy only)",
+		Aliases: []string{"ip"},
+	})
+
+	f = c.set.NewSet("Global Options")
+	f.StringVar(&flag.StringVar{
+		Name:    flagNameKubeConfig,
+		Aliases: []string{"c"},
+		Target:  &c.flagKubeConfig,
+		Default: "",
+		Usage:   "Set the path to kubeconfig file.",
+	})
+	f.StringVar(&flag.StringVar{
+		Name:    flagNameKubeContext,
+		Target:  &c.flagKubeContext,
+		Default: "",
+		Usage:   "Set the Kubernetes context to use.",
+	})
+
+	f.StringVar(&flag.StringVar{
+		Name:    flagNameNamespace,
+		Target:  &c.flagNamespace,
+		Usage:   "The namespace the pod is in.",
+		Aliases: []string{"n"},
+	})
+
+	c.help = c.set.Help()
+}
+
+// Run executes the list command.
+func (c *ProxyCommand) Run(args []string) int {
+	c.once.Do(c.init)
+	c.Log.ResetNamed("list")
+	defer common.CloseWithError(c.BaseCommand)
+
+	// Parse the command line flags.
+	if err := c.set.Parse(args); err != nil {
+		c.UI.Output("Error parsing arguments: %v", err.Error(), terminal.WithErrorStyle())
+		return 1
+	}
+
+	// Validate the command line flags.
+	if err := c.validateFlags(); err != nil {
+		c.UI.Output("Invalid argument: %v", err.Error(), terminal.WithErrorStyle())
+		return 1
+	}
+
+	if c.kubernetes == nil {
+		if err := c.initKubernetes(); err != nil {
+			c.UI.Output("Error initializing Kubernetes client: %v", err.Error(), terminal.WithErrorStyle())
+			return 1
+		}
+	}
+
+	if err := c.Troubleshoot(); err != nil {
+		c.UI.Output("Error running troubleshoot: %v", err.Error(), terminal.WithErrorStyle())
+		return 1
+	}
+
+	return 0
+}
+
+// validateFlags ensures that the flags passed in by the can be used.
+func (c *ProxyCommand) validateFlags() error {
+
+	if (c.flagUpstreamEnvoyID == "" && c.flagUpstreamIP == "") || (c.flagUpstreamEnvoyID != "" && c.flagUpstreamIP != "") {
+		return fmt.Errorf("-upstream-envoy-id OR -upstream-ip is required.\n Please run `consul troubleshoot upstreams` to find the corresponding upstream.")
+	}
+
+	if c.flagPod == "" {
+		return fmt.Errorf("-pod flag is required")
+	}
+
+	if errs := validation.ValidateNamespaceName(c.flagNamespace, false); c.flagNamespace != "" && len(errs) > 0 {
+		return fmt.Errorf("invalid namespace name passed for -namespace/-n: %v", strings.Join(errs, "; "))
+	}
+
+	return nil
+}
+
+// initKubernetes initializes the Kubernetes client.
+func (c *ProxyCommand) initKubernetes() (err error) {
+	settings := helmCLI.New()
+
+	if c.flagKubeConfig != "" {
+		settings.KubeConfig = c.flagKubeConfig
+	}
+
+	if c.flagKubeContext != "" {
+		settings.KubeContext = c.flagKubeContext
+	}
+
+	if c.restConfig == nil {
+		if c.restConfig, err = settings.RESTClientGetter().ToRESTConfig(); err != nil {
+			return fmt.Errorf("error creating Kubernetes REST config %v", err)
+		}
+	}
+
+	if c.kubernetes == nil {
+		if c.kubernetes, err = kubernetes.NewForConfig(c.restConfig); err != nil {
+			return fmt.Errorf("error creating Kubernetes client %v", err)
+		}
+	}
+
+	if c.flagNamespace == "" {
+		c.flagNamespace = settings.Namespace()
+	}
+
+	return nil
+}
+
+func (c *ProxyCommand) Troubleshoot() error {
+	pf := common.PortForward{
+		Namespace:  c.flagNamespace,
+		PodName:    c.flagPod,
+		RemotePort: defaultAdminPort,
+		KubeClient: c.kubernetes,
+		RestConfig: c.restConfig,
+	}
+
+	endpoint, err := pf.Open(c.Ctx)
+	if err != nil {
+		return err
+	}
+	defer pf.Close()
+
+	adminAddr, adminPort, err := net.SplitHostPort(endpoint)
+	if err != nil {
+		return err
+	}
+
+	adminAddrIP, err := net.ResolveIPAddr("ip", adminAddr)
+	if err != nil {
+		return err
+	}
+
+	t, err := troubleshoot.NewTroubleshoot(adminAddrIP, adminPort)
+	if err != nil {
+		return err
+	}
+
+	// err = t.GetEnvoyConfigDump()
+	// if err != nil {
+	// 	return err
+	// }
+
+	messages, err := t.RunAllTests(c.flagUpstreamEnvoyID, c.flagUpstreamIP)
+	if err != nil {
+		return err
+	}
+
+	c.UI.Output("Validation", terminal.WithHeaderStyle())
+	for _, o := range messages {
+		if o.Success {
+			c.UI.Output(o.Message, terminal.WithSuccessStyle())
+		} else {
+			c.UI.Output(o.Message, terminal.WithErrorStyle())
+			if o.PossibleActions != "" {
+				c.UI.Output(fmt.Sprintf("possible actions: %v", o.PossibleActions), terminal.WithInfoStyle())
+			}
+		}
+	}
+
+	return nil
+}
+
+// AutocompleteFlags returns a mapping of supported flags and autocomplete
+// options for this command. The map key for the Flags map should be the
+// complete flag such as "-foo" or "--foo".
+func (c *ProxyCommand) AutocompleteFlags() complete.Flags {
+	return complete.Flags{
+		fmt.Sprintf("-%s", flagNameNamespace):   complete.PredictNothing,
+		fmt.Sprintf("-%s", flagNameKubeConfig):  complete.PredictFiles("*"),
+		fmt.Sprintf("-%s", flagNameKubeContext): complete.PredictNothing,
+	}
+}
+
+// AutocompleteArgs returns the argument predictor for this command.
+// Since argument completion is not supported, this will return
+// complete.PredictNothing.
+func (c *ProxyCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (c *ProxyCommand) Synopsis() string {
+	return synopsis
+}
+
+func (c *ProxyCommand) Help() string {
+	return help
+}
+
+const (
+	synopsis = "Troubleshoots service mesh issues."
+	help     = `
+Usage: consul-k8s troubleshoot proxy [options]
+
+  Connect to a pod with a proxy and troubleshoots service mesh communication issues.
+
+  Requires a pod and upstream service SNI.
+
+  Examples:
+    $ consul-k8s troubleshoot proxy -pod pod1 -upstream foo
+
+	where 'pod1' is the pod running a consul proxy and 'foo' is the upstream envoy ID which 
+	can be obtained by running:
+	$ consul-k8s troubleshoot upstreams [options]
+`
+)