Capture additional clustermesh-related troubleshooting information as part of sysdumps

defaults/defaults.go

@@ -37,6 +37,7 @@ const (
 	HubbleGenerateCertsCronJobName = "hubble-generate-certs"
 
 	ClusterMeshDeploymentName             = "clustermesh-apiserver"
+	ClusterMeshBinaryName                 = "/usr/bin/clustermesh-apiserver"
 	ClusterMeshContainerName              = "apiserver"
 	ClusterMeshPodSelector                = "k8s-app=clustermesh-apiserver"
 	ClusterMeshMetricsPortName            = "apiserv-metrics"

k8s/client.go

@@ -7,10 +7,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -34,6 +37,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/httpstream"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/resource"
@@ -43,6 +47,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth" // Register all auth providers (azure, gcp, oidc, openstack, ..).
 	"k8s.io/client-go/rest"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/transport/spdy"
 
 	"github.com/cilium/cilium-cli/defaults"
 )
@@ -734,6 +739,94 @@ func (c *Client) ProxyGet(ctx context.Context, namespace, name, url string) (str
 	return string(rawbody), nil
 }
 
+func (c *Client) ProxyTCP(ctx context.Context, namespace, name string, port uint16, handler func(io.ReadWriteCloser) error) error {
+	request := c.Clientset.CoreV1().RESTClient().Post().
+		Resource(corev1.ResourcePods.String()).
+		Namespace(namespace).
+		Name(name).
+		SubResource("portforward")
+
+	transport, upgrader, err := spdy.RoundTripperFor(c.Config)
+	if err != nil {
+		return fmt.Errorf("creating round tripper: %w", err)
+	}
+
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, request.URL())
+
+	const portForwardProtocolV1Name = "portforward.k8s.io"
+	conn, proto, err := dialer.Dial(portForwardProtocolV1Name)
+	if err != nil {
+		return fmt.Errorf("connecting: %w", err)
+	}
+
+	defer conn.Close()
+	if proto != portForwardProtocolV1Name {
+		return fmt.Errorf("unable to negotiate protocol: client supports %q, server returned %q", portForwardProtocolV1Name, proto)
+	}
+
+	go func() {
+		select {
+		case <-ctx.Done():
+			// Close aborts all remaining streams, and unblocks read operations.
+			conn.Close()
+		case <-conn.CloseChan():
+		}
+	}()
+
+	return stream(conn, port, handler)
+}
+
+// The following is an adapted version of part of the client-go's port-forward connection handling implementation:
+// https://github.com/kubernetes/client-go/blob/4ebe42d8c9c18f464fcc7b4f15b3a632db4cbdb2/tools/portforward/portforward.go#L335-L416
+func stream(conn httpstream.Connection, port uint16, handler func(io.ReadWriteCloser) error) error {
+	headers := http.Header{}
+	headers.Set(corev1.StreamType, corev1.StreamTypeError)
+	headers.Set(corev1.PortHeader, strconv.FormatUint(uint64(port), 10))
+
+	errorStream, err := conn.CreateStream(headers)
+	if err != nil {
+		return fmt.Errorf("creating error stream: %w", err)
+	}
+	// we're not writing to this stream
+	errorStream.Close()
+	defer conn.RemoveStreams(errorStream)
+
+	errorDone := make(chan error)
+	go func() {
+		defer close(errorDone)
+		message, err := io.ReadAll(errorStream)
+		switch {
+		case err != nil:
+			errorDone <- fmt.Errorf("reading from error stream: %w", err)
+		case len(message) > 0:
+			errorDone <- errors.New(string(message))
+		}
+	}()
+
+	headers.Set(corev1.StreamType, corev1.StreamTypeData)
+	dataStream, err := conn.CreateStream(headers)
+	if err != nil {
+		return fmt.Errorf("creating data stream: %w", err)
+	}
+	defer conn.RemoveStreams(dataStream)
+
+	dataDone := make(chan error)
+	go func() {
+		defer close(dataDone)
+		if err := handler(dataStream); err != nil {
+			dataDone <- err
+		}
+	}()
+
+	// Wait for both goroutines to terminate
+	err = <-dataDone
+	if err2 := <-errorDone; err2 != nil {
+		err = err2
+	}
+
+	return err
+}
+
 func (c *Client) ListUnstructured(ctx context.Context, gvr schema.GroupVersionResource, namespace *string, o metav1.ListOptions) (*unstructured.UnstructuredList, error) {
 	if namespace == nil {
 		return c.DynamicClientset.Resource(gvr).List(ctx, o)

sysdump/client.go

@@ -6,6 +6,7 @@ package sysdump
 import (
 	"bytes"
 	"context"
+	"io"
 
 	"github.com/blang/semver/v4"
 	ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
@@ -40,6 +41,7 @@ type KubernetesClient interface {
 	GetLogs(ctx context.Context, namespace, name, container string, opts corev1.PodLogOptions) (string, error)
 	GetPodsTable(ctx context.Context) (*metav1.Table, error)
 	ProxyGet(ctx context.Context, namespace, name, url string) (string, error)
+	ProxyTCP(ctx context.Context, namespace, name string, port uint16, handler func(io.ReadWriteCloser) error) error
 	GetSecret(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.Secret, error)
 	GetCiliumVersion(ctx context.Context, p *corev1.Pod) (*semver.Version, error)
 	GetVersion(ctx context.Context) (string, error)

sysdump/constants.go

@@ -9,6 +9,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
+	"github.com/google/gops/signal"
+
 	"github.com/cilium/cilium-cli/defaults"
 )
 
@@ -72,12 +74,9 @@ const (
 	ciliumNodesFileName                      = "ciliumnodes-<ts>.yaml"
 	ciliumNodeConfigsFileName                = "ciliumnodeconfigs-<ts>.yaml"
 	ciliumOperatorDeploymentFileName         = "cilium-operator-deployment-<ts>.yaml"
-	ciliumOperatorPodMetricsFileName         = "cilium-operator-pod-%s-metrics-<ts>.yaml"
 	ciliumPodIPPoolsFileName                 = "ciliumpodippools-<ts>.yaml"
 	clustermeshApiserverDeploymentFileName   = "clustermesh-apiserver-deployment-<ts>.yaml"
-	clustermeshEtcdMetricsFileName           = "cilium-clustermesh-etcd-%s-metrics-<ts>.yaml"
-	clustermeshMetricsFileName               = "clustermesh-apiserver-%s-metrics-<ts>.yaml"
-	clustermeshKVStoreMeshMetricsFileName    = "clustermesh-kvstoremesh-%s-metrics-<ts>.yaml"
+	metricsFileName                          = "metrics-%s-%s-<ts>.txt"
 	cniConfigMapFileName                     = "cni-configmap-<ts>.yaml"
 	cniConfigFileName                        = "cniconf-%s-%s-<ts>.txt"
 	eniconfigsFileName                       = "aws-eniconfigs-<ts>.yaml"
@@ -152,9 +151,9 @@ var (
 		"stack",
 		"stats",
 	}
-	gopsProfiling = []string{
-		"pprof-heap",
-		"pprof-cpu",
+	gopsProfiling = map[string]byte{
+		"pprof-heap": signal.HeapProfile,
+		"pprof-cpu":  signal.CPUProfile,
 	}
 	gopsTrace = "trace"