connectivity: add endpointslice clustermesh sync test

This adds endpointslice synchronization testing inside a clustermesh. It does that by checking that we can get a DNS answer with a global headless service with endpointslice synchronization enabled across two clusters. Testing it via DNS ensures that the created EndpointSlice are correctly linked to the headless service. Signed-off-by: Arthur Outhenin-Chalandre <[email protected]>
cilium · Apr 17, 2024 · d7fa6d4 · d7fa6d4
1 parent f0cf9f0
commit d7fa6d4
Show file tree

Hide file tree

Showing 8 changed files with 155 additions and 24 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -48,6 +48,7 @@
 /connectivity/builder/no_ipsec_xfrm_errors.go @cilium/sig-encryption
 /connectivity/builder/node_to_node_encryption.go @cilium/sig-encryption
 /connectivity/builder/pod_to_pod_encryption.go @cilium/sig-encryption
+/connectivity/tests/clustermesh-endpointslice-sync.go @cilium/sig-clustermesh
 /connectivity/tests/egressgateway.go @cilium/egress-gateway
 /connectivity/tests/encryption.go @cilium/sig-encryption
 /connectivity/tests/errors.go @cilium/sig-agent @cilium/sig-datapath

diff --git a/connectivity/builder/builder.go b/connectivity/builder/builder.go
@@ -116,6 +116,7 @@ var (
 		clientEgressToEchoServiceAccountDeny{},
 		clientEgressToCidrDeny{},
 		clientEgressToCidrDenyDefault{},
+		clusterMeshEndpointSliceSync{},
 		health{},
 		northSouthLoadbalancing{},
 		podToPodEncryption{},

diff --git a/connectivity/builder/endpointslice-clustermesh-sync.go b/connectivity/builder/endpointslice-clustermesh-sync.go
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package builder
+
+import (
+	"github.com/cilium/cilium-cli/connectivity/check"
+	"github.com/cilium/cilium-cli/connectivity/tests"
+	"github.com/cilium/cilium-cli/utils/features"
+)
+
+type clusterMeshEndpointSliceSync struct{}
+
+func (t clusterMeshEndpointSliceSync) build(ct *check.ConnectivityTest, _ map[string]string) {
+	newTest("clustermesh-endpointslice-sync", ct).
+		WithCondition(func() bool { return ct.Params().MultiCluster != "" }).
+		WithFeatureRequirements(features.RequireEnabled(features.ClusterMeshEnableEndpointSync)).
+		WithScenarios(tests.ClusterMeshEndpointSliceSync())
+}
diff --git a/connectivity/check/context.go b/connectivity/check/context.go
@@ -1049,6 +1049,17 @@ func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam features.IPFamily) [
 	return cmd
 }
 
+func (ct *ConnectivityTest) DigCommandService(peer TestPeer, ipFam features.IPFamily) []string {
+	cmd := []string{"dig"}
+	if ipFam == features.IPFamilyV4 {
+		cmd = append(cmd, "A")
+	} else if ipFam == features.IPFamilyV6 {
+		cmd = append(cmd, "AAAA")
+	}
+	cmd = append(cmd, "+time=2", peer.Name())
+	return cmd
+}
+
 func (ct *ConnectivityTest) RandomClientPod() *Pod {
 	for _, p := range ct.clientPods {
 		return &p
@@ -1108,7 +1119,19 @@ func (ct *ConnectivityTest) EchoPods() map[string]Pod {
 	return ct.echoPods
 }
 
+// EchoServices returns all the non headless services
 func (ct *ConnectivityTest) EchoServices() map[string]Service {
+	svcs := map[string]Service{}
+	for name, svc := range ct.echoServices {
+		if svc.Service.Spec.ClusterIP == corev1.ClusterIPNone {
+			continue
+		}
+		svcs[name] = svc
+	}
+	return svcs
+}
+
+func (ct *ConnectivityTest) EchoServicesAll() map[string]Service {
 	return ct.echoServices
 }
 

diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go
@@ -43,15 +43,16 @@ const (
 
 	DNSTestServerContainerName = "dns-test-server"
 
-	echoSameNodeDeploymentName     = "echo-same-node"
-	echoOtherNodeDeploymentName    = "echo-other-node"
-	echoExternalNodeDeploymentName = "echo-external-node"
-	corednsConfigMapName           = "coredns-configmap"
-	corednsConfigVolumeName        = "coredns-config-volume"
-	kindEchoName                   = "echo"
-	kindEchoExternalNodeName       = "echo-external-node"
-	kindClientName                 = "client"
-	kindPerfName                   = "perf"
+	echoSameNodeDeploymentName                 = "echo-same-node"
+	echoOtherNodeDeploymentName                = "echo-other-node"
+	EchoOtherNodeDeploymentHeadlessServiceName = "echo-other-node-headless"
+	echoExternalNodeDeploymentName             = "echo-external-node"
+	corednsConfigMapName                       = "coredns-configmap"
+	corednsConfigVolumeName                    = "coredns-config-volume"
+	kindEchoName                               = "echo"
+	kindEchoExternalNodeName                   = "echo-external-node"
+	kindClientName                             = "client"
+	kindPerfName                               = "perf"
 
 	hostNetNSDeploymentName          = "host-netns"
 	hostNetNSDeploymentNameNonCilium = "host-netns-non-cilium" // runs on non-Cilium test nodes
@@ -524,18 +525,33 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
 
 	if ct.params.MultiCluster != "" {
 		_, err = ct.clients.src.GetService(ctx, ct.params.TestNamespace, echoOtherNodeDeploymentName, metav1.GetOptions{})
+		svc := newService(echoOtherNodeDeploymentName, map[string]string{"name": echoOtherNodeDeploymentName}, serviceLabels, "http", 8080)
+		svc.ObjectMeta.Annotations = map[string]string{}
+		svc.ObjectMeta.Annotations["service.cilium.io/global"] = "true"
+		svc.ObjectMeta.Annotations["io.cilium/global-service"] = "true"
+
 		if err != nil {
 			ct.Logf("✨ [%s] Deploying %s service...", ct.clients.src.ClusterName(), echoOtherNodeDeploymentName)
-			svc := newService(echoOtherNodeDeploymentName, map[string]string{"name": echoOtherNodeDeploymentName}, serviceLabels, "http", 8080)
-			svc.ObjectMeta.Annotations = map[string]string{}
-			svc.ObjectMeta.Annotations["service.cilium.io/global"] = "true"
-			svc.ObjectMeta.Annotations["io.cilium/global-service"] = "true"
-
 			_, err = ct.clients.src.CreateService(ctx, ct.params.TestNamespace, svc, metav1.CreateOptions{})
 			if err != nil {
 				return err
 			}
 		}
+
+		_, err = ct.clients.src.GetService(ctx, ct.params.TestNamespace, EchoOtherNodeDeploymentHeadlessServiceName, metav1.GetOptions{})
+		svcHeadless := svc.DeepCopy()
+		svcHeadless.Name = EchoOtherNodeDeploymentHeadlessServiceName
+		svcHeadless.Spec.ClusterIP = corev1.ClusterIPNone
+		svcHeadless.Spec.Type = corev1.ServiceTypeClusterIP
+		svcHeadless.ObjectMeta.Annotations["service.cilium.io/global-sync-endpoint-slices"] = "true"
+
+		if err != nil {
+			ct.Logf("✨ [%s] Deploying %s service...", ct.clients.src.ClusterName(), EchoOtherNodeDeploymentHeadlessServiceName)
+			_, err = ct.clients.src.CreateService(ctx, ct.params.TestNamespace, svcHeadless, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
+		}
 	}
 
 	hostPort := 0
@@ -740,22 +756,38 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
 	if !ct.params.SingleNode || ct.params.MultiCluster != "" {
 
 		_, err = ct.clients.dst.GetService(ctx, ct.params.TestNamespace, echoOtherNodeDeploymentName, metav1.GetOptions{})
-		if err != nil {
-			ct.Logf("✨ [%s] Deploying echo-other-node service...", ct.clients.dst.ClusterName())
-			svc := newService(echoOtherNodeDeploymentName, map[string]string{"name": echoOtherNodeDeploymentName}, serviceLabels, "http", 8080)
-
-			if ct.params.MultiCluster != "" {
-				svc.ObjectMeta.Annotations = map[string]string{}
-				svc.ObjectMeta.Annotations["service.cilium.io/global"] = "true"
-				svc.ObjectMeta.Annotations["io.cilium/global-service"] = "true"
-			}
+		svc := newService(echoOtherNodeDeploymentName, map[string]string{"name": echoOtherNodeDeploymentName}, serviceLabels, "http", 8080)
+		if ct.params.MultiCluster != "" {
+			svc.ObjectMeta.Annotations = map[string]string{}
+			svc.ObjectMeta.Annotations["service.cilium.io/global"] = "true"
+			svc.ObjectMeta.Annotations["io.cilium/global-service"] = "true"
+		}
 
+		if err != nil {
+			ct.Logf("✨ [%s] Deploying %s service...", ct.clients.dst.ClusterName(), echoOtherNodeDeploymentName)
 			_, err = ct.clients.dst.CreateService(ctx, ct.params.TestNamespace, svc, metav1.CreateOptions{})
 			if err != nil {
 				return err
 			}
 		}
 
+		if ct.params.MultiCluster != "" {
+			svcHeadless := svc.DeepCopy()
+			svcHeadless.Name = EchoOtherNodeDeploymentHeadlessServiceName
+			svcHeadless.Spec.ClusterIP = corev1.ClusterIPNone
+			svcHeadless.Spec.Type = corev1.ServiceTypeClusterIP
+			svcHeadless.ObjectMeta.Annotations["service.cilium.io/global-sync-endpoint-slices"] = "true"
+			_, err = ct.clients.dst.GetService(ctx, ct.params.TestNamespace, EchoOtherNodeDeploymentHeadlessServiceName, metav1.GetOptions{})
+
+			if err != nil {
+				ct.Logf("✨ [%s] Deploying %s service...", ct.clients.dst.ClusterName(), EchoOtherNodeDeploymentHeadlessServiceName)
+				_, err = ct.clients.dst.CreateService(ctx, ct.params.TestNamespace, svcHeadless, metav1.CreateOptions{})
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		_, err = ct.clients.dst.GetDeployment(ctx, ct.params.TestNamespace, echoOtherNodeDeploymentName, metav1.GetOptions{})
 		if err != nil {
 			ct.Logf("✨ [%s] Deploying other-node deployment...", ct.clients.dst.ClusterName())
@@ -1055,6 +1087,7 @@ func (ct *ConnectivityTest) deleteDeployments(ctx context.Context, client *k8s.C
 	_ = client.DeleteServiceAccount(ctx, ct.params.TestNamespace, client3DeploymentName, metav1.DeleteOptions{})
 	_ = client.DeleteService(ctx, ct.params.TestNamespace, echoSameNodeDeploymentName, metav1.DeleteOptions{})
 	_ = client.DeleteService(ctx, ct.params.TestNamespace, echoOtherNodeDeploymentName, metav1.DeleteOptions{})
+	_ = client.DeleteService(ctx, ct.params.TestNamespace, EchoOtherNodeDeploymentHeadlessServiceName, metav1.DeleteOptions{})
 	_ = client.DeleteConfigMap(ctx, ct.params.TestNamespace, corednsConfigMapName, metav1.DeleteOptions{})
 	_ = client.DeleteNamespace(ctx, ct.params.TestNamespace, metav1.DeleteOptions{})
 

diff --git a/connectivity/check/wait.go b/connectivity/check/wait.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/cilium/cilium/api/v1/models"
 	"golang.org/x/exp/slices"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/cilium/cilium-cli/defaults"
@@ -165,6 +166,11 @@ func WaitForService(ctx context.Context, log Logger, client Pod, service Service
 	ctx, cancel := context.WithTimeout(ctx, ShortTimeout)
 	defer cancel()
 
+	if service.Service.Spec.ClusterIP == corev1.ClusterIPNone {
+		// If the cluster is headless there is nothing to wait for here
+		return nil
+	}
+
 	for {
 		stdout, err := client.K8sClient.ExecInPod(ctx,
 			client.Namespace(), client.NameWithoutNamespace(), "",
@@ -204,6 +210,11 @@ func WaitForServiceEndpoints(ctx context.Context, log Logger, agent Pod, service
 	ctx, cancel := context.WithTimeout(ctx, ShortTimeout)
 	defer cancel()
 
+	if service.Service.Spec.ClusterIP == corev1.ClusterIPNone {
+		// If the cluster is headless there is nothing to wait for here
+		return nil
+	}
+
 	for {
 		err := checkServiceEndpoints(ctx, agent, service, backends, families)
 		if err == nil {

diff --git a/connectivity/tests/clustermesh-endpointslice-sync.go b/connectivity/tests/clustermesh-endpointslice-sync.go
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package tests
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cilium/cilium-cli/connectivity/check"
+	"github.com/cilium/cilium-cli/utils/features"
+)
+
+func ClusterMeshEndpointSliceSync() check.Scenario {
+	return &clusterMeshEndpointSliceSync{}
+}
+
+type clusterMeshEndpointSliceSync struct{}
+
+func (s *clusterMeshEndpointSliceSync) Name() string {
+	return "clustermesh-endpointslice-sync"
+}
+
+func (s *clusterMeshEndpointSliceSync) Run(ctx context.Context, t *check.Test) {
+	ct := t.Context()
+	client := ct.RandomClientPod()
+
+	service, ok := ct.EchoServicesAll()[check.EchoOtherNodeDeploymentHeadlessServiceName]
+	if !ok {
+		t.Fatalf("Cannot get %s service", check.EchoOtherNodeDeploymentHeadlessServiceName)
+	}
+
+	t.ForEachIPFamily(func(ipFam features.IPFamily) {
+		t.NewAction(s, fmt.Sprintf("dig-%s", ipFam), client, service, ipFam).Run(func(a *check.Action) {
+			a.ExecInPod(ctx, ct.DigCommandService(service, ipFam))
+		})
+	})
+}
diff --git a/utils/features/features.go b/utils/features/features.go
@@ -67,7 +67,8 @@ const (
 
 	CiliumIPAMMode Feature = "ipam"
 
-	IPsecEnabled Feature = "enable-ipsec"
+	IPsecEnabled                  Feature = "enable-ipsec"
+	ClusterMeshEnableEndpointSync Feature = "clustermesh-enable-endpoint-sync"
 )
 
 // Feature is the name of a Cilium Feature (e.g. l7-proxy, cni chaining mode etc)
@@ -284,6 +285,10 @@ func (fs Set) ExtractFromConfigMap(cm *v1.ConfigMap) {
 	fs[IPsecEnabled] = Status{
 		Enabled: cm.Data[string(IPsecEnabled)] == "true",
 	}
+
+	fs[ClusterMeshEnableEndpointSync] = Status{
+		Enabled: cm.Data[string(ClusterMeshEnableEndpointSync)] == "true",
+	}
 }
 
 func (fs Set) ExtractFromNodes(nodesWithoutCilium map[string]struct{}) {