connectivity: add egress-gateway-with-l7-policy test

egress-gateway-with-l7-policy checks if traffic from Pods that
are selected by both Egress Gateway Policy and L7 Network Policy
is properly SNATed with an Egress IP.

Signed-off-by: Yusuke Suzuki <[email protected]>

Author: ysksuzuki

connectivity/builder/builder.go

@@ -209,6 +209,7 @@ func concurrentTests(connTests []*check.ConnectivityTest) error {
 		nodeToNodeEncryption{},
 		egressGateway{},
 		egressGatewayExcludedCidrs{},
+		egressGatewayWithL7Policy{},
 		podToNodeCidrpolicy{},
 		northSouthLoadbalancingWithL7Policy{},
 		echoIngressL7{},

connectivity/builder/egress_gateway_with_l7_policy.go

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package builder
+
+import (
+	_ "embed"
+	"github.com/cilium/cilium-cli/connectivity/check"
+	"github.com/cilium/cilium-cli/connectivity/tests"
+	"github.com/cilium/cilium-cli/utils/features"
+	"github.com/cilium/cilium/pkg/versioncheck"
+)
+
+//go:embed manifests/client-egress-icmp.yaml
+var clientEgressICMPYAML string
+
+//go:embed manifests/client-egress-l7-http-external-node.yaml
+var clientEgressL7HTTPExternalYAML string
+
+type egressGatewayWithL7Policy struct{}
+
+func (t egressGatewayWithL7Policy) build(ct *check.ConnectivityTest, templates map[string]string) {
+	newTest("egress-gateway-with-l7-policy", ct).
+		WithCondition(func() bool {
+			return versioncheck.MustCompile(">=1.16.0")(ct.CiliumVersion) && ct.Params().IncludeUnsafeTests
+		}).
+		WithCiliumPolicy(clientEgressICMPYAML).
+		WithCiliumPolicy(clientEgressOnlyDNSPolicyYAML).  // DNS resolution only
+		WithCiliumPolicy(clientEgressL7HTTPExternalYAML). // L7 allow policy with HTTP introspection
+		WithCiliumEgressGatewayPolicy(check.CiliumEgressGatewayPolicyParams{
+			Name:            "cegp-sample-client",
+			PodSelectorKind: "client",
+		}).
+		WithCiliumEgressGatewayPolicy(check.CiliumEgressGatewayPolicyParams{
+			Name:            "cegp-sample-echo",
+			PodSelectorKind: "echo",
+		}).
+		WithIPRoutesFromOutsideToPodCIDRs().
+		WithFeatureRequirements(
+			features.RequireEnabled(features.EgressGateway),
+			features.RequireEnabled(features.L7Proxy),
+			features.RequireEnabled(features.NodeWithoutCilium),
+		).
+		WithScenarios(tests.EgressGateway())
+}

connectivity/builder/manifests/client-egress-icmp.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: "cilium.io/v2"
+kind: CiliumNetworkPolicy
+metadata:
+  name: client-egress-icmp
+spec:
+  description: "Allow clients to send ICMP"
+  endpointSelector:
+    matchLabels:
+      kind: client
+  egress:
+  - icmps:
+    - fields:
+      - type: 8
+        family: IPv4
+      - type: 128
+        family: IPv6

connectivity/builder/manifests/client-egress-l7-http-external-node.yaml

@@ -0,0 +1,26 @@
+---
+# All clients are allowed to contact
+# echo-external-node.cilium-test.svc.cluster.local/client-ip
+# on port http-8080.
+# The toFQDNs section relies on DNS introspection being performed by
+# the client-egress-only-dns policy.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: client-egress-l7-http-external-node
+spec:
+  description: "Allow GET echo-external-node.cilium-test.svc.cluster.local:8080/client-ip"
+  endpointSelector:
+    matchLabels:
+      any:kind: client
+  egress:
+  - toFQDNs:
+    - matchName: "echo-external-node.cilium-test.svc.cluster.local"
+    toPorts:
+    - ports:
+      - port: "8080"
+        protocol: TCP
+      rules:
+        http:
+        - method: GET
+          path: /client-ip

connectivity/check/context.go

@@ -58,18 +58,19 @@ type ConnectivityTest struct {
 	// Clients for source and destination clusters.
 	clients *deploymentClients
 
-	ciliumPods        map[string]Pod
-	echoPods          map[string]Pod
-	echoExternalPods  map[string]Pod
-	clientPods        map[string]Pod
-	clientCPPods      map[string]Pod
-	perfClientPods    []Pod
-	perfServerPod     []Pod
-	PerfResults       []common.PerfSummary
-	echoServices      map[string]Service
-	ingressService    map[string]Service
-	k8sService        Service
-	externalWorkloads map[string]ExternalWorkload
+	ciliumPods           map[string]Pod
+	echoPods             map[string]Pod
+	echoExternalPods     map[string]Pod
+	clientPods           map[string]Pod
+	clientCPPods         map[string]Pod
+	perfClientPods       []Pod
+	perfServerPod        []Pod
+	PerfResults          []common.PerfSummary
+	echoServices         map[string]Service
+	echoExternalServices map[string]Service
+	ingressService       map[string]Service
+	k8sService           Service
+	externalWorkloads    map[string]ExternalWorkload
 
 	hostNetNSPodsByNode      map[string]Pod
 	secondaryNetworkNodeIPv4 map[string]string // node name => secondary ip
@@ -207,6 +208,7 @@ func NewConnectivityTest(client *k8s.Client, p Parameters, version string, logge
 		perfServerPod:            []Pod{},
 		PerfResults:              []common.PerfSummary{},
 		echoServices:             make(map[string]Service),
+		echoExternalServices:     make(map[string]Service),
 		ingressService:           make(map[string]Service),
 		externalWorkloads:        make(map[string]ExternalWorkload),
 		hostNetNSPodsByNode:      make(map[string]Pod),
@@ -1131,6 +1133,10 @@ func (ct *ConnectivityTest) EchoServicesAll() map[string]Service {
 	return ct.echoServices
 }
 
+func (ct *ConnectivityTest) EchoExternalServices() map[string]Service {
+	return ct.echoExternalServices
+}
+
 func (ct *ConnectivityTest) ExternalEchoPods() map[string]Pod {
 	return ct.echoExternalPods
 }

connectivity/check/deployment.go

@@ -905,6 +905,16 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error {
 				if err != nil {
 					return fmt.Errorf("unable to create deployment %s: %s", echoExternalNodeDeploymentName, err)
 				}
+
+				svc := newService(echoExternalNodeDeploymentName,
+					map[string]string{"name": echoExternalNodeDeploymentName, "kind": kindEchoExternalNodeName},
+					map[string]string{"kind": kindEchoExternalNodeName}, "http", 8080)
+				svc.Spec.ClusterIP = corev1.ClusterIPNone
+				svc.Spec.Type = corev1.ServiceTypeClusterIP
+				_, err := ct.clients.src.CreateService(ctx, ct.params.TestNamespace, svc, metav1.CreateOptions{})
+				if err != nil {
+					return fmt.Errorf("unable to create service %s: %w", echoExternalNodeDeploymentName, err)
+				}
 			}
 		} else {
 			ct.Infof("Skipping tests that require a node Without Cilium")
@@ -1243,6 +1253,17 @@ func (ct *ConnectivityTest) validateDeployment(ctx context.Context) error {
 				port:      uint32(ct.Params().ExternalDeploymentPort), // listen port of the echo server inside the container
 			}
 		}
+
+		echoExternalServices, err := ct.clients.dst.ListServices(ctx, ct.params.TestNamespace, metav1.ListOptions{LabelSelector: "kind=" + kindEchoExternalNodeName})
+		if err != nil {
+			return fmt.Errorf("unable to list echo external services: %w", err)
+		}
+
+		for _, echoExternalService := range echoExternalServices.Items {
+			ct.echoExternalServices[echoExternalService.Name] = Service{
+				Service: echoExternalService.DeepCopy(),
+			}
+		}
 	}
 
 	for _, cp := range ct.clientPods {

connectivity/check/peer.go

@@ -201,7 +201,7 @@ func (s Service) Path() string {
 // Address returns the network address of the Service.
 func (s Service) Address(family features.IPFamily) string {
 	// If the cluster IP is empty (headless service case) or the IP family is set to any, return the service name
-	if s.Service.Spec.ClusterIP == "" || family == features.IPFamilyAny {
+	if s.Service.Spec.ClusterIP == "" || s.Service.Spec.ClusterIP == corev1.ClusterIPNone || family == features.IPFamilyAny {
 		return fmt.Sprintf("%s.%s", s.Service.Name, s.Service.Namespace)
 	}
 
@@ -256,6 +256,12 @@ func (s Service) ToNodeportService(node *corev1.Node) NodeportService {
 	}
 }
 
+func (s Service) ToEchoIPService() EchoIPService {
+	return EchoIPService{
+		Service: s,
+	}
+}
+
 // NodeportService wraps a Service and exposes it through its nodeport, acting as a peer in a connectivity test.
 // It implements interface TestPeer.
 type NodeportService struct {
@@ -499,3 +505,11 @@ type EchoIPPod struct {
 func (p EchoIPPod) Path() string {
 	return p.path + "/client-ip"
 }
+
+type EchoIPService struct {
+	Service
+}
+
+func (s EchoIPService) Path() string {
+	return s.URLPath + "/client-ip"
+}

connectivity/tests/egressgateway.go

@@ -235,6 +235,26 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) {
 		i++
 	}
 
+	// Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service using DNS)
+	i = 0
+	for _, client := range ct.ClientPods() {
+		client := client
+
+		for _, externalEchoSvc := range ct.EchoExternalServices() {
+			externalEcho := externalEchoSvc.ToEchoIPService()
+
+			t.NewAction(s, fmt.Sprintf("curl-external-echo-service-%d", i), &client, externalEcho, features.IPFamilyV4).Run(func(a *check.Action) {
+				a.ExecInPod(ctx, ct.CurlCommandWithOutput(externalEcho, features.IPFamilyV4, "-4"))
+				clientIP := extractClientIPFromResponse(a.CmdOutput())
+
+				if !clientIP.Equal(egressGatewayNodeInternalIP) {
+					t.Fatal("Request reached external echo service with wrong source IP")
+				}
+			})
+			i++
+		}
+	}
+
 	// Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service)
 	i = 0
 	for _, client := range ct.ClientPods() {