test: add e2e testcases for Gateway with NodeLevel sidecar (#578)

.github/workflows/main.yml

@@ -253,7 +253,7 @@ jobs:
     strategy:
       matrix:
         focus: [ "" ]
-        bucket: [ 6 ]
+        bucket: [ 6, 7 ]
         k8s:
           - version: "latest"
             os: ubuntu-22.04
@@ -336,7 +336,7 @@ jobs:
       matrix:
         k8s_version: [ "" ]
         focus: [ "" ]
-        bucket: [ 7 ]
+        bucket: [ 10 ]
     env:
       CTR_TAG: ${{ github.sha }}
       CTR_REGISTRY: "localhost:5000"

.mergify.yml

@@ -25,9 +25,13 @@ pull_request_rules:
       - check-success=Go gateway test e2e (6, v1.19.16-k3s1, ubuntu-20.04)
       - check-success=Go gateway test e2e (6, v1.21.16-k3s1, ubuntu-22.04)
       - check-success=Go gateway test e2e (6, v1.23.17-k3s1, ubuntu-22.04)
-      - check-success=Go ingress test e2e (7)
+      - check-success=Go gateway test e2e (7, latest, ubuntu-22.04)
+      - check-success=Go gateway test e2e (7, v1.19.16-k3s1, ubuntu-20.04)
+      - check-success=Go gateway test e2e (7, v1.21.16-k3s1, ubuntu-22.04)
+      - check-success=Go gateway test e2e (7, v1.23.17-k3s1, ubuntu-22.04)
       - check-success=Go statefulset test e2e (8)
       - check-success=Go retry test e2e (9)
+      - check-success=Go ingress test e2e (10)
       - "#approved-reviews-by >= 1"
       - or:
           - and:

tests/e2e/e2e_fsm_ingress_test.go

@@ -21,7 +21,7 @@ const nsIngress = "test"
 var _ = FSMDescribe("Test traffic among FSM Ingress",
 	FSMDescribeInfo{
 		Tier:   1,
-		Bucket: 7,
+		Bucket: 10,
 		OS:     OSCrossPlatform,
 	},
 	func() {

tests/e2e/e2e_fsm_reinstall_test.go

@@ -30,7 +30,7 @@ var _ = FSMDescribe("Test reinstalling FSM in the same namespace with the same m
 			By("Reinstalling FSM")
 			// Invoke the CLI directly because Td.InstallFSM unconditionally
 			// creates the namespace which fails when it already exists.
-			stdout, stderr, err = Td.RunLocal(filepath.FromSlash("../../bin/fsm"), "install", "--verbose", "--timeout=5m", "--fsm-namespace", opts.ControlPlaneNS, "--set", "fsm.image.registry="+opts.ContainerRegistryLoc+",fsm.image.tag="+opts.FsmImagetag, "--set", "fsm.fsmIngress.enabled=false")
+			stdout, stderr, err = Td.RunLocal(filepath.FromSlash("../../bin/fsm"), "install", "--verbose", "--timeout=5m", "--fsm-namespace", opts.ControlPlaneNS, "--set", "fsm.image.registry="+opts.ContainerRegistryLoc+",fsm.image.tag="+opts.FsmImageTag, "--set", "fsm.fsmIngress.enabled=false")
 			Td.T.Log(stdout)
 			if err != nil {
 				Td.T.Logf("stderr:\n%s", stderr)

tests/e2e/e2e_gatewayapi_test.go

@@ -36,7 +36,7 @@ const (
 	nsUDPSvc    = "udp"
 )
 
-var _ = FSMDescribe("Test traffic among FSM Gateway",
+var _ = FSMDescribe("Test traffic routing by FSM Gateway with trafficInterceptionMode(PodLevel)",
 	FSMDescribeInfo{
 		Tier:   1,
 		Bucket: 6,
@@ -45,40 +45,59 @@ var _ = FSMDescribe("Test traffic among FSM Gateway",
 	func() {
 		Context("Test traffic from client to backend service routing by FSM Gateway", func() {
 			It("allow traffic of multiple protocols through Gateway", func() {
-				// Install FSM
-				installOpts := Td.GetFSMInstallOpts()
-				installOpts.EnableIngress = false
-				installOpts.EnableGateway = true
-				installOpts.EnableServiceLB = Td.InstType == KindCluster
-
-				Expect(Td.InstallFSM(installOpts)).To(Succeed())
-				Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, &metav1.LabelSelector{
-					MatchLabels: map[string]string{
-						"app.kubernetes.io/instance": "fsm",
-						"app.kubernetes.io/name":     "flomesh.io",
-					},
-				})).To(Succeed())
-
-				testDeployFSMGateway()
-
-				testFSMGatewayHTTPTrafficSameNamespace()
-				testFSMGatewayHTTPTrafficCrossNamespace()
-				testFSMGatewayHTTPSTraffic()
-				testFSMGatewayTLSTerminate()
-				testFSMGatewayTLSPassthrough()
-
-				testFSMGatewayGRPCTrafficSameNamespace()
-				testFSMGatewayGRPCTrafficCrossNamespace()
-				testFSMGatewayGRPCSTraffic()
+				testGatewayAPI(constants.TrafficInterceptionModePodLevel)
+			})
+		})
+	})
 
-				testFSMGatewayTCPTrafficSameNamespace()
-				testFSMGatewayTCPTrafficCrossNamespace()
-				testFSMGatewayUDPTrafficSameNamespace()
-				testFSMGatewayUDPTrafficCrossNamespace()
+var _ = FSMDescribe("Test traffic routing by FSM Gateway with trafficInterceptionMode(NodeLevel)",
+	FSMDescribeInfo{
+		Tier:   1,
+		Bucket: 7,
+		OS:     OSCrossPlatform,
+	},
+	func() {
+		Context("Test traffic from client to backend service routing by FSM Gateway", func() {
+			It("allow traffic of multiple protocols through Gateway", func() {
+				testGatewayAPI(constants.TrafficInterceptionModeNodeLevel)
 			})
 		})
 	})
 
+func testGatewayAPI(trafficInterceptionMode string) {
+	// Install FSM
+	installOpts := Td.GetFSMInstallOpts()
+	installOpts.EnableIngress = false
+	installOpts.EnableGateway = true
+	installOpts.EnableServiceLB = Td.InstType == KindCluster
+	installOpts.TrafficInterceptionMode = trafficInterceptionMode
+
+	Expect(Td.InstallFSM(installOpts)).To(Succeed())
+	Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app.kubernetes.io/instance": "fsm",
+			"app.kubernetes.io/name":     "flomesh.io",
+		},
+	})).To(Succeed())
+
+	testDeployFSMGateway()
+
+	testFSMGatewayHTTPTrafficSameNamespace()
+	testFSMGatewayHTTPTrafficCrossNamespace()
+	testFSMGatewayHTTPSTraffic()
+	testFSMGatewayTLSTerminate()
+	testFSMGatewayTLSPassthrough()
+
+	testFSMGatewayGRPCTrafficSameNamespace()
+	testFSMGatewayGRPCTrafficCrossNamespace()
+	testFSMGatewayGRPCSTraffic()
+
+	testFSMGatewayTCPTrafficSameNamespace()
+	testFSMGatewayTCPTrafficCrossNamespace()
+	testFSMGatewayUDPTrafficSameNamespace()
+	testFSMGatewayUDPTrafficCrossNamespace()
+}
+
 func testDeployFSMGateway() {
 	// Create namespaces
 	Expect(Td.CreateNs(nsGateway, nil)).To(Succeed())
@@ -1554,7 +1573,6 @@ func testFSMGatewayGRPCSTraffic() {
 	}, 5, Td.ReqSuccessTimeout)
 
 	Expect(cond).To(BeTrue(), "Failed testing GRPCs traffic from grpcurl(localhost) to destination %s/%s", grpcReq.Destination, grpcReq.Symbol)
-
 }
 
 func testFSMGatewayTLSPassthrough() {

tests/e2e/e2e_namespacedingress_test.go

@@ -14,7 +14,7 @@ import (
 var _ = FSMDescribe("Test traffic among FSM NamespacedIngress",
 	FSMDescribeInfo{
 		Tier:   2,
-		Bucket: 7,
+		Bucket: 10,
 		OS:     OSCrossPlatform,
 	},
 	func() {

tests/framework/common.go

@@ -101,6 +101,7 @@ var imageNames = []string{
 	"fsm-ingress",
 	"fsm-gateway",
 	"fsm-curl",
+	"fsm-xnetmgmt",
 }
 
 // Since parseFlags is global, this is the Ginkgo way to do it.
@@ -567,7 +568,7 @@ func (td *FsmTestData) GetFSMInstallOpts(options ...InstallFsmOpt) InstallFSMOpt
 		CertManager:             defaultCertManager,
 		ContainerRegistryLoc:    td.CtrRegistryServer,
 		ContainerRegistrySecret: td.CtrRegistryPassword,
-		FsmImagetag:             td.FsmImageTag,
+		FsmImageTag:             td.FsmImageTag,
 		DeployGrafana:           false,
 		DeployPrometheus:        false,
 		DeployJaeger:            false,
@@ -769,7 +770,7 @@ func (td *FsmTestData) InstallFSM(instOpts InstallFSMOpts) error {
 
 	instOpts.SetOverrides = append(instOpts.SetOverrides,
 		fmt.Sprintf("fsm.image.registry=%s", instOpts.ContainerRegistryLoc),
-		fmt.Sprintf("fsm.image.tag=%s", instOpts.FsmImagetag),
+		fmt.Sprintf("fsm.image.tag=%s", instOpts.FsmImageTag),
 		fmt.Sprintf("fsm.certificateProvider.kind=%s", instOpts.CertManager),
 		fmt.Sprintf("fsm.enableEgress=%v", instOpts.EgressEnabled),
 		fmt.Sprintf("fsm.enablePermissiveTrafficPolicy=%v", instOpts.EnablePermissiveMode),
@@ -794,6 +795,11 @@ func (td *FsmTestData) InstallFSM(instOpts InstallFSMOpts) error {
 		fmt.Sprintf("fsm.egressGateway.enabled=%v", instOpts.EnableEgressGateway),
 	)
 
+	if instOpts.TrafficInterceptionMode != "" &&
+		(instOpts.TrafficInterceptionMode == constants.TrafficInterceptionModeNodeLevel || instOpts.TrafficInterceptionMode == constants.TrafficInterceptionModePodLevel) {
+		instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("fsm.trafficInterceptionMode=%v", instOpts.TrafficInterceptionMode))
+	}
+
 	if instOpts.LocalProxyMode != "" {
 		instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("fsm.localProxyMode=%s", instOpts.LocalProxyMode))
 	}

tests/framework/common_apps.go

@@ -597,7 +597,7 @@ func (td *FsmTestData) GetOSSpecificTCPEchoPod(podName string, namespace string,
 	var image string
 	var command string
 	installOpts := Td.GetFSMInstallOpts()
-	image = fmt.Sprintf("%s/fsm-demo-tcp-echo-server:%s", installOpts.ContainerRegistryLoc, installOpts.FsmImagetag)
+	image = fmt.Sprintf("%s/fsm-demo-tcp-echo-server:%s", installOpts.ContainerRegistryLoc, installOpts.FsmImageTag)
 	command = "/tcp-echo-server"
 	return Td.SimplePodApp(
 		SimplePodAppDef{

tests/framework/types.go

@@ -107,7 +107,7 @@ type InstallFSMOpts struct {
 	CertManager             string
 	ContainerRegistryLoc    string
 	ContainerRegistrySecret string
-	FsmImagetag             string
+	FsmImageTag             string
 	DeployGrafana           bool
 	DeployPrometheus        bool
 	DeployJaeger            bool
@@ -150,6 +150,8 @@ type InstallFSMOpts struct {
 	EnableIngressBackendPolicy    bool
 	EnableAccessControlPolicy     bool
 	EnableRetryPolicy             bool
+
+	TrafficInterceptionMode string
 }
 
 // InstallFsmOpt is a function type for setting install options