From 8f9c3518845b3ec7209d63c0f11d9e93160627b7 Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Tue, 7 Jan 2025 07:15:58 +0800 Subject: [PATCH] test: add e2e testcases for Gateway with NodeLevel sidecar (#578) --- .github/workflows/main.yml | 4 +- .mergify.yml | 6 +- tests/e2e/e2e_fsm_ingress_test.go | 2 +- tests/e2e/e2e_fsm_reinstall_test.go | 2 +- tests/e2e/e2e_gatewayapi_test.go | 80 +++++++++++++++---------- tests/e2e/e2e_namespacedingress_test.go | 2 +- tests/framework/common.go | 10 +++- tests/framework/common_apps.go | 2 +- tests/framework/types.go | 4 +- 9 files changed, 71 insertions(+), 41 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a1612899..1caa4c248 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -253,7 +253,7 @@ jobs: strategy: matrix: focus: [ "" ] - bucket: [ 6 ] + bucket: [ 6, 7 ] k8s: - version: "latest" os: ubuntu-22.04 @@ -336,7 +336,7 @@ jobs: matrix: k8s_version: [ "" ] focus: [ "" ] - bucket: [ 7 ] + bucket: [ 10 ] env: CTR_TAG: ${{ github.sha }} CTR_REGISTRY: "localhost:5000" diff --git a/.mergify.yml b/.mergify.yml index f6837bbe7..3ae2a85e4 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -25,9 +25,13 @@ pull_request_rules: - check-success=Go gateway test e2e (6, v1.19.16-k3s1, ubuntu-20.04) - check-success=Go gateway test e2e (6, v1.21.16-k3s1, ubuntu-22.04) - check-success=Go gateway test e2e (6, v1.23.17-k3s1, ubuntu-22.04) - - check-success=Go ingress test e2e (7) + - check-success=Go gateway test e2e (7, latest, ubuntu-22.04) + - check-success=Go gateway test e2e (7, v1.19.16-k3s1, ubuntu-20.04) + - check-success=Go gateway test e2e (7, v1.21.16-k3s1, ubuntu-22.04) + - check-success=Go gateway test e2e (7, v1.23.17-k3s1, ubuntu-22.04) - check-success=Go statefulset test e2e (8) - check-success=Go retry test e2e (9) + - check-success=Go ingress test e2e (10) - "#approved-reviews-by >= 1" - or: - and: diff --git a/tests/e2e/e2e_fsm_ingress_test.go b/tests/e2e/e2e_fsm_ingress_test.go index 05747f639..833b424fb 100644 --- a/tests/e2e/e2e_fsm_ingress_test.go +++ b/tests/e2e/e2e_fsm_ingress_test.go @@ -21,7 +21,7 @@ const nsIngress = "test" var _ = FSMDescribe("Test traffic among FSM Ingress", FSMDescribeInfo{ Tier: 1, - Bucket: 7, + Bucket: 10, OS: OSCrossPlatform, }, func() { diff --git a/tests/e2e/e2e_fsm_reinstall_test.go b/tests/e2e/e2e_fsm_reinstall_test.go index 985a5db1e..254a0b29f 100644 --- a/tests/e2e/e2e_fsm_reinstall_test.go +++ b/tests/e2e/e2e_fsm_reinstall_test.go @@ -30,7 +30,7 @@ var _ = FSMDescribe("Test reinstalling FSM in the same namespace with the same m By("Reinstalling FSM") // Invoke the CLI directly because Td.InstallFSM unconditionally // creates the namespace which fails when it already exists. - stdout, stderr, err = Td.RunLocal(filepath.FromSlash("../../bin/fsm"), "install", "--verbose", "--timeout=5m", "--fsm-namespace", opts.ControlPlaneNS, "--set", "fsm.image.registry="+opts.ContainerRegistryLoc+",fsm.image.tag="+opts.FsmImagetag, "--set", "fsm.fsmIngress.enabled=false") + stdout, stderr, err = Td.RunLocal(filepath.FromSlash("../../bin/fsm"), "install", "--verbose", "--timeout=5m", "--fsm-namespace", opts.ControlPlaneNS, "--set", "fsm.image.registry="+opts.ContainerRegistryLoc+",fsm.image.tag="+opts.FsmImageTag, "--set", "fsm.fsmIngress.enabled=false") Td.T.Log(stdout) if err != nil { Td.T.Logf("stderr:\n%s", stderr) diff --git a/tests/e2e/e2e_gatewayapi_test.go b/tests/e2e/e2e_gatewayapi_test.go index 9f711eb1c..c4f7a5cc0 100644 --- a/tests/e2e/e2e_gatewayapi_test.go +++ b/tests/e2e/e2e_gatewayapi_test.go @@ -36,7 +36,7 @@ const ( nsUDPSvc = "udp" ) -var _ = FSMDescribe("Test traffic among FSM Gateway", +var _ = FSMDescribe("Test traffic routing by FSM Gateway with trafficInterceptionMode(PodLevel)", FSMDescribeInfo{ Tier: 1, Bucket: 6, @@ -45,40 +45,59 @@ var _ = FSMDescribe("Test traffic among FSM Gateway", func() { Context("Test traffic from client to backend service routing by FSM Gateway", func() { It("allow traffic of multiple protocols through Gateway", func() { - // Install FSM - installOpts := Td.GetFSMInstallOpts() - installOpts.EnableIngress = false - installOpts.EnableGateway = true - installOpts.EnableServiceLB = Td.InstType == KindCluster - - Expect(Td.InstallFSM(installOpts)).To(Succeed()) - Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/instance": "fsm", - "app.kubernetes.io/name": "flomesh.io", - }, - })).To(Succeed()) - - testDeployFSMGateway() - - testFSMGatewayHTTPTrafficSameNamespace() - testFSMGatewayHTTPTrafficCrossNamespace() - testFSMGatewayHTTPSTraffic() - testFSMGatewayTLSTerminate() - testFSMGatewayTLSPassthrough() - - testFSMGatewayGRPCTrafficSameNamespace() - testFSMGatewayGRPCTrafficCrossNamespace() - testFSMGatewayGRPCSTraffic() + testGatewayAPI(constants.TrafficInterceptionModePodLevel) + }) + }) + }) - testFSMGatewayTCPTrafficSameNamespace() - testFSMGatewayTCPTrafficCrossNamespace() - testFSMGatewayUDPTrafficSameNamespace() - testFSMGatewayUDPTrafficCrossNamespace() +var _ = FSMDescribe("Test traffic routing by FSM Gateway with trafficInterceptionMode(NodeLevel)", + FSMDescribeInfo{ + Tier: 1, + Bucket: 7, + OS: OSCrossPlatform, + }, + func() { + Context("Test traffic from client to backend service routing by FSM Gateway", func() { + It("allow traffic of multiple protocols through Gateway", func() { + testGatewayAPI(constants.TrafficInterceptionModeNodeLevel) }) }) }) +func testGatewayAPI(trafficInterceptionMode string) { + // Install FSM + installOpts := Td.GetFSMInstallOpts() + installOpts.EnableIngress = false + installOpts.EnableGateway = true + installOpts.EnableServiceLB = Td.InstType == KindCluster + installOpts.TrafficInterceptionMode = trafficInterceptionMode + + Expect(Td.InstallFSM(installOpts)).To(Succeed()) + Expect(Td.WaitForPodsRunningReady(Td.FsmNamespace, 3, &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/instance": "fsm", + "app.kubernetes.io/name": "flomesh.io", + }, + })).To(Succeed()) + + testDeployFSMGateway() + + testFSMGatewayHTTPTrafficSameNamespace() + testFSMGatewayHTTPTrafficCrossNamespace() + testFSMGatewayHTTPSTraffic() + testFSMGatewayTLSTerminate() + testFSMGatewayTLSPassthrough() + + testFSMGatewayGRPCTrafficSameNamespace() + testFSMGatewayGRPCTrafficCrossNamespace() + testFSMGatewayGRPCSTraffic() + + testFSMGatewayTCPTrafficSameNamespace() + testFSMGatewayTCPTrafficCrossNamespace() + testFSMGatewayUDPTrafficSameNamespace() + testFSMGatewayUDPTrafficCrossNamespace() +} + func testDeployFSMGateway() { // Create namespaces Expect(Td.CreateNs(nsGateway, nil)).To(Succeed()) @@ -1554,7 +1573,6 @@ func testFSMGatewayGRPCSTraffic() { }, 5, Td.ReqSuccessTimeout) Expect(cond).To(BeTrue(), "Failed testing GRPCs traffic from grpcurl(localhost) to destination %s/%s", grpcReq.Destination, grpcReq.Symbol) - } func testFSMGatewayTLSPassthrough() { diff --git a/tests/e2e/e2e_namespacedingress_test.go b/tests/e2e/e2e_namespacedingress_test.go index ad384d789..511225e55 100644 --- a/tests/e2e/e2e_namespacedingress_test.go +++ b/tests/e2e/e2e_namespacedingress_test.go @@ -14,7 +14,7 @@ import ( var _ = FSMDescribe("Test traffic among FSM NamespacedIngress", FSMDescribeInfo{ Tier: 2, - Bucket: 7, + Bucket: 10, OS: OSCrossPlatform, }, func() { diff --git a/tests/framework/common.go b/tests/framework/common.go index 14ced4dfe..1122b118e 100644 --- a/tests/framework/common.go +++ b/tests/framework/common.go @@ -101,6 +101,7 @@ var imageNames = []string{ "fsm-ingress", "fsm-gateway", "fsm-curl", + "fsm-xnetmgmt", } // Since parseFlags is global, this is the Ginkgo way to do it. @@ -567,7 +568,7 @@ func (td *FsmTestData) GetFSMInstallOpts(options ...InstallFsmOpt) InstallFSMOpt CertManager: defaultCertManager, ContainerRegistryLoc: td.CtrRegistryServer, ContainerRegistrySecret: td.CtrRegistryPassword, - FsmImagetag: td.FsmImageTag, + FsmImageTag: td.FsmImageTag, DeployGrafana: false, DeployPrometheus: false, DeployJaeger: false, @@ -769,7 +770,7 @@ func (td *FsmTestData) InstallFSM(instOpts InstallFSMOpts) error { instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("fsm.image.registry=%s", instOpts.ContainerRegistryLoc), - fmt.Sprintf("fsm.image.tag=%s", instOpts.FsmImagetag), + fmt.Sprintf("fsm.image.tag=%s", instOpts.FsmImageTag), fmt.Sprintf("fsm.certificateProvider.kind=%s", instOpts.CertManager), fmt.Sprintf("fsm.enableEgress=%v", instOpts.EgressEnabled), fmt.Sprintf("fsm.enablePermissiveTrafficPolicy=%v", instOpts.EnablePermissiveMode), @@ -794,6 +795,11 @@ func (td *FsmTestData) InstallFSM(instOpts InstallFSMOpts) error { fmt.Sprintf("fsm.egressGateway.enabled=%v", instOpts.EnableEgressGateway), ) + if instOpts.TrafficInterceptionMode != "" && + (instOpts.TrafficInterceptionMode == constants.TrafficInterceptionModeNodeLevel || instOpts.TrafficInterceptionMode == constants.TrafficInterceptionModePodLevel) { + instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("fsm.trafficInterceptionMode=%v", instOpts.TrafficInterceptionMode)) + } + if instOpts.LocalProxyMode != "" { instOpts.SetOverrides = append(instOpts.SetOverrides, fmt.Sprintf("fsm.localProxyMode=%s", instOpts.LocalProxyMode)) } diff --git a/tests/framework/common_apps.go b/tests/framework/common_apps.go index 47caa713a..663dc562c 100644 --- a/tests/framework/common_apps.go +++ b/tests/framework/common_apps.go @@ -597,7 +597,7 @@ func (td *FsmTestData) GetOSSpecificTCPEchoPod(podName string, namespace string, var image string var command string installOpts := Td.GetFSMInstallOpts() - image = fmt.Sprintf("%s/fsm-demo-tcp-echo-server:%s", installOpts.ContainerRegistryLoc, installOpts.FsmImagetag) + image = fmt.Sprintf("%s/fsm-demo-tcp-echo-server:%s", installOpts.ContainerRegistryLoc, installOpts.FsmImageTag) command = "/tcp-echo-server" return Td.SimplePodApp( SimplePodAppDef{ diff --git a/tests/framework/types.go b/tests/framework/types.go index 939a77e57..467a5ddb4 100644 --- a/tests/framework/types.go +++ b/tests/framework/types.go @@ -107,7 +107,7 @@ type InstallFSMOpts struct { CertManager string ContainerRegistryLoc string ContainerRegistrySecret string - FsmImagetag string + FsmImageTag string DeployGrafana bool DeployPrometheus bool DeployJaeger bool @@ -150,6 +150,8 @@ type InstallFSMOpts struct { EnableIngressBackendPolicy bool EnableAccessControlPolicy bool EnableRetryPolicy bool + + TrafficInterceptionMode string } // InstallFsmOpt is a function type for setting install options