-
Notifications
You must be signed in to change notification settings - Fork 1.6k
E2E: Splits maglev test into two tests: IPv4 & IPv6 #11801
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
bef4902
685bc73
48343f6
05c53cd
e20c8f3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,6 +54,8 @@ var _ = describe.CalicoDescribe( | |
| describe.WithFeature("Maglev"), | ||
| describe.WithCategory(describe.Networking), | ||
| describe.WithExternalNode(), | ||
| describe.WithDataplane(describe.BPF), | ||
| describe.WithAWS(), | ||
| "Maglev load balancing tests", | ||
| func() { | ||
| var ( | ||
|
|
@@ -104,82 +106,89 @@ var _ = describe.CalicoDescribe( | |
| framework.Logf("Node name to IPv6 mapping: %v", maglevTests.nodeNameToIPv6) | ||
| }) | ||
|
|
||
| It("test service ip load balancing behavior before and after maglev annotation", func() { | ||
| // Ensure we have at least 3 nodes for the test | ||
| Expect(len(nodeNames)).Should(BeNumerically(">=", 3), "Need at least 3 nodes for this test") | ||
|
|
||
| // Deploy 20 backend pods on node 1 (first node) | ||
| maglevTests.DeployBackendPods(20, []string{nodeNames[0]}) | ||
| // Deploy service "netexec" backed by the 20 pods | ||
| maglevTests.DeployService() | ||
| // Add route to external node where packets to service cluster IP go to node 2 | ||
| maglevTests.SetupExternalNodeClientRoutingToSpecificNode(extNode, nodeNames[1]) // node 2 (second node) | ||
|
|
||
| // Test random backend selection without Maglev annotation for both IPv4 and IPv6 | ||
| maglevTests.TestRandomBackendSelection(extNode, false) // IPv4 test | ||
| maglevTests.TestRandomBackendSelection(extNode, true) // IPv6 test | ||
|
|
||
| // Enable Maglev on the same service by adding annotation | ||
| maglevTests.EnableMaglev() | ||
|
|
||
| // Set a fixed source port for consistent hashing tests | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Test Maglev consistent hashing with the annotation for both IPv4 and IPv6 using first source port | ||
| backendViaNode2IPv4Port1 := maglevTests.TestMaglevConsistentHashing(extNode, false) // IPv4 test with port 12345 | ||
| backendViaNode2IPv6Port1 := maglevTests.TestMaglevConsistentHashing(extNode, true) // IPv6 test with port 12345 | ||
|
|
||
| // Test Maglev consistent hashing with a different source port to verify different flows can hash to different backends | ||
| maglevTests.SetSourcePort(23456) // Change source port | ||
| backendViaNode2IPv4Port2 := maglevTests.TestMaglevConsistentHashing(extNode, false) // IPv4 test with port 23456 | ||
| backendViaNode2IPv6Port2 := maglevTests.TestMaglevConsistentHashing(extNode, true) // IPv6 test with port 23456 | ||
| framework.Logf("Maglev hashing with different source ports: IPv4 port 12345->%s, port 23456->%s; IPv6 port 12345->%s, port 23456->%s", | ||
| backendViaNode2IPv4Port1, backendViaNode2IPv4Port2, backendViaNode2IPv6Port1, backendViaNode2IPv6Port2) | ||
|
|
||
| // Reset source port for next tests | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Then: Remove the routes from external node to service cluster IPs via node 2 | ||
| maglevTests.RemoveExternalNodeClientRoutes(extNode, nodeNames[1]) | ||
|
|
||
| // Add route to external node where packets to service cluster IP go to node 3 | ||
| maglevTests.SetupExternalNodeClientRoutingToSpecificNode(extNode, nodeNames[2]) // node 3 (third node) | ||
|
|
||
| // Test Maglev consistent hashing again for both IPv4 and IPv6 via node 3 with first source port | ||
| backendViaNode3IPv4Port1 := maglevTests.TestMaglevConsistentHashing(extNode, false) // IPv4 test via node 3 with port 12345 | ||
| backendViaNode3IPv6Port1 := maglevTests.TestMaglevConsistentHashing(extNode, true) // IPv6 test via node 3 with port 12345 | ||
|
|
||
| // Test Maglev consistent hashing via node 3 with a different source port | ||
| maglevTests.SetSourcePort(23456) // Change source port | ||
| backendViaNode3IPv4Port2 := maglevTests.TestMaglevConsistentHashing(extNode, false) // IPv4 test via node 3 with port 23456 | ||
| backendViaNode3IPv6Port2 := maglevTests.TestMaglevConsistentHashing(extNode, true) // IPv6 test via node 3 with port 23456 | ||
| framework.Logf("Maglev hashing via node 3 with different source ports: IPv4 port 12345->%s, port 23456->%s; IPv6 port 12345->%s, port 23456->%s", | ||
| backendViaNode3IPv4Port1, backendViaNode3IPv4Port2, backendViaNode3IPv6Port1, backendViaNode3IPv6Port2) | ||
|
|
||
| // Reset source port | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Assert that the backend selected via node 3 is the same as that selected via node 2 (for same source port) | ||
| Expect(backendViaNode3IPv4Port1).Should(Equal(backendViaNode2IPv4Port1), | ||
| fmt.Sprintf("Expected IPv4 backend selection to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2IPv4Port1, backendViaNode3IPv4Port1)) | ||
|
|
||
| Expect(backendViaNode3IPv6Port1).Should(Equal(backendViaNode2IPv6Port1), | ||
| fmt.Sprintf("Expected IPv6 backend selection to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2IPv6Port1, backendViaNode3IPv6Port1)) | ||
|
|
||
| // Also verify that the second source port routes to the same backend across nodes | ||
| Expect(backendViaNode3IPv4Port2).Should(Equal(backendViaNode2IPv4Port2), | ||
| fmt.Sprintf("Expected IPv4 backend selection (port 23456) to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2IPv4Port2, backendViaNode3IPv4Port2)) | ||
|
|
||
| Expect(backendViaNode3IPv6Port2).Should(Equal(backendViaNode2IPv6Port2), | ||
| fmt.Sprintf("Expected IPv6 backend selection (port 23456) to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2IPv6Port2, backendViaNode3IPv6Port2)) | ||
|
|
||
| framework.Logf("Maglev cross-node consistency verified for both source ports: IPv4 port 12345->%s, port 23456->%s; IPv6 port 12345->%s, port 23456->%s", | ||
| backendViaNode2IPv4Port1, backendViaNode2IPv4Port2, backendViaNode2IPv6Port1, backendViaNode2IPv6Port2) | ||
| }) | ||
| makeMaglevTest := func(isIPv6 bool) func() { | ||
|
|
||
| ipVer := "IPv4" | ||
| if isIPv6 { | ||
| ipVer = "IPv6" | ||
| } | ||
| return func() { | ||
| if isIPv6 { | ||
| if len(maglevTests.nodeNameToIPv6) == 0 { | ||
| Skip("IPv6 is not configured, skipping IPv6 Maglev test") | ||
| } | ||
| } else { | ||
| if len(maglevTests.nodeNameToIPv4) == 0 { | ||
| Skip("IPv4 is not configured, skipping IPv4 Maglev test") | ||
| } | ||
| } | ||
| // Ensure we have at least 3 nodes for the test | ||
| Expect(len(nodeNames)).Should(BeNumerically(">=", 3), "Need at least 3 nodes for this test") | ||
|
|
||
| // Deploy 20 backend pods on node 1 (first node) | ||
| maglevTests.DeployBackendPods(20, []string{nodeNames[0]}) | ||
| // Deploy service "netexec" backed by the 20 pods | ||
| maglevTests.DeployService() | ||
| // Add route to external node where packets to service cluster IP go to node 2 | ||
| maglevTests.SetupExternalNodeClientRoutingToSpecificNode(extNode, nodeNames[1]) // node 2 (second node) | ||
|
|
||
| // Test random backend selection without Maglev annotation | ||
| maglevTests.TestRandomBackendSelection(extNode, isIPv6) | ||
|
|
||
| // Enable Maglev on the same service by adding annotation | ||
| maglevTests.EnableMaglev() | ||
|
|
||
| // Set a fixed source port for consistent hashing tests | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Test Maglev consistent hashing with the annotation using first source port | ||
| backendViaNode2Port1 := maglevTests.TestMaglevConsistentHashing(extNode, isIPv6) // test with port 12345 | ||
|
|
||
| // Test Maglev consistent hashing with a different source port to verify different flows can hash to different backends | ||
| maglevTests.SetSourcePort(23456) // Change source port | ||
| backendViaNode2Port2 := maglevTests.TestMaglevConsistentHashing(extNode, isIPv6) // test with port 23456 | ||
| framework.Logf("Maglev hashing with different source ports: %s port 12345->%s, port 23456->%s", | ||
| ipVer, backendViaNode2Port1, backendViaNode2Port2) | ||
|
|
||
| // Reset source port for next tests | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Then: Remove the routes from external node to service cluster IPs via node 2 | ||
| maglevTests.RemoveExternalNodeClientRoutes(extNode, nodeNames[1]) | ||
|
|
||
| // Add route to external node where packets to service cluster IP go to node 3 | ||
| maglevTests.SetupExternalNodeClientRoutingToSpecificNode(extNode, nodeNames[2]) // node 3 (third node) | ||
|
|
||
| // Test Maglev consistent hashing again via node 3 with first source port | ||
| backendViaNode3Port1 := maglevTests.TestMaglevConsistentHashing(extNode, isIPv6) // test via node 3 with port 12345 | ||
|
|
||
| // Test Maglev consistent hashing via node 3 with a different source port | ||
| maglevTests.SetSourcePort(23456) // Change source port | ||
| backendViaNode3Port2 := maglevTests.TestMaglevConsistentHashing(extNode, isIPv6) // test via node 3 with port 23456 | ||
| framework.Logf("Maglev hashing via node 3 with different source ports: %s port 12345->%s, port 23456->%s", | ||
| ipVer, backendViaNode3Port1, backendViaNode3Port2) | ||
|
|
||
| // Reset source port | ||
| maglevTests.SetSourcePort(12345) | ||
|
|
||
| // Assert that the backend selected via node 3 is the same as that selected via node 2 (for same source port) | ||
| Expect(backendViaNode3Port1).Should(Equal(backendViaNode2Port1), | ||
| fmt.Sprintf("Expected backend selection to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2Port1, backendViaNode3Port1)) | ||
|
|
||
| // Also verify that the second source port routes to the same backend across nodes | ||
| Expect(backendViaNode3Port2).Should(Equal(backendViaNode2Port2), | ||
| fmt.Sprintf("Expected backend selection (port 23456) to be consistent across nodes: node 2 selected %s, node 3 selected %s", | ||
| backendViaNode2Port2, backendViaNode3Port2)) | ||
|
Comment on lines
+175
to
+182
|
||
|
|
||
| framework.Logf("Maglev cross-node consistency verified for both source ports: %s port 12345->%s, port 23456->%s", | ||
| ipVer, backendViaNode2Port1, backendViaNode2Port2) | ||
| } | ||
| } | ||
|
|
||
| It("test service ip load balancing behavior before and after maglev annotation (IPv4)", makeMaglevTest(false)) | ||
| It("test service ip load balancing behavior before and after maglev annotation (IPv6)", makeMaglevTest(true)) | ||
|
|
||
| }) | ||
|
|
||
| type MaglevTests struct { | ||
|
|
@@ -215,6 +224,22 @@ func NewMaglevTests(f *framework.Framework) *MaglevTests { | |
| } | ||
| } | ||
|
|
||
| // IPFamilies returns a list of families compatible with this cluster's configuration - for use in service creation. | ||
| func (m *MaglevTests) IPFamilies() []v1.IPFamily { | ||
| families := make([]v1.IPFamily, 0) | ||
| if len(m.nodeNameToIPv4) > 0 { | ||
| framework.Logf("Found IPv4 node IPs; Adding IPv4Protocol to IPFamily list") | ||
| families = append(families, v1.IPv4Protocol) | ||
| } | ||
|
|
||
| if len(m.nodeNameToIPv6) > 0 { | ||
| framework.Logf("Found IPv6 node IPs; Adding IPv6Protocol to IPFamily list") | ||
| families = append(families, v1.IPv6Protocol) | ||
| } | ||
|
|
||
| return families | ||
| } | ||
|
|
||
| // parseBackendResponse parses the JSON response from netexec and returns the backend pod name | ||
| func (m *MaglevTests) parseBackendResponse(output string) (string, error) { | ||
| // NetexecResponse represents the JSON response from the netexec service | ||
|
|
@@ -304,7 +329,7 @@ func (m *MaglevTests) DeployBackendPods(numPods int, nodes []string) { | |
| } | ||
|
|
||
| func (m *MaglevTests) DeployService() { | ||
| By("deploying dual-stack service") | ||
| By("deploying service") | ||
|
|
||
| service := &v1.Service{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
|
|
@@ -313,7 +338,7 @@ func (m *MaglevTests) DeployService() { | |
| }, | ||
| Spec: v1.ServiceSpec{ | ||
| Type: v1.ServiceTypeClusterIP, | ||
| IPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}, | ||
| IPFamilies: m.IPFamilies(), | ||
| IPFamilyPolicy: ptr.To(v1.IPFamilyPolicyPreferDualStack), | ||
| Selector: map[string]string{ | ||
| "app": "netexec", | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This PR introduces new test selectors (
WithDataplane(describe.BPF)andWithAWS()) that significantly narrow where the test runs, but that scope change isn’t mentioned in the PR description. If Maglev testing is not strictly BPF+AWS-only, consider removing or justifying these constraints so coverage isn’t unintentionally reduced.Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The Maglev feature is BPF-only, and we know the AWS fabric can support the Maglev traffic from external node to cluster-nodes