Skip to content

Commit 61c2ee9

Browse files
zhangzujianzhangzujian
committed
refactor lb svc e2e
Signed-off-by: zhangzujian <[email protected]>
1 parent 627e2fd commit 61c2ee9

File tree

5 files changed

+235
-137
lines changed

5 files changed

+235
-137
lines changed

‎Makefile

-1
Original file line numberDiff line numberDiff line change
@@ -784,7 +784,6 @@ kind-install-lb-svc:
784784
$(call kind_load_image,kube-ovn,$(VPC_NAT_GW_IMG))
785785
@$(MAKE) ENABLE_LB_SVC=true CNI_CONFIG_PRIORITY=10 kind-install
786786
@$(MAKE) kind-install-multus
787-
kubectl apply -f yamls/lb-svc-attachment.yaml
788787

789788
.PHONY: kind-install-webhook
790789
kind-install-webhook: kind-install

‎test/e2e/framework/cni.go

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package framework
2+
3+
import (
4+
"encoding/json"
5+
6+
"github.com/containernetworking/cni/pkg/types"
7+
nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
8+
"k8s.io/kubernetes/test/e2e/framework"
9+
10+
"github.com/onsi/ginkgo/v2"
11+
12+
"github.com/kubeovn/kube-ovn/pkg/netconf"
13+
"github.com/kubeovn/kube-ovn/pkg/request"
14+
"github.com/kubeovn/kube-ovn/pkg/util"
15+
)
16+
17+
const CNIVersion = "0.3.1"
18+
19+
// https://github.com/containernetworking/plugins/blob/main/plugins/main/macvlan/macvlan.go#L37
20+
type MacvlanNetConf struct {
21+
netconf.NetConf
22+
Master string `json:"master"`
23+
Mode string `json:"mode"`
24+
MTU int `json:"mtu"`
25+
Mac string `json:"mac,omitempty"`
26+
LinkContNs bool `json:"linkInContainer,omitempty"`
27+
28+
RuntimeConfig struct {
29+
Mac string `json:"mac,omitempty"`
30+
} `json:"runtimeConfig,omitempty"`
31+
}
32+
33+
func MakeMacvlanNetworkAttachmentDefinition(name, namespace, master, mode, provider string, routes []request.Route) *nadv1.NetworkAttachmentDefinition {
34+
ginkgo.GinkgoHelper()
35+
36+
config := &MacvlanNetConf{
37+
NetConf: netconf.NetConf{
38+
NetConf: types.NetConf{
39+
CNIVersion: CNIVersion,
40+
Type: "macvlan",
41+
},
42+
IPAM: &netconf.IPAMConf{
43+
Type: util.CniTypeName,
44+
ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock",
45+
Provider: provider,
46+
Routes: routes,
47+
},
48+
},
49+
Master: master,
50+
Mode: mode,
51+
LinkContNs: true,
52+
}
53+
buf, err := json.MarshalIndent(config, "", " ")
54+
framework.ExpectNoError(err)
55+
56+
return MakeNetworkAttachmentDefinition(name, namespace, string(buf))
57+
}
58+
59+
func MakeOVNNetworkAttachmentDefinition(name, namespace, provider string, routes []request.Route) *nadv1.NetworkAttachmentDefinition {
60+
ginkgo.GinkgoHelper()
61+
62+
config := &netconf.NetConf{
63+
NetConf: types.NetConf{
64+
CNIVersion: CNIVersion,
65+
Type: util.CniTypeName,
66+
},
67+
ServerSocket: "/run/openvswitch/kube-ovn-daemon.sock",
68+
Provider: provider,
69+
Routes: routes,
70+
}
71+
buf, err := json.MarshalIndent(config, "", " ")
72+
framework.ExpectNoError(err)
73+
74+
return MakeNetworkAttachmentDefinition(name, namespace, string(buf))
75+
}

‎test/e2e/lb-svc/e2e_test.go

+156-42
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ import (
77
"math/big"
88
"math/rand/v2"
99
"net"
10+
"strconv"
1011
"testing"
1112
"time"
1213

1314
dockernetwork "github.com/docker/docker/api/types/network"
15+
nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
1416
corev1 "k8s.io/api/core/v1"
1517
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1618
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,6 +23,8 @@ import (
2123
k8sframework "k8s.io/kubernetes/test/e2e/framework"
2224
"k8s.io/kubernetes/test/e2e/framework/config"
2325
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
26+
e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
27+
"k8s.io/utils/ptr"
2428

2529
"github.com/onsi/ginkgo/v2"
2630

@@ -31,8 +35,6 @@ import (
3135
"github.com/kubeovn/kube-ovn/test/e2e/framework/kind"
3236
)
3337

34-
const subnetProvider = "lb-svc-attachment.kube-system"
35-
3638
func init() {
3739
klog.SetOutput(ginkgo.GinkgoWriter)
3840

@@ -56,20 +58,27 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
5658

5759
var skip bool
5860
var cs clientset.Interface
61+
var podClient *framework.PodClient
5962
var subnetClient *framework.SubnetClient
6063
var serviceClient *framework.ServiceClient
6164
var deploymentClient *framework.DeploymentClient
62-
var clusterName, subnetName, namespaceName, serviceName, deploymentName string
65+
var nadClient *framework.NetworkAttachmentDefinitionClient
66+
var provider, nadName, clusterName, subnetName, namespaceName, serviceName, deploymentName, serverPodName, clientPodName string
6367
var dockerNetwork *dockernetwork.Inspect
6468
var cidr, gateway string
6569
ginkgo.BeforeEach(func() {
6670
cs = f.ClientSet
71+
podClient = f.PodClient()
6772
subnetClient = f.SubnetClient()
6873
serviceClient = f.ServiceClient()
6974
deploymentClient = f.DeploymentClient()
75+
nadClient = f.NetworkAttachmentDefinitionClient()
7076
namespaceName = f.Namespace.Name
77+
nadName = "nad-" + framework.RandomSuffix()
7178
subnetName = "subnet-" + framework.RandomSuffix()
7279
serviceName = "service-" + framework.RandomSuffix()
80+
serverPodName = "pod-" + framework.RandomSuffix()
81+
clientPodName = "pod-" + framework.RandomSuffix()
7382
deploymentName = lbSvcDeploymentName(serviceName)
7483

7584
if skip {
@@ -96,6 +105,13 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
96105
dockerNetwork = network
97106
}
98107

108+
provider = fmt.Sprintf("%s.%s", nadName, namespaceName)
109+
110+
ginkgo.By("Creating network attachment definition " + nadName)
111+
nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil)
112+
nad = nadClient.Create(nad)
113+
framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config)
114+
99115
ginkgo.By("Creating subnet " + subnetName)
100116
for _, config := range dockerNetwork.IPAM.Config {
101117
if util.CheckProtocol(config.Subnet) == apiv1.ProtocolIPv4 {
@@ -112,38 +128,54 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
112128
}
113129
}
114130
subnet := framework.MakeSubnet(subnetName, "", cidr, gateway, "", "", excludeIPs, nil, []string{namespaceName})
115-
subnet.Spec.Provider = subnetProvider
131+
subnet.Spec.Provider = provider
116132
_ = subnetClient.Create(subnet)
117133
})
118134
ginkgo.AfterEach(func() {
119-
ginkgo.By("Deleting deployment " + deploymentName)
120-
deploymentClient.DeleteSync(deploymentName)
135+
ginkgo.By("Deleting pod " + clientPodName)
136+
podClient.DeleteSync(clientPodName)
137+
138+
ginkgo.By("Deleting pod " + serverPodName)
139+
podClient.DeleteSync(serverPodName)
121140

122141
ginkgo.By("Deleting service " + serviceName)
123142
serviceClient.DeleteSync(serviceName)
124143

144+
ginkgo.By("Deleting deployment " + deploymentName)
145+
deploymentClient.DeleteSync(deploymentName)
146+
125147
ginkgo.By("Deleting subnet " + subnetName)
126148
subnetClient.DeleteSync(subnetName)
149+
150+
ginkgo.By("Deleting network attachment definition " + nadName)
151+
nadClient.Delete(nadName)
127152
})
128153

129154
framework.ConformanceIt("should allocate dynamic external IP for service", func() {
155+
ginkgo.By("Creating server pod " + serverPodName)
156+
labels := map[string]string{"app": serviceName}
157+
port := 8000 + rand.Int32N(1000)
158+
args := []string{"netexec", "--http-port", strconv.Itoa(int(port))}
159+
serverPod := framework.MakePod(namespaceName, serverPodName, labels, nil, framework.AgnhostImage, nil, args)
160+
_ = podClient.CreateSync(serverPod)
161+
130162
ginkgo.By("Creating service " + serviceName)
131163
ports := []corev1.ServicePort{{
132164
Name: "tcp",
133165
Protocol: corev1.ProtocolTCP,
134-
Port: 80,
135-
TargetPort: intstr.FromInt32(80),
166+
Port: port,
167+
TargetPort: intstr.FromInt32(port),
136168
}}
137169
annotations := map[string]string{
138-
subnetProvider + ".kubernetes.io/logical_switch": subnetName,
170+
util.AttachmentProvider: provider,
139171
}
140-
selector := map[string]string{"app": "lb-svc-dynamic"}
141-
service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, selector, ports, corev1.ServiceAffinityNone)
142-
_ = serviceClient.CreateSync(service, func(s *corev1.Service) (bool, error) {
172+
service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, labels, ports, corev1.ServiceAffinityNone)
173+
service.Spec.AllocateLoadBalancerNodePorts = ptr.To(false)
174+
service = serviceClient.CreateSync(service, func(s *corev1.Service) (bool, error) {
143175
return len(s.Spec.ClusterIPs) != 0, nil
144176
}, "cluster ips are not empty")
145177

146-
ginkgo.By("Waiting for deployment " + deploymentName + " to be ready")
178+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
147179
framework.WaitUntil(2*time.Second, time.Minute, func(ctx context.Context) (bool, error) {
148180
_, err := deploymentClient.DeploymentInterface.Get(ctx, deploymentName, metav1.GetOptions{})
149181
if err == nil {
@@ -164,44 +196,91 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
164196
framework.ExpectNoError(err)
165197
framework.ExpectHaveLen(pods.Items, 1)
166198

167-
ginkgo.By("Checking pod annotations")
168-
key := fmt.Sprintf(util.AllocatedAnnotationTemplate, subnetProvider)
169-
framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, key, "true")
170-
cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, subnetProvider)
171-
ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, subnetProvider)
172-
framework.ExpectHaveKey(pods.Items[0].Annotations, cidrKey)
173-
framework.ExpectHaveKey(pods.Items[0].Annotations, ipKey)
174-
cidr := pods.Items[0].Annotations[cidrKey]
175-
ip := pods.Items[0].Annotations[ipKey]
176-
framework.ExpectTrue(util.CIDRContainIP(cidr, ip))
177-
178-
ginkgo.By("Checking service external IP")
199+
ginkgo.By("Checking LB pod annotations")
200+
pod := &pods.Items[0]
201+
key := fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)
202+
framework.ExpectHaveKeyWithValue(pod.Annotations, key, "true")
203+
cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, provider)
204+
ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)
205+
framework.ExpectHaveKey(pod.Annotations, cidrKey)
206+
framework.ExpectHaveKey(pod.Annotations, ipKey)
207+
lbIP := pod.Annotations[ipKey]
208+
framework.ExpectIPInCIDR(lbIP, pod.Annotations[cidrKey])
209+
210+
ginkgo.By("Checking service status")
179211
framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) {
180212
service = serviceClient.Get(serviceName)
181213
return len(service.Status.LoadBalancer.Ingress) != 0, nil
182214
}, ".status.loadBalancer.ingress is not empty")
183-
framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, ip)
215+
framework.ExpectHaveLen(service.Status.LoadBalancer.Ingress, 1)
216+
framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, lbIP)
217+
218+
ginkgo.By("Creating client pod " + clientPodName)
219+
annotations = map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", namespaceName, nadName)}
220+
cmd := []string{"sh", "-c", "sleep infinity"}
221+
clientPod := framework.MakePod(namespaceName, clientPodName, nil, annotations, f.KubeOVNImage, cmd, nil)
222+
clientPod = podClient.CreateSync(clientPod)
223+
224+
ginkgo.By("Checking service connectivity from client pod " + clientPodName)
225+
curlCmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
226+
ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
227+
_ = e2epodoutput.RunHostCmdOrDie(clientPod.Namespace, clientPod.Name, curlCmd)
228+
229+
ginkgo.By("Deleting lb svc pod " + pod.Name)
230+
podClient.DeleteSync(pod.Name)
231+
232+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
233+
err = deploymentClient.WaitToComplete(deployment)
234+
framework.ExpectNoError(err, "deployment failed to complete")
235+
236+
ginkgo.By("Getting pods for deployment " + deploymentName)
237+
pods, err = deploymentClient.GetPods(deployment)
238+
framework.ExpectNoError(err)
239+
framework.ExpectHaveLen(pods.Items, 1)
240+
lbIP = pods.Items[0].Annotations[ipKey]
241+
242+
ginkgo.By("Checking service connectivity from client pod " + clientPodName)
243+
curlCmd = fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
244+
framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) {
245+
ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
246+
_, err = e2epodoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, curlCmd)
247+
return err == nil, nil
248+
}, "")
249+
250+
ginkgo.By("Deleting service " + serviceName)
251+
serviceClient.DeleteSync(serviceName)
252+
253+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be deleted automatically")
254+
err = deploymentClient.WaitToDisappear(deploymentName, 2*time.Second, 2*time.Minute)
255+
framework.ExpectNoError(err, "deployment failed to disappear")
184256
})
185257

186258
framework.ConformanceIt("should allocate static external IP for service", func() {
259+
ginkgo.By("Creating server pod " + serverPodName)
260+
labels := map[string]string{"app": serviceName}
261+
port := 8000 + rand.Int32N(1000)
262+
args := []string{"netexec", "--http-port", strconv.Itoa(int(port))}
263+
serverPod := framework.MakePod(namespaceName, serverPodName, labels, nil, framework.AgnhostImage, nil, args)
264+
_ = podClient.CreateSync(serverPod)
265+
187266
ginkgo.By("Creating service " + serviceName)
188-
base := util.IP2BigInt(gateway)
189-
lbIP := util.BigInt2Ip(base.Add(base, big.NewInt(50+rand.Int64N(50))))
190267
ports := []corev1.ServicePort{{
191268
Name: "tcp",
192269
Protocol: corev1.ProtocolTCP,
193-
Port: 80,
194-
TargetPort: intstr.FromInt32(80),
270+
Port: port,
271+
TargetPort: intstr.FromInt32(port),
195272
}}
196273
annotations := map[string]string{
197-
subnetProvider + ".kubernetes.io/logical_switch": subnetName,
274+
util.AttachmentProvider: provider,
198275
}
199-
selector := map[string]string{"app": "lb-svc-static"}
200-
service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, selector, ports, corev1.ServiceAffinityNone)
276+
base := util.IP2BigInt(gateway)
277+
lbIP := util.BigInt2Ip(base.Add(base, big.NewInt(50+rand.Int64N(50))))
278+
service := framework.MakeService(serviceName, corev1.ServiceTypeLoadBalancer, annotations, labels, ports, corev1.ServiceAffinityNone)
201279
service.Spec.LoadBalancerIP = lbIP
280+
service.Spec.AllocateLoadBalancerNodePorts = ptr.To(false)
202281
_ = serviceClient.Create(service)
203282

204-
ginkgo.By("Waiting for deployment " + deploymentName + " to be ready")
283+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
205284
framework.WaitUntil(2*time.Second, time.Minute, func(ctx context.Context) (bool, error) {
206285
_, err := deploymentClient.DeploymentInterface.Get(ctx, deploymentName, metav1.GetOptions{})
207286
if err == nil {
@@ -221,19 +300,54 @@ var _ = framework.SerialDescribe("[group:lb-svc]", func() {
221300
framework.ExpectNoError(err)
222301
framework.ExpectHaveLen(pods.Items, 1)
223302

224-
ginkgo.By("Checking pod annotations")
225-
key := fmt.Sprintf(util.AllocatedAnnotationTemplate, subnetProvider)
226-
framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, key, "true")
227-
ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, subnetProvider)
228-
framework.ExpectHaveKeyWithValue(pods.Items[0].Annotations, ipKey, lbIP)
229-
cidr := pods.Items[0].Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, subnetProvider)]
230-
framework.ExpectTrue(util.CIDRContainIP(cidr, lbIP))
303+
ginkgo.By("Checking LB pod annotations")
304+
pod := &pods.Items[0]
305+
key := fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)
306+
framework.ExpectHaveKeyWithValue(pod.Annotations, key, "true")
307+
ipKey := fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)
308+
framework.ExpectHaveKeyWithValue(pod.Annotations, ipKey, lbIP)
309+
cidrKey := fmt.Sprintf(util.CidrAnnotationTemplate, provider)
310+
framework.ExpectHaveKey(pod.Annotations, cidrKey)
311+
framework.ExpectIPInCIDR(lbIP, pod.Annotations[cidrKey])
231312

232-
ginkgo.By("Checking service external IP")
313+
ginkgo.By("Checking service status")
233314
framework.WaitUntil(2*time.Second, time.Minute, func(_ context.Context) (bool, error) {
234315
service = serviceClient.Get(serviceName)
235316
return len(service.Status.LoadBalancer.Ingress) != 0, nil
236317
}, ".status.loadBalancer.ingress is not empty")
318+
framework.ExpectHaveLen(service.Status.LoadBalancer.Ingress, 1)
237319
framework.ExpectEqual(service.Status.LoadBalancer.Ingress[0].IP, lbIP)
320+
321+
ginkgo.By("Creating client pod " + clientPodName)
322+
annotations = map[string]string{nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", namespaceName, nadName)}
323+
cmd := []string{"sh", "-c", "sleep infinity"}
324+
clientPod := framework.MakePod(namespaceName, clientPodName, nil, annotations, f.KubeOVNImage, cmd, nil)
325+
clientPod = podClient.CreateSync(clientPod)
326+
327+
ginkgo.By("Checking service connectivity from client pod " + clientPodName)
328+
curlCmd := fmt.Sprintf("curl -q -s --connect-timeout 2 --max-time 2 %s/clientip", util.JoinHostPort(lbIP, port))
329+
ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
330+
_ = e2epodoutput.RunHostCmdOrDie(clientPod.Namespace, clientPod.Name, curlCmd)
331+
332+
ginkgo.By("Deleting lb svc pod " + pod.Name)
333+
podClient.DeleteSync(pod.Name)
334+
335+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be ready")
336+
err = deploymentClient.WaitToComplete(deployment)
337+
framework.ExpectNoError(err, "deployment failed to complete")
338+
339+
ginkgo.By("Checking service connectivity from client pod " + clientPodName)
340+
framework.WaitUntil(2*time.Second, 30*time.Second, func(_ context.Context) (bool, error) {
341+
ginkgo.By(fmt.Sprintf(`Executing %q in pod %s/%s`, curlCmd, clientPod.Namespace, clientPod.Name))
342+
_, err = e2epodoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, curlCmd)
343+
return err == nil, nil
344+
}, "")
345+
346+
ginkgo.By("Deleting service " + serviceName)
347+
serviceClient.DeleteSync(serviceName)
348+
349+
ginkgo.By("Waiting for LB deployment " + deploymentName + " to be deleted automatically")
350+
err = deploymentClient.WaitToDisappear(deploymentName, 2*time.Second, 2*time.Minute)
351+
framework.ExpectNoError(err, "deployment failed to disappear")
238352
})
239353
})

0 commit comments

Comments
 (0)