Skip to content

Commit c2a39f0

Browse files
whoooaauren
whooo
authored and
aauren
committed
add loadbalancer address allocator
This adds a simple controller that will watch for services of type LoadBalancer and try to allocated addresses from the specified IPv4 and/or IPv6 ranges. It's assumed that kube-router (or another network controller) will announce the addresses. As the controller uses leases for leader election and updates the service status new RBAC permissions are required.
1 parent 9990f3b commit c2a39f0

17 files changed

+1592
-35
lines changed

‎daemonset/generic-kuberouter-all-features-advertise-routes.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ spec:
8484
valueFrom:
8585
fieldRef:
8686
fieldPath: spec.nodeName
87+
- name: POD_NAME
88+
valueFrom:
89+
fieldRef:
90+
fieldPath: metadata.name
8791
- name: KUBE_ROUTER_CNI_CONF_FILE
8892
value: /etc/cni/net.d/10-kuberouter.conflist
8993
livenessProbe:
@@ -206,6 +210,20 @@ rules:
206210
- get
207211
- list
208212
- watch
213+
- apiGroups:
214+
- "coordination.k8s.io"
215+
resources:
216+
- leases
217+
verbs:
218+
- get
219+
- create
220+
- update
221+
- apiGroups:
222+
- ""
223+
resources:
224+
- services/status
225+
verbs:
226+
- update
209227

210228
---
211229
kind: ClusterRoleBinding

‎daemonset/generic-kuberouter-all-features.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ spec:
8080
valueFrom:
8181
fieldRef:
8282
fieldPath: spec.nodeName
83+
- name: POD_NAME
84+
valueFrom:
85+
fieldRef:
86+
fieldPath: metadata.name
8387
- name: KUBE_ROUTER_CNI_CONF_FILE
8488
value: /etc/cni/net.d/10-kuberouter.conflist
8589
livenessProbe:
@@ -202,6 +206,20 @@ rules:
202206
- get
203207
- list
204208
- watch
209+
- apiGroups:
210+
- "coordination.k8s.io"
211+
resources:
212+
- leases
213+
verbs:
214+
- get
215+
- create
216+
- update
217+
- apiGroups:
218+
- ""
219+
resources:
220+
- services/status
221+
verbs:
222+
- update
205223
---
206224
kind: ClusterRoleBinding
207225
apiVersion: rbac.authorization.k8s.io/v1

‎daemonset/generic-kuberouter-only-advertise-routes.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ spec:
4343
valueFrom:
4444
fieldRef:
4545
fieldPath: spec.nodeName
46+
- name: POD_NAME
47+
valueFrom:
48+
fieldRef:
49+
fieldPath: metadata.name
4650
livenessProbe:
4751
httpGet:
4852
path: /healthz
@@ -114,6 +118,20 @@ rules:
114118
- get
115119
- list
116120
- watch
121+
- apiGroups:
122+
- "coordination.k8s.io"
123+
resources:
124+
- leases
125+
verbs:
126+
- get
127+
- create
128+
- update
129+
- apiGroups:
130+
- ""
131+
resources:
132+
- services/status
133+
verbs:
134+
- update
117135

118136
---
119137
kind: ClusterRoleBinding

‎daemonset/generic-kuberouter.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ spec:
6060
valueFrom:
6161
fieldRef:
6262
fieldPath: spec.nodeName
63+
- name: POD_NAME
64+
valueFrom:
65+
fieldRef:
66+
fieldPath: metadata.name
6367
- name: KUBE_ROUTER_CNI_CONF_FILE
6468
value: /etc/cni/net.d/10-kuberouter.conflist
6569
livenessProbe:
@@ -169,6 +173,20 @@ rules:
169173
- get
170174
- list
171175
- watch
176+
- apiGroups:
177+
- "coordination.k8s.io"
178+
resources:
179+
- leases
180+
verbs:
181+
- get
182+
- create
183+
- update
184+
- apiGroups:
185+
- ""
186+
resources:
187+
- services/status
188+
verbs:
189+
- update
172190

173191
---
174192
kind: ClusterRoleBinding

‎daemonset/kubeadm-kuberouter-all-features-dsr.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ spec:
6161
valueFrom:
6262
fieldRef:
6363
fieldPath: spec.nodeName
64+
- name: POD_NAME
65+
valueFrom:
66+
fieldRef:
67+
fieldPath: metadata.name
6468
- name: KUBE_ROUTER_CNI_CONF_FILE
6569
value: /etc/cni/net.d/10-kuberouter.conflist
6670
livenessProbe:
@@ -185,6 +189,20 @@ rules:
185189
- get
186190
- list
187191
- watch
192+
- apiGroups:
193+
- "coordination.k8s.io"
194+
resources:
195+
- leases
196+
verbs:
197+
- get
198+
- create
199+
- update
200+
- apiGroups:
201+
- ""
202+
resources:
203+
- services/status
204+
verbs:
205+
- update
188206
---
189207
kind: ClusterRoleBinding
190208
apiVersion: rbac.authorization.k8s.io/v1

‎daemonset/kubeadm-kuberouter-all-features-hostport.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ spec:
6868
valueFrom:
6969
fieldRef:
7070
fieldPath: spec.nodeName
71+
- name: POD_NAME
72+
valueFrom:
73+
fieldRef:
74+
fieldPath: metadata.name
7175
- name: KUBE_ROUTER_CNI_CONF_FILE
7276
value: /etc/cni/net.d/10-kuberouter.conflist
7377
livenessProbe:
@@ -184,6 +188,20 @@ rules:
184188
- get
185189
- list
186190
- watch
191+
- apiGroups:
192+
- "coordination.k8s.io"
193+
resources:
194+
- leases
195+
verbs:
196+
- get
197+
- create
198+
- update
199+
- apiGroups:
200+
- ""
201+
resources:
202+
- services/status
203+
verbs:
204+
- update
187205
---
188206
kind: ClusterRoleBinding
189207
apiVersion: rbac.authorization.k8s.io/v1

‎daemonset/kubeadm-kuberouter-all-features.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ spec:
6161
valueFrom:
6262
fieldRef:
6363
fieldPath: spec.nodeName
64+
- name: POD_NAME
65+
valueFrom:
66+
fieldRef:
67+
fieldPath: metadata.name
6468
- name: KUBE_ROUTER_CNI_CONF_FILE
6569
value: /etc/cni/net.d/10-kuberouter.conflist
6670
livenessProbe:
@@ -177,6 +181,20 @@ rules:
177181
- get
178182
- list
179183
- watch
184+
- apiGroups:
185+
- "coordination.k8s.io"
186+
resources:
187+
- leases
188+
verbs:
189+
- get
190+
- create
191+
- update
192+
- apiGroups:
193+
- ""
194+
resources:
195+
- services/status
196+
verbs:
197+
- update
180198
---
181199
kind: ClusterRoleBinding
182200
apiVersion: rbac.authorization.k8s.io/v1

‎daemonset/kubeadm-kuberouter.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ spec:
6060
valueFrom:
6161
fieldRef:
6262
fieldPath: spec.nodeName
63+
- name: POD_NAME
64+
valueFrom:
65+
fieldRef:
66+
fieldPath: metadata.name
6367
- name: KUBE_ROUTER_CNI_CONF_FILE
6468
value: /etc/cni/net.d/10-kuberouter.conflist
6569
livenessProbe:
@@ -173,6 +177,20 @@ rules:
173177
- get
174178
- list
175179
- watch
180+
- apiGroups:
181+
- "coordination.k8s.io"
182+
resources:
183+
- leases
184+
verbs:
185+
- get
186+
- create
187+
- update
188+
- apiGroups:
189+
- ""
190+
resources:
191+
- services/status
192+
verbs:
193+
- update
176194
---
177195
kind: ClusterRoleBinding
178196
apiVersion: rbac.authorization.k8s.io/v1

‎docs/load-balancer-allocator.md

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Load Balancer allocator
2+
3+
## What does it do
4+
5+
The load balancer allocator controller looks for services with the type LoadBalancer and tries to allocate addresses for it if needed.
6+
The controller doesn't enable any announcement of the addresses by default, so `--advertise-loadbalancer-ip` should be set to true and BGP peers configured.
7+
8+
## Load balancer classes
9+
10+
By default the controller allocates addresses for all LoadBalancer services with the where `loadBalancerClass` is empty or set to one of "default" or "kube-router".
11+
If `--loadbalancer-default-class` is set to false, the controller will only handle services with the class set to "kube-router".
12+
13+
## RBAC permissions
14+
15+
The controller needs some extra permissions to get, create and update leases for leader election and to update services with allocated addresses.
16+
17+
Example permissions:
18+
```yaml
19+
kind: ClusterRole
20+
apiVersion: rbac.authorization.k8s.io/v1
21+
metadata:
22+
name: kube-router
23+
namespace: kube-system
24+
rules:
25+
- apiGroups:
26+
- "coordination.k8s.io"
27+
resources:
28+
- leases
29+
verbs:
30+
- get
31+
- create
32+
- update
33+
- apiGroups:
34+
- ""
35+
resources:
36+
- services/status
37+
verbs:
38+
- update
39+
```
40+
41+
## Environment variables
42+
43+
The controller uses the environment variable `POD_NAME` as the identify for the lease used for leader election.
44+
Using the kubernetes downward api to set `POD_NAME` to the pod name the lease identify will match the current leader.
45+
```yaml
46+
---
47+
apiVersion: apps/v1
48+
kind: DaemonSet
49+
metadata:
50+
labels:
51+
k8s-app: kube-router
52+
tier: node
53+
name: kube-router
54+
namespace: kube-system
55+
spec:
56+
...
57+
template:
58+
metadata:
59+
....
60+
spec:
61+
...
62+
env:
63+
- name: POD_NAME
64+
valueFrom:
65+
fieldRef:
66+
fieldPath: metadata.name
67+
...
68+
```
69+
70+
The environment variable `POD_NAMESPACE` can also be specified to set the namespace used for the lease.
71+
By default the namespace is looked up from within the pod using `/var/run/secrets/kubernetes.io/serviceaccount/namespace`.
72+
73+
## Running outside kubernetes
74+
75+
When running the controller outside a pod, both `POD_NAME` and `POD_NAMESPACE` must set for the controller to work.
76+
`POD_NAME` should be unique per instance, so using for example the hostname of the machine might be a good idea.
77+
`POD_NAMESPACE` must be the same across all instances running in the same cluster.
78+
79+
## Notes
80+
81+
It's not possible to specify the addresses for the load balancer services. A externalIP service can be used instead.

‎docs/user-guide.md

+4
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ Usage of kube-router:
9494
--ipvs-permit-all Enables rule to accept all incoming traffic to service VIP's on the node. (default true)
9595
--ipvs-sync-period duration The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 5m0s)
9696
--kubeconfig string Path to kubeconfig file with authorization information (the master location is set by the master flag).
97+
--loadbalancer-default-class Handle loadbalancer services without a class (default true)
98+
--loadbalancer-ip-range strings CIDR values from which loadbalancer services addresses are assigned (can be specified multiple times)
99+
--loadbalancer-sync-period duration The delay between checking for missed services (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s)
97100
--masquerade-all SNAT all traffic to cluster IP/node port.
98101
--master string The address of the Kubernetes API server (overrides any value in kubeconfig).
99102
--metrics-path string Prometheus metrics path (default "/metrics")
@@ -113,6 +116,7 @@ Usage of kube-router:
113116
--router-id string BGP router-id. Must be specified in a ipv6 only cluster, "generate" can be specified to generate the router id.
114117
--routes-sync-period duration The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 5m0s)
115118
--run-firewall Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true)
119+
--run-loadbalancer Enable loadbalancer address allocator
116120
--run-router Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true)
117121
--run-service-proxy Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true)
118122
--runtime-endpoint string Path to CRI compatible container runtime socket (used for DSR mode). Currently known working with containerd.

‎pkg/cmd/kube-router.go

+14
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"syscall"
99
"time"
1010

11+
"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/lballoc"
1112
"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/netpol"
1213
"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/proxy"
1314
"github.com/cloudnativelabs/kube-router/v2/pkg/controllers/routing"
@@ -204,6 +205,19 @@ func (kr *KubeRouter) Run() error {
204205
go npc.Run(healthChan, stopCh, &wg)
205206
}
206207

208+
if kr.Config.RunLoadBalancer {
209+
klog.V(0).Info("running load balancer allocator controller")
210+
lbc, err := lballoc.NewLoadBalancerController(kr.Client, kr.Config, svcInformer)
211+
if err != nil {
212+
return errors.New("Failed to create load balancer allocator: " + err.Error())
213+
}
214+
215+
svcInformer.AddEventHandler(lbc)
216+
217+
wg.Add(1)
218+
go lbc.Run(healthChan, stopCh, &wg)
219+
}
220+
207221
// Handle SIGINT and SIGTERM
208222
ch := make(chan os.Signal, 1)
209223
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

0 commit comments

Comments
 (0)