Skip to content

Commit

Permalink
daemon, service: add bpf-lb-proto-diff flag
Browse files Browse the repository at this point in the history 
add a new bpf-lb-proto-diff flag that disables the
service protocol differentiation logic, allowing to keep the old Cilium
behavior where service protocols are not distinguished

Signed-off-by: Gilberto Bertin <[email protected]>
  • Loading branch information
@jibi @d-honeybadger
jibi authored and d-honeybadger committed Jan 27, 2025
1 parent 56f6cea commit eafff99
Show file tree
Hide file tree
Showing 9 changed files with 106 additions and 8 deletions.
1 change: 1 addition & 0 deletions Documentation/cmdref/cilium-agent.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 18 additions & 6 deletions bpf/bpf_sock.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,9 @@ static __always_inline int __sock4_xlate_fwd(struct bpf_sock_addr *ctx,
struct lb4_key key = {
.address = dst_ip,
.dport = dst_port,
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
}, orig_key = key;
struct lb4_service *backend_slot;
bool backend_from_affinity = false;
Expand All @@ -326,7 +328,7 @@ static __always_inline int __sock4_xlate_fwd(struct bpf_sock_addr *ctx,
/* Restore the original key's protocol as lb4_lookup_service
* has overwritten it.
*/
key.proto = protocol;
lb4_key_set_protocol(&key, protocol);
svc = sock4_wildcard_lookup_full(&key, in_hostns);
}
if (!svc)
Expand Down Expand Up @@ -492,7 +494,9 @@ static __always_inline int __sock4_post_bind(struct bpf_sock *ctx,
struct lb4_key key = {
.address = ctx->src_ip4,
.dport = ctx_src_port(ctx),
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
};

if (!sock_proto_enabled(protocol) ||
Expand All @@ -508,7 +512,7 @@ static __always_inline int __sock4_post_bind(struct bpf_sock *ctx,
* Restore the original key's protocol as lb4_lookup_service
* has overwritten it.
*/
key.proto = protocol;
lb4_key_set_protocol(&key, protocol);
svc = sock4_wildcard_lookup(&key, false, false, true);
}

Expand Down Expand Up @@ -606,15 +610,17 @@ static __always_inline int __sock4_xlate_rev(struct bpf_sock_addr *ctx,
struct lb4_key svc_key = {
.address = val->address,
.dport = val->port,
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
};

svc = lb4_lookup_service(&svc_key, true, false);
if (!svc) {
/* Restore the original key's protocol as lb4_lookup_service
* has overwritten it.
*/
svc_key.proto = protocol;
lb4_key_set_protocol(&svc_key, protocol);
svc = sock4_wildcard_lookup_full(&svc_key,
ctx_in_hostns(ctx_full, NULL));
}
Expand Down Expand Up @@ -879,7 +885,9 @@ static __always_inline int __sock6_post_bind(struct bpf_sock *ctx)
struct lb6_service *svc;
struct lb6_key key = {
.dport = ctx_src_port(ctx),
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
};

if (!sock_proto_enabled(protocol) ||
Expand All @@ -893,7 +901,7 @@ static __always_inline int __sock6_post_bind(struct bpf_sock *ctx)
/* Restore the original key's protocol as lb6_lookup_service
* has overwritten it.
*/
key.proto = protocol;
lb6_key_set_protocol(&key, protocol);
svc = sock6_wildcard_lookup(&key, false, false, true);
if (!svc)
return sock6_post_bind_v4_in_v6(ctx);
Expand Down Expand Up @@ -1011,7 +1019,9 @@ static __always_inline int __sock6_xlate_fwd(struct bpf_sock_addr *ctx,
__u8 protocol = ctx_protocol(ctx);
struct lb6_key key = {
.dport = dst_port,
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
}, orig_key;
struct lb6_service *backend_slot;
bool backend_from_affinity = false;
Expand All @@ -1034,7 +1044,7 @@ static __always_inline int __sock6_xlate_fwd(struct bpf_sock_addr *ctx,
/* Restore the original key's protocol as lb6_lookup_service
* has overwritten it.
*/
key.proto = protocol;
lb6_key_set_protocol(&key, protocol);
svc = sock6_wildcard_lookup_full(&key, in_hostns);
}
if (!svc)
Expand Down Expand Up @@ -1219,15 +1229,17 @@ static __always_inline int __sock6_xlate_rev(struct bpf_sock_addr *ctx)
struct lb6_key svc_key = {
.address = val->address,
.dport = val->port,
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
.proto = protocol,
#endif
};

svc = lb6_lookup_service(&svc_key, true, false);
if (!svc) {
/* Restore the original key's protocol as lb6_lookup_service
* has overwritten it.
*/
svc_key.proto = protocol;
lb6_key_set_protocol(&svc_key, protocol);
svc = sock6_wildcard_lookup_full(&svc_key,
ctx_in_hostns(ctx, NULL));
}
Expand Down
28 changes: 26 additions & 2 deletions bpf/lib/lb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,10 +487,19 @@ static __always_inline int lb6_rev_nat(struct __ctx_buff *ctx, int l4_off,
return __lb6_rev_nat(ctx, l4_off, tuple, flags, nat);
}

static __always_inline void
lb6_key_set_protocol(struct lb6_key *key __maybe_unused,
__u8 protocol __maybe_unused)
{
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
key->proto = protocol;
#endif
}

static __always_inline void
lb6_fill_key(struct lb6_key *key, struct ipv6_ct_tuple *tuple)
{
key->proto = tuple->nexthdr;
lb6_key_set_protocol(key, tuple->nexthdr);
ipv6_addr_copy(&key->address, &tuple->daddr);
key->dport = tuple->sport;
}
Expand Down Expand Up @@ -1021,6 +1030,12 @@ struct lb6_service *lb6_lookup_service(struct lb6_key *key __maybe_unused,
return NULL;
}

static __always_inline void
lb6_key_set_protocol(struct lb6_key *key __maybe_unused,
__u8 protocol __maybe_unused)
{
}

static __always_inline
struct lb6_service *__lb6_lookup_backend_slot(struct lb6_key *key __maybe_unused)
{
Expand Down Expand Up @@ -1144,10 +1159,19 @@ static __always_inline int lb4_rev_nat(struct __ctx_buff *ctx, int l3_off, int l
loopback, has_l4_header);
}

static __always_inline void
lb4_key_set_protocol(struct lb4_key *key __maybe_unused,
__u8 protocol __maybe_unused)
{
#if defined(ENABLE_SERVICE_PROTOCOL_DIFFERENTIATION)
key->proto = protocol;
#endif
}

static __always_inline void
lb4_fill_key(struct lb4_key *key, const struct ipv4_ct_tuple *tuple)
{
key->proto = tuple->nexthdr;
lb4_key_set_protocol(key, tuple->nexthdr);
key->address = tuple->daddr;
/* CT tuple has ports in reverse order: */
key->dport = tuple->sport;
Expand Down
3 changes: 3 additions & 0 deletions daemon/cmd/daemon_main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,9 @@ func initializeFlags() {
option.NodePortAccelerationNative, option.NodePortAccelerationDisabled))
option.BindEnv(Vp, option.LoadBalancerAcceleration)

flags.Bool(option.LoadBalancerProtocolDifferentiation, true, "Enable support for service protocol differentiation (TCP, UDP, SCTP)")
option.BindEnv(vp, option.LoadBalancerProtocolDifferentiation)

Check failure on line 626 in daemon/cmd/daemon_main.go

View workflow job for this annotation

GitHub Actions / Precheck 

undefined: vp

Check failure on line 626 in daemon/cmd/daemon_main.go

View workflow job for this annotation

GitHub Actions / Precheck 

undefined: vp

Check failure on line 626 in daemon/cmd/daemon_main.go

View workflow job for this annotation

GitHub Actions / Lint Source Code 

undefined: vp) (typecheck)

Check failure on line 626 in daemon/cmd/daemon_main.go

View workflow job for this annotation

GitHub Actions / Lint Source Code 

undefined: vp) (typecheck)

Check failure on line 626 in daemon/cmd/daemon_main.go

View workflow job for this annotation

GitHub Actions / Lint Source Code 

undefined: vp (typecheck)

flags.Uint(option.MaglevTableSize, maglev.DefaultTableSize, "Maglev per service backend table size (parameter M)")
option.BindEnv(Vp, option.MaglevTableSize)

Expand Down
3 changes: 3 additions & 0 deletions install/kubernetes/cilium/templates/cilium-configmap.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,9 @@ data:
{{- if hasKey .Values.loadBalancer "serviceTopology" }}
enable-service-topology: {{ .Values.loadBalancer.serviceTopology | quote }}
{{- end }}
{{- if hasKey .Values.loadBalancer "protocolDifferentiation" }}
bpf-lb-proto-diff: {{ .Values.loadBalancer.protocolDifferentiation.enabled | quote }}
{{- end }}

{{- end }}
{{- if hasKey .Values.maglev "tableSize" }}
Expand Down
1 change: 1 addition & 0 deletions pkg/datapath/linux/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,7 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC
cDefinesMap["IPV6_RSS_PREFIX_BITS"] = "128"
}
}

if option.Config.NodePortAcceleration != option.NodePortAccelerationDisabled {
cDefinesMap["ENABLE_NODEPORT_ACCELERATION"] = "1"
}
Expand Down
32 changes: 32 additions & 0 deletions pkg/k8s/watchers/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,35 @@ func (k *K8sWatcher) deleteK8sServiceV1(svc *slim_corev1.Service, swg *lock.Stop
}
return nil
}

func stripServiceProtocol(svc *k8s.Service) *k8s.Service {
if svc == nil {
return nil
}

svc = svc.DeepCopy()

for _, port := range svc.Ports {
port.Protocol = "NONE"
}

for _, nodePort := range svc.NodePorts {
for _, port := range nodePort {
port.Protocol = "NONE"
}
}

return svc
}

func stripEndpointsProtocol(endpoints *k8s.Endpoints) *k8s.Endpoints {
endpoints = endpoints.DeepCopy()

for _, backend := range endpoints.Backends {
for _, port := range backend.Ports {
port.Protocol = "NONE"
}
}

return endpoints
}
6 changes: 6 additions & 0 deletions pkg/k8s/watchers/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,12 @@ func (k *K8sWatcher) addK8sSVCs(svcID k8s.ServiceID, oldSvc, svc *k8s.Service, e
return nil
}

if !option.Config.LoadBalancerProtocolDifferentiation {
oldSvc = stripServiceProtocol(oldSvc)
svc = stripServiceProtocol(svc)
endpoints = stripEndpointsProtocol(endpoints)
}

scopedLog := log.WithFields(logrus.Fields{
logfields.K8sSvcName: svcID.Name,
logfields.K8sNamespace: svcID.Namespace,
Expand Down
16 changes: 16 additions & 0 deletions pkg/option/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,13 @@ const (
// Alias to NodePortAcceleration
LoadBalancerAcceleration = "bpf-lb-acceleration"

// LoadBalancerExternalControlPlane switch skips connectivity to kube-apiserver
// which is relevant in lb-only mode
LoadBalancerExternalControlPlane = "bpf-lb-external-control-plane"

// LoadBalancerProtocolDifferentiation enables support for service protocol differentiation (TCP, UDP, SCTP)
LoadBalancerProtocolDifferentiation = "bpf-lb-proto-diff"

// MaglevTableSize determines the size of the backend table per service
MaglevTableSize = "bpf-lb-maglev-table-size"

Expand Down Expand Up @@ -2066,6 +2073,13 @@ type DaemonConfig struct {
LoadBalancerRSSv6CIDR string
LoadBalancerRSSv6 net.IPNet

// LoadBalancerExternalControlPlane tells whether to not use kube-apiserver as
// its control plane in lb-only mode.
LoadBalancerExternalControlPlane bool

// LoadBalancerProtocolDifferentiation enables support for service protocol differentiation (TCP, UDP, SCTP)
LoadBalancerProtocolDifferentiation bool

// EnablePMTUDiscovery indicates whether to send ICMP fragmentation-needed
// replies to the client (when needed).
EnablePMTUDiscovery bool
Expand Down Expand Up @@ -3616,6 +3630,8 @@ func (c *DaemonConfig) populateDevices(vp *viper.Viper) {
for dev := range devSet {
c.devices = append(c.devices, dev)
}

c.LoadBalancerProtocolDifferentiation = vp.GetBool(LoadBalancerProtocolDifferentiation)
}

func (c *DaemonConfig) populateLoadBalancerSettings(vp *viper.Viper) {
Expand Down

0 comments on commit eafff99

Please sign in to comment.