Skip to content

Enhance Felix route table for elevated priority programming #11904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nelljerram merged 10 commits into from
Mar 2, 2026
Merged

Enhance Felix route table for elevated priority programming #11904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d855b18
Enhance Felix route table for elevated priority programming
nelljerram Jan 23, 2026
a1ddb45
Index routetable structs on kernelRouteKey instead of just CIDR
nelljerram Feb 25, 2026
336461c
Use RouteKey in RouteTable API
nelljerram Feb 27, 2026
6732be7
Merge remote-tracking branch 'origin/master' into routing-priority
nelljerram Feb 27, 2026
2a048ad
Fix IPv6 RouteRemove not normalizing Priority 0 to 1024
nelljerram Feb 27, 2026
18164b3
Add unit tests for routes with different priorities sharing the same …
nelljerram Feb 27, 2026
b436550
Add live migration routing sequence unit tests
nelljerram Mar 2, 2026
91a6e43
Document conntrack tracker limitation with multiple route priorities
nelljerram Mar 2, 2026
694b55a
Merge remote-tracking branch 'origin/master' into routing-priority
nelljerram Mar 2, 2026
a7f8081
Improve comment about live migration and conntrack state cleanup
nelljerram Mar 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions felix/dataplane/linux/bpf_ep_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3727,13 +3727,17 @@ func (m *bpfEndpointManager) ensureBPFDevices() error {
if m.v4 != nil {
m.routeTableV4.RouteUpdate(dataplanedefs.BPFInDev, routetable.Target{
Type: routetable.TargetTypeLinkLocalUnicast,
CIDR: bpfnatGWCIDR,
RouteKey: routetable.RouteKey{
CIDR: bpfnatGWCIDR,
},
})
}
if m.v6 != nil {
m.routeTableV6.RouteUpdate(dataplanedefs.BPFInDev, routetable.Target{
Type: routetable.TargetTypeLinkLocalUnicast,
CIDR: bpfnatGWCIDRv6,
RouteKey: routetable.RouteKey{
CIDR: bpfnatGWCIDRv6,
},
})
}

Expand Down Expand Up @@ -4450,7 +4454,9 @@ var (
func (m *bpfEndpointManager) setRoute(cidr ip.CIDR) {
target := routetable.Target{
Type: routetable.TargetTypeGlobalUnicast,
CIDR: cidr,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
}

if cidr.Version() == 6 {
Expand All @@ -4472,10 +4478,10 @@ func (m *bpfEndpointManager) setRoute(cidr ip.CIDR) {

func (m *bpfEndpointManager) delRoute(cidr ip.CIDR) {
if m.v6 != nil && cidr.Version() == 6 {
m.routeTableV6.RouteRemove(dataplanedefs.BPFInDev, cidr)
m.routeTableV6.RouteRemove(dataplanedefs.BPFInDev, routetable.RouteKey{CIDR: cidr})
}
if m.v4 != nil && cidr.Version() == 4 {
m.routeTableV4.RouteRemove(dataplanedefs.BPFInDev, cidr)
m.routeTableV4.RouteRemove(dataplanedefs.BPFInDev, routetable.RouteKey{CIDR: cidr})
}
logrus.WithFields(logrus.Fields{
"cidr": cidr,
Expand Down
5 changes: 4 additions & 1 deletion felix/dataplane/linux/endpoint_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -825,7 +825,10 @@ func (m *endpointManager) resolveWorkloadEndpoints() {
logCxt.Debug("Endpoint up, adding routes")
for _, s := range ipStrings {
routeTargets = append(routeTargets, routetable.Target{
CIDR: ip.MustParseCIDROrIP(s),
RouteKey: routetable.RouteKey{

CIDR: ip.MustParseCIDROrIP(s),
},
DestMAC: mac,
})
}
Expand Down
50 changes: 37 additions & 13 deletions felix/dataplane/linux/endpoint_mgr_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,7 @@ func (t *mockRouteTable) SetRoutes(routeClass routetable.RouteClass, ifaceName s
t.currentRoutes[ifaceName] = targets
}

func (t *mockRouteTable) RouteRemove(routeClass routetable.RouteClass, ifaceName string, cidr ip.CIDR) {
func (t *mockRouteTable) RouteRemove(routeClass routetable.RouteClass, ifaceName string, routeKey routetable.RouteKey) {
}

func (t *mockRouteTable) RouteUpdate(routeClass routetable.RouteClass, ifaceName string, target routetable.Target) {
Expand Down Expand Up @@ -1870,12 +1870,16 @@ func endpointManagerTests(ipVersion uint8, flowlogs bool) func() {
It("should set routes", func() {
if ipVersion == 6 {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
}})
} else {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
}})
}
Expand Down Expand Up @@ -1976,30 +1980,42 @@ func endpointManagerTests(ipVersion uint8, flowlogs bool) func() {
if ipVersion == 6 {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{
{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
{
CIDR: ip.MustParseCIDROrIP("2001:db8:3::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:3::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
{
CIDR: ip.MustParseCIDROrIP("2001:db8:4::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:4::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
})
} else {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{
{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
{
CIDR: ip.MustParseCIDROrIP("172.16.1.3/32"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("172.16.1.3/32"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
{
CIDR: ip.MustParseCIDROrIP("172.18.1.4/32"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("172.18.1.4/32"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
})
Expand Down Expand Up @@ -2045,14 +2061,18 @@ func endpointManagerTests(ipVersion uint8, flowlogs bool) func() {
if ipVersion == 6 {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{
{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
})
} else {
routeTable.checkRoutes("cali12345-ab", []routetable.Target{
{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
},
})
Expand Down Expand Up @@ -2117,12 +2137,16 @@ func endpointManagerTests(ipVersion uint8, flowlogs bool) func() {
It("should have set routes for new iface", func() {
if ipVersion == 6 {
routeTable.checkRoutes("cali12345-cd", []routetable.Target{{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("2001:db8:2::2/128"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
}})
} else {
routeTable.checkRoutes("cali12345-cd", []routetable.Target{{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.240.0/24"),
},
DestMAC: testutils.MustParseMAC("01:02:03:04:05:06"),
}})
}
Expand Down
6 changes: 4 additions & 2 deletions felix/dataplane/linux/ipip_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,10 @@ func (m *ipipManager) tunnelRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routetabl
}

return &routetable.Target{
Type: routetable.TargetTypeOnLink,
CIDR: cidr,
Type: routetable.TargetTypeOnLink,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
GW: ip.FromString(remoteAddr),
Protocol: m.routeProtocol,
MTU: m.dpConfig.IPIPMTU,
Expand Down
6 changes: 4 additions & 2 deletions felix/dataplane/linux/ipip_mgr_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,10 @@ var _ = Describe("IPIPManager", func() {
Expect(rt.currentRoutes[dataplanedefs.IPIPIfaceName]).To(HaveLen(1))
Expect(rt.currentRoutes[dataplanedefs.IPIPIfaceName][0]).To(Equal(
routetable.Target{
Type: "onlink",
CIDR: ip.MustParseCIDROrIP("10.0.1.1/32"),
Type: "onlink",
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.1.1/32"),
},
GW: ip.FromString("172.0.2.2"),
Protocol: 80,
}))
Expand Down
24 changes: 16 additions & 8 deletions felix/dataplane/linux/noencap_mgr_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,16 +142,20 @@ var _ = Describe("NoEncap Manager", func() {
Expect(rt.currentRoutes[routetable.InterfaceNone]).To(HaveLen(1))
Expect(rt.currentRoutes[routetable.InterfaceNone][0]).To(Equal(
routetable.Target{
Type: "blackhole",
CIDR: ip.MustParseCIDROrIP("192.168.0.100/26"),
Type: "blackhole",
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("192.168.0.100/26"),
},
Protocol: 80,
}))

Expect(rt.currentRoutes["eth0"]).To(HaveLen(1))
Expect(rt.currentRoutes["eth0"][0]).To(Equal(
routetable.Target{
Type: "noencap",
CIDR: ip.MustParseCIDROrIP("192.168.0.0/26"),
Type: "noencap",
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("192.168.0.0/26"),
},
GW: ip.FromString("172.0.2.2"),
Protocol: 80,
}))
Expand Down Expand Up @@ -225,16 +229,20 @@ var _ = Describe("NoEncap Manager", func() {
Expect(rt.currentRoutes[routetable.InterfaceNone]).To(HaveLen(1))
Expect(rt.currentRoutes[routetable.InterfaceNone][0]).To(Equal(
routetable.Target{
Type: "blackhole",
CIDR: ip.MustParseCIDROrIP("dead:beef::1:30/112"),
Type: "blackhole",
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("dead:beef::1:30/112"),
},
Protocol: 80,
}))

Expect(rt.currentRoutes["eth0"]).To(HaveLen(1))
Expect(rt.currentRoutes["eth0"][0]).To(Equal(
routetable.Target{
Type: "noencap",
CIDR: ip.MustParseCIDROrIP("dead:beef::2:10/112"),
Type: "noencap",
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("dead:beef::2:10/112"),
},
GW: ip.FromString("fc00:10:10::1"),
Protocol: 80,
}))
Expand Down
12 changes: 8 additions & 4 deletions felix/dataplane/linux/route_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,10 @@ func blackholeRoutes(localIPAMBlocks map[string]*proto.RouteUpdate, proto netlin
continue
}
rtt = append(rtt, routetable.Target{
Type: routetable.TargetTypeBlackhole,
CIDR: cidr,
Type: routetable.TargetTypeBlackhole,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
Protocol: proto,
})
}
Expand All @@ -421,8 +423,10 @@ func (m *routeManager) noEncapRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routeta
return nil
}
noEncapRoute := routetable.Target{
Type: routetable.TargetTypeNoEncap,
CIDR: cidr,
Type: routetable.TargetTypeNoEncap,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
GW: ip.FromString(r.DstNodeIp),
Protocol: m.routeProtocol,
}
Expand Down
14 changes: 9 additions & 5 deletions felix/dataplane/linux/vxlan_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,10 @@ func (m *vxlanManager) tunnelRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routetab
// We treat remote tunnel routes as directly connected. They don't have a gateway of
// the VTEP because they ARE the VTEP!
return &routetable.Target{
CIDR: cidr,
MTU: m.mtu,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
MTU: m.mtu,
}
}

Expand All @@ -286,9 +288,11 @@ func (m *vxlanManager) tunnelRoute(cidr ip.CIDR, r *proto.RouteUpdate) *routetab
}
return &routetable.Target{
Type: routetable.TargetTypeVXLAN,
CIDR: cidr,
GW: ip.FromString(vtepAddr),
MTU: m.mtu,
RouteKey: routetable.RouteKey{
CIDR: cidr,
},
GW: ip.FromString(vtepAddr),
MTU: m.mtu,
}
}

Expand Down
12 changes: 8 additions & 4 deletions felix/dataplane/linux/vxlan_mgr_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,8 +432,10 @@ var _ = Describe("VXLANManager", func() {
Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV4]).To(HaveLen(1))
Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV4][0]).To(Equal(
routetable.Target{
CIDR: ip.MustParseCIDROrIP("10.0.1.1/32"),
MTU: 4444,
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("10.0.1.1/32"),
},
MTU: 4444,
}))

// Delete the route.
Expand Down Expand Up @@ -466,8 +468,10 @@ var _ = Describe("VXLANManager", func() {
Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV6]).To(HaveLen(1))
Expect(rt.currentRoutes[dataplanedefs.VXLANIfaceNameV6][0]).To(Equal(
routetable.Target{
CIDR: ip.MustParseCIDROrIP("fc00:10:244::1/112"),
MTU: 6666,
RouteKey: routetable.RouteKey{
CIDR: ip.MustParseCIDROrIP("fc00:10:244::1/112"),
},
MTU: 6666,
}))

// Delete the route.
Expand Down
2 changes: 1 addition & 1 deletion felix/netlinkshim/mocknetlink/netlink.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1163,7 +1163,7 @@ func KeyForRoute(route *netlink.Route) string {
if table == 0 {
table = unix.RT_TABLE_MAIN
}
key := fmt.Sprintf("%v-%v", table, route.Dst)
key := fmt.Sprintf("%v-%v-%v", table, route.Dst, route.Priority)
log.WithField("routeKey", key).Debug("Calculated route key")
return key
}
Expand Down
4 changes: 3 additions & 1 deletion felix/routetable/bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ outer:
for i := range 256 {
for j := range 256 {
rt.RouteUpdate(RouteClassLocalWorkload, ifaceName, Target{
CIDR: ip.MustParseCIDROrIP(fmt.Sprintf("10.0.%d.%d/32", i, j)),
RouteKey: RouteKey{
CIDR: ip.MustParseCIDROrIP(fmt.Sprintf("10.0.%d.%d/32", i, j)),
},
})
n++
if n == numRoutes {
Expand Down
24 changes: 23 additions & 1 deletion felix/routetable/defs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package routetable

import (
"errors"
"fmt"
"net"
"reflect"

Expand Down Expand Up @@ -69,9 +70,30 @@ var (
ErrIfaceDown = errors.New("interface down")
)

// RouteKey represents the kernel's FIB key, and is also what we use on the RouteTable API to
// identify each route. The kernel and RouteTable API allow routes to coexist as long as they have
// different keys.
type RouteKey struct {
// Destination CIDR; route matches traffic to this destination.
CIDR ip.CIDR
// TOS is the Type-of-Service field. For example, one app may mark its
// packets as "high importance" and that will take a different route to
// another app.
//
// Kernel uses the TOS=0 route if there isn't a more precise match.
TOS int
// Priority is the routing metric / distance. Given two routes with the
// same CIDR, the kernel prefers the route with the _lower_ priority.
Priority int
}

func (k RouteKey) String() string {
return fmt.Sprintf("%s(tos=%x metric=%d)", k.CIDR.String(), k.TOS, k.Priority)
}

type Target struct {
RouteKey
Type TargetType
CIDR ip.CIDR
GW ip.Addr
Src ip.Addr
DestMAC net.HardwareAddr
Expand Down
Loading