Skip to content

Commit 4ab7a38

Browse files
BSWANGBSWANG
authored
Merge pull request #746 from l1b0k/feat/fix_rule
fix: repair the missing ip rules and routes
2 parents e8c2532 + 931b5da commit 4ab7a38

File tree

7 files changed

+239
-4
lines changed

7 files changed

+239
-4
lines changed

‎cmd/terway/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ func main() {
5757
log.Info(version.Version)
5858

5959
ctx := ctrl.SetupSignalHandler()
60+
ctx = ctrl.LoggerInto(ctx, ctrl.Log)
6061
err = daemon.Run(ctx, utils.NormalizePath(defaultSocketPath), readonlyListen, utils.NormalizePath(configFilePath), daemonMode)
6162

6263
if err != nil {

‎daemon/daemon.go

+4
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,10 @@ func (n *networkService) gcPods(ctx context.Context) error {
563563

564564
podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name)
565565
if _, ok := exist[podID]; ok {
566+
err = ruleSync(ctx, podRes)
567+
if err != nil {
568+
serviceLog.Error(err, "error sync pod rule")
569+
}
566570
continue
567571
}
568572

‎daemon/rule_linux.go

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package daemon
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
7+
"github.com/samber/lo"
8+
"github.com/vishvananda/netlink"
9+
10+
"github.com/AliyunContainerService/terway/pkg/link"
11+
"github.com/AliyunContainerService/terway/pkg/utils/nodecap"
12+
"github.com/AliyunContainerService/terway/plugin/datapath"
13+
"github.com/AliyunContainerService/terway/plugin/driver/types"
14+
"github.com/AliyunContainerService/terway/plugin/driver/utils"
15+
"github.com/AliyunContainerService/terway/rpc"
16+
terwayTypes "github.com/AliyunContainerService/terway/types"
17+
"github.com/AliyunContainerService/terway/types/daemon"
18+
)
19+
20+
func ruleSync(ctx context.Context, res daemon.PodResources) error {
21+
if res.PodInfo == nil {
22+
return nil
23+
}
24+
25+
if res.PodInfo.PodNetworkType != daemon.PodNetworkTypeENIMultiIP {
26+
return nil
27+
}
28+
29+
switch nodecap.GetNodeCapabilities(nodecap.NodeCapabilityDataPath) {
30+
case "datapathv2", "veth", "":
31+
default:
32+
return nil
33+
}
34+
35+
netConf := make([]*rpc.NetConf, 0)
36+
err := json.Unmarshal([]byte(res.NetConf), &netConf)
37+
if err != nil {
38+
return nil
39+
}
40+
41+
links, err := netlink.LinkList()
42+
if err != nil {
43+
return err
44+
}
45+
46+
for _, conf := range netConf {
47+
if conf.BasicInfo == nil || conf.ENIInfo == nil ||
48+
conf.BasicInfo.PodIP == nil {
49+
continue
50+
}
51+
ifName := "eth0"
52+
if conf.IfName != "" {
53+
ifName = conf.IfName
54+
}
55+
56+
hostVethName, _ := link.VethNameForPod(res.PodInfo.Name, res.PodInfo.Namespace, ifName, "cali")
57+
58+
// check host veth ,make sure pod is present
59+
hostVeth, ok := lo.Find(links, func(item netlink.Link) bool {
60+
return hostVethName == item.Attrs().Name
61+
})
62+
if !ok {
63+
continue
64+
}
65+
66+
eni, ok := lo.Find(links, func(item netlink.Link) bool {
67+
if _, ok := item.(*netlink.Device); !ok {
68+
return false
69+
}
70+
return item.Attrs().HardwareAddr.String() == conf.ENIInfo.MAC
71+
})
72+
if !ok {
73+
continue
74+
}
75+
76+
setUp := &types.SetupConfig{
77+
ContainerIPNet: &terwayTypes.IPNetSet{},
78+
GatewayIP: &terwayTypes.IPSet{},
79+
ENIIndex: eni.Attrs().Index,
80+
}
81+
if conf.BasicInfo.PodIP.IPv4 != "" {
82+
setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv4 + "/32")
83+
}
84+
if conf.BasicInfo.PodIP.IPv6 != "" {
85+
setUp.ContainerIPNet.SetIPNet(conf.BasicInfo.PodIP.IPv6 + "/128")
86+
}
87+
setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv4)
88+
setUp.GatewayIP.SetIP(conf.BasicInfo.GatewayIP.IPv6)
89+
90+
// 1. route point to hostVeth
91+
table := utils.GetRouteTableID(eni.Attrs().Index)
92+
93+
eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, table)
94+
hostVethConf := datapath.GenerateHostPeerCfgForPolicy(setUp, hostVeth, table)
95+
96+
// default via 10.xx.xx.253 dev eth1 onlink table 1003
97+
for _, route := range eniConf.Routes {
98+
_, err = utils.EnsureRoute(ctx, route)
99+
if err != nil {
100+
return err
101+
}
102+
}
103+
// 10.xx.xx.xx dev calixx scope link
104+
for _, route := range hostVethConf.Routes {
105+
_, err = utils.EnsureRoute(ctx, route)
106+
if err != nil {
107+
return err
108+
}
109+
}
110+
111+
for _, rule := range hostVethConf.Rules {
112+
_, err = utils.EnsureIPRule(ctx, rule)
113+
if err != nil {
114+
return err
115+
}
116+
}
117+
}
118+
119+
return nil
120+
}

‎daemon/rule_linux_test.go

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
//go:build linux
2+
3+
package daemon
4+
5+
import (
6+
"net"
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/vishvananda/netlink"
11+
12+
"github.com/AliyunContainerService/terway/plugin/datapath"
13+
"github.com/AliyunContainerService/terway/plugin/driver/types"
14+
terwayTypes "github.com/AliyunContainerService/terway/types"
15+
)
16+
17+
func TestGenerateConfig(t *testing.T) {
18+
setUp := &types.SetupConfig{
19+
ContainerIPNet: &terwayTypes.IPNetSet{
20+
IPv4: &net.IPNet{
21+
IP: net.ParseIP("10.0.0.2"),
22+
Mask: net.CIDRMask(32, 32),
23+
},
24+
IPv6: &net.IPNet{
25+
IP: net.ParseIP("fd0::2"),
26+
Mask: net.CIDRMask(128, 128),
27+
},
28+
},
29+
GatewayIP: &terwayTypes.IPSet{
30+
IPv4: net.ParseIP("169.254.169.254"),
31+
IPv6: net.ParseIP("fd0::1"),
32+
},
33+
ENIIndex: 1,
34+
}
35+
eni := &netlink.GenericLink{
36+
LinkAttrs: netlink.LinkAttrs{
37+
Index: 1,
38+
Name: "eth1",
39+
},
40+
}
41+
eniConf := datapath.GenerateENICfgForPolicy(setUp, eni, 1)
42+
43+
assert.True(t, eniConf.Routes[0].Equal(netlink.Route{
44+
Dst: &net.IPNet{
45+
IP: net.ParseIP("0.0.0.0"),
46+
Mask: net.CIDRMask(0, 0),
47+
},
48+
Gw: net.ParseIP("169.254.169.254"),
49+
Table: 1,
50+
LinkIndex: 1,
51+
Scope: netlink.SCOPE_UNIVERSE,
52+
Flags: int(netlink.FLAG_ONLINK),
53+
}))
54+
assert.True(t, eniConf.Routes[2].Equal(netlink.Route{
55+
Dst: &net.IPNet{
56+
IP: net.ParseIP("::"),
57+
Mask: net.CIDRMask(0, 0),
58+
},
59+
Gw: net.ParseIP("fd0::1"),
60+
Table: 1,
61+
LinkIndex: 1,
62+
Scope: netlink.SCOPE_UNIVERSE,
63+
Flags: int(netlink.FLAG_ONLINK),
64+
}))
65+
}
66+
67+
func TestGenerateHostPeerCfgForPolicy(t *testing.T) {
68+
setUp := &types.SetupConfig{
69+
ContainerIPNet: &terwayTypes.IPNetSet{
70+
IPv4: &net.IPNet{
71+
IP: net.ParseIP("10.0.0.2"),
72+
Mask: net.CIDRMask(32, 32),
73+
},
74+
IPv6: &net.IPNet{
75+
IP: net.ParseIP("fd0::2"),
76+
Mask: net.CIDRMask(128, 128),
77+
},
78+
},
79+
GatewayIP: &terwayTypes.IPSet{
80+
IPv4: net.ParseIP("169.254.169.254"),
81+
IPv6: net.ParseIP("fd0::1"),
82+
},
83+
ENIIndex: 1,
84+
}
85+
veth := &netlink.GenericLink{
86+
LinkAttrs: netlink.LinkAttrs{
87+
Index: 1,
88+
Name: "calixxx",
89+
},
90+
}
91+
vethConf := datapath.GenerateHostPeerCfgForPolicy(setUp, veth, 1)
92+
assert.Equal(t, 2, len(vethConf.Routes))
93+
assert.Equal(t, 4, len(vethConf.Rules))
94+
}

‎daemon/rule_unsupported.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//go:build !linux
2+
3+
package daemon
4+
5+
import (
6+
"context"
7+
8+
"github.com/AliyunContainerService/terway/types/daemon"
9+
)
10+
11+
func ruleSync(ctx context.Context, res daemon.PodResources) error {
12+
return nil
13+
}

‎plugin/datapath/policy_router_linux.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ func generateContCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, mac net
172172
return contCfg
173173
}
174174

175-
func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
175+
func GenerateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
176176
var addrs []*netlink.Addr
177177
var routes []*netlink.Route
178178
var rules []*netlink.Rule
@@ -247,7 +247,7 @@ func generateHostPeerCfgForPolicy(cfg *types.SetupConfig, link netlink.Link, tab
247247
}
248248
}
249249

250-
func generateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
250+
func GenerateENICfgForPolicy(cfg *types.SetupConfig, link netlink.Link, table int) *nic.Conf {
251251
var routes []*netlink.Route
252252
var rules []*netlink.Rule
253253
var neighs []*netlink.Neigh
@@ -377,7 +377,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n
377377

378378
table := utils.GetRouteTableID(eni.Attrs().Index)
379379

380-
eniCfg := generateENICfgForPolicy(cfg, eni, table)
380+
eniCfg := GenerateENICfgForPolicy(cfg, eni, table)
381381
err = nic.Setup(ctx, eni, eniCfg)
382382
if err != nil {
383383
return fmt.Errorf("setup eni config, %w", err)
@@ -390,7 +390,7 @@ func (d *PolicyRoute) Setup(ctx context.Context, cfg *types.SetupConfig, netNS n
390390
}
391391
}
392392

393-
hostVETHCfg := generateHostPeerCfgForPolicy(cfg, hostVETH, table)
393+
hostVETHCfg := GenerateHostPeerCfgForPolicy(cfg, hostVETH, table)
394394
err = nic.Setup(ctx, hostVETH, hostVETHCfg)
395395
if err != nil {
396396
return fmt.Errorf("setup host veth config, %w", err)

‎plugin/driver/utils/utils_linux.go

+3
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ func NewIPNet1(ipNet *terwayTypes.IPNetSet) []*netlink.Addr {
174174
}
175175

176176
func NewIPNetToMaxMask(ipNet *terwayTypes.IPNetSet) []*netlink.Addr {
177+
if ipNet == nil {
178+
return nil
179+
}
177180
var addrs []*netlink.Addr
178181
if ipNet.IPv4 != nil {
179182
addrs = append(addrs, &netlink.Addr{IPNet: NewIPNetWithMaxMask(ipNet.IPv4)})

0 commit comments

Comments
 (0)