From 4351f69336155a085519b9337d0b01aa89ef1bc1 Mon Sep 17 00:00:00 2001 From: l1b0k Date: Fri, 21 May 2021 15:02:22 +0800 Subject: [PATCH] cni: fix netmask set in VPC and ipvlan mode Signed-off-by: l1b0k --- plugin/driver/drivers.go | 2 +- plugin/driver/ipvlan.go | 4 ++-- plugin/driver/raw_nic.go | 1 + plugin/driver/utils.go | 37 +++++++++++++++++++++++++++---------- plugin/driver/veth.go | 2 +- plugin/terway/cni.go | 39 ++++++++++++++++++++------------------- types/types.go | 10 ++++++++++ 7 files changed, 62 insertions(+), 33 deletions(-) diff --git a/plugin/driver/drivers.go b/plugin/driver/drivers.go index 16e63b1c..e581d9af 100644 --- a/plugin/driver/drivers.go +++ b/plugin/driver/drivers.go @@ -14,7 +14,7 @@ type SetupConfig struct { HostVETHName string ContainerIfName string - ContainerIPNet *terwayTypes.IPNetSet // ipNet type with mask /32 or /128 + ContainerIPNet *terwayTypes.IPNetSet GatewayIP *terwayTypes.IPSet MTU int ENIIndex int diff --git a/plugin/driver/ipvlan.go b/plugin/driver/ipvlan.go index 4acced6b..9bedaf92 100644 --- a/plugin/driver/ipvlan.go +++ b/plugin/driver/ipvlan.go @@ -477,13 +477,13 @@ func (d *IPvlanDriver) teardownInitNamespace(parents map[int]struct{}, container continue } if containerIP.IPv4 != nil { - err = exec(initLink, containerIP.IPv4) + err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv4)) if err != nil { return err } } if containerIP.IPv6 != nil { - err = exec(initLink, containerIP.IPv6) + err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv6)) if err != nil { return err } diff --git a/plugin/driver/raw_nic.go b/plugin/driver/raw_nic.go index 9672c604..d0b493ce 100644 --- a/plugin/driver/raw_nic.go +++ b/plugin/driver/raw_nic.go @@ -88,6 +88,7 @@ func (r *RawNicDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error { if err != nil { return fmt.Errorf("error set link %s MTU %d, %w", nicLink.Attrs().Name, cfg.MTU, err) } + IPNetToMaxMask(cfg.ContainerIPNet) err = SetupLink(nicLink, cfg) if err != nil { return err diff --git a/plugin/driver/utils.go b/plugin/driver/utils.go index 6fc8b703..eef556b9 100644 --- a/plugin/driver/utils.go +++ b/plugin/driver/utils.go @@ -44,6 +44,11 @@ func NewDefaultLogger() *logrus.Logger { return logger } +func SetLogDebug() { + DefaultLogger.SetLevel(logrus.DebugLevel) + logrus.SetOutput(os.Stderr) +} + // JSONStr json to str func JSONStr(v interface{}) string { b, err := json.Marshal(v) @@ -328,7 +333,7 @@ func EnsureHostToContainerRoute(link netlink.Link, ipNetSet *terwayTypes.IPNetSe err := exec(&netlink.Route{ LinkIndex: linkIndex, Scope: netlink.SCOPE_LINK, - Dst: ipNetSet.IPv4, + Dst: NewIPNetWithMaxMask(ipNetSet.IPv4), }) if err != nil { return changed, err @@ -338,7 +343,7 @@ func EnsureHostToContainerRoute(link netlink.Link, ipNetSet *terwayTypes.IPNetSe err := exec(&netlink.Route{ LinkIndex: linkIndex, Scope: netlink.SCOPE_LINK, - Dst: ipNetSet.IPv6, + Dst: NewIPNetWithMaxMask(ipNetSet.IPv6), }) if err != nil { return changed, err @@ -418,6 +423,15 @@ func NewIPNetWithMaxMask(ipNet *net.IPNet) *net.IPNet { } } +func IPNetToMaxMask(ipNet *terwayTypes.IPNetSet) { + if ipNet.IPv4 != nil { + ipNet.IPv4 = NewIPNetWithMaxMask(ipNet.IPv4) + } + if ipNet.IPv6 != nil { + ipNet.IPv6 = NewIPNetWithMaxMask(ipNet.IPv6) + } +} + func FindIPRules(ipNet *net.IPNet, found func(rule *netlink.Rule) error) error { var ruleList []netlink.Rule var err error @@ -499,13 +513,14 @@ func EnsurePolicyRule(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, tableID } if ipNetSet.IPv4 != nil { + v4 := NewIPNetWithMaxMask(ipNetSet.IPv4) // 2. add host to container rule toContainerRule := netlink.NewRule() - toContainerRule.Dst = ipNetSet.IPv4 + toContainerRule.Dst = v4 toContainerRule.Table = mainRouteTable toContainerRule.Priority = toContainerPriority - err := exec(ipNetSet.IPv4, toContainerRule) + err := exec(v4, toContainerRule) if err != nil { return changed, err } @@ -513,23 +528,25 @@ func EnsurePolicyRule(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, tableID // 3. add from container rule fromContainerRule := netlink.NewRule() fromContainerRule.IifName = link.Attrs().Name - fromContainerRule.Src = ipNetSet.IPv4 + fromContainerRule.Src = v4 fromContainerRule.Table = tableID fromContainerRule.Priority = fromContainerPriority - err = exec(ipNetSet.IPv4, fromContainerRule) + err = exec(v4, fromContainerRule) if err != nil { return changed, err } } if ipNetSet.IPv6 != nil { + v6 := NewIPNetWithMaxMask(ipNetSet.IPv6) + // 2. add host to container rule toContainerRule := netlink.NewRule() - toContainerRule.Dst = ipNetSet.IPv6 + toContainerRule.Dst = v6 toContainerRule.Table = mainRouteTable toContainerRule.Priority = toContainerPriority - err := exec(ipNetSet.IPv6, toContainerRule) + err := exec(v6, toContainerRule) if err != nil { return changed, err } @@ -537,11 +554,11 @@ func EnsurePolicyRule(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, tableID // 3. add from container rule fromContainerRule := netlink.NewRule() fromContainerRule.IifName = link.Attrs().Name - fromContainerRule.Src = ipNetSet.IPv6 + fromContainerRule.Src = v6 fromContainerRule.Table = tableID fromContainerRule.Priority = fromContainerPriority - err = exec(ipNetSet.IPv6, fromContainerRule) + err = exec(v6, fromContainerRule) if err != nil { return changed, err } diff --git a/plugin/driver/veth.go b/plugin/driver/veth.go index 9022defe..f880fdd1 100644 --- a/plugin/driver/veth.go +++ b/plugin/driver/veth.go @@ -84,7 +84,7 @@ func (d *VETHDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error { if err != nil { return fmt.Errorf("error find link %s in container, %w", contVETH.Attrs().Name, err) } - + IPNetToMaxMask(cfg.ContainerIPNet) err = SetupLink(contLink, cfg) if err != nil { return err diff --git a/plugin/terway/cni.go b/plugin/terway/cni.go index f9663105..837c60ec 100644 --- a/plugin/terway/cni.go +++ b/plugin/terway/cni.go @@ -17,7 +17,6 @@ import ( "github.com/AliyunContainerService/terway/rpc" terwayTypes "github.com/AliyunContainerService/terway/types" "github.com/AliyunContainerService/terway/version" - "github.com/sirupsen/logrus" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" @@ -158,7 +157,7 @@ func cmdAdd(args *skel.CmdArgs) error { defer cniNetns.Close() if conf.Debug { - driver.DefaultLogger.SetLevel(logrus.DebugLevel) + driver.SetLogDebug() } logger = logger.WithFields(map[string]interface{}{ "netns": args.Netns, @@ -231,9 +230,6 @@ func cmdAdd(args *skel.CmdArgs) error { }() hostVETHName, _ := link.VethNameForPod(string(k8sConfig.K8S_POD_NAME), string(k8sConfig.K8S_POD_NAMESPACE), defaultVethPrefix) - var ( - allocatedIPAddr net.IPNet - ) var containerIPNet *terwayTypes.IPNetSet var gatewayIPSet *terwayTypes.IPSet @@ -244,13 +240,14 @@ func cmdAdd(args *skel.CmdArgs) error { return fmt.Errorf("eni multi ip return result is empty: %v", allocResult) } podIP := allocResult.GetENIMultiIP().GetENIConfig().GetPodIP() + subNet := allocResult.GetENIMultiIP().GetENIConfig().GetSubnet() gatewayIP := allocResult.GetENIMultiIP().GetENIConfig().GetGatewayIP() eniMAC := allocResult.GetENIMultiIP().GetENIConfig().GetMAC() ingress := allocResult.GetENIMultiIP().GetPodConfig().GetIngress() egress := allocResult.GetENIMultiIP().GetPodConfig().GetEgress() serviceCIDR := allocResult.GetENIMultiIP().GetServiceCIDR() - containerIPNet, err = terwayTypes.BuildIPNet(podIP, &rpc.IPSet{IPv4: "0.0.0.0/32", IPv6: "::/128"}) + containerIPNet, err = terwayTypes.BuildIPNet(podIP, subNet) if err != nil { return err } @@ -356,6 +353,13 @@ func cmdAdd(args *skel.CmdArgs) error { podIPAddr := ipamResult.IPs[0].Address gateway := ipamResult.IPs[0].Gateway + containerIPNet = &terwayTypes.IPNetSet{ + IPv4: &podIPAddr, + } + gatewayIPSet = &terwayTypes.IPSet{ + IPv4: gateway, + } + ingress := allocResult.GetVPCIP().GetPodConfig().GetIngress() egress := allocResult.GetVPCIP().GetPodConfig().GetEgress() l, err := driver.GrabFileLock(terwayCNILock) @@ -368,15 +372,11 @@ func cmdAdd(args *skel.CmdArgs) error { setupCfg := &driver.SetupConfig{ HostVETHName: hostVETHName, ContainerIfName: args.IfName, - ContainerIPNet: &terwayTypes.IPNetSet{ - IPv4: &podIPAddr, - }, - GatewayIP: &terwayTypes.IPSet{ - IPv4: gateway, - }, - MTU: conf.MTU, - Ingress: ingress, - Egress: egress, + ContainerIPNet: containerIPNet, + GatewayIP: gatewayIPSet, + MTU: conf.MTU, + Ingress: ingress, + Egress: egress, } err = veth.Setup(setupCfg, cniNetns) @@ -508,7 +508,7 @@ func cmdAdd(args *skel.CmdArgs) error { K8SPodNamespace: string(k8sConfig.K8S_POD_NAMESPACE), EventType: rpc.EventType_EventTypeNormal, Reason: "AllocIPSucceed", - Message: fmt.Sprintf("Alloc IP %s for Pod", allocatedIPAddr.String()), + Message: fmt.Sprintf("Alloc IP %s for Pod", containerIPNet.String()), }) return types.PrintResult(result, confVersion) @@ -523,7 +523,7 @@ func cmdDel(args *skel.CmdArgs) error { defer cniNetns.Close() if conf.Debug { - driver.DefaultLogger.SetLevel(logrus.DebugLevel) + driver.SetLogDebug() } logger = logger.WithFields(map[string]interface{}{ "netns": args.Netns, @@ -676,7 +676,7 @@ func cmdCheck(args *skel.CmdArgs) error { defer cniNetns.Close() if conf.Debug { - driver.DefaultLogger.SetLevel(logrus.DebugLevel) + driver.SetLogDebug() } logger = logger.WithFields(map[string]interface{}{ "netns": args.Netns, @@ -720,8 +720,9 @@ func cmdCheck(args *skel.CmdArgs) error { } podIP := getResult.GetENIMultiIP().GetENIConfig().GetPodIP() + subNet := getResult.GetENIMultiIP().GetENIConfig().GetSubnet() - containerIPNet, err = terwayTypes.BuildIPNet(podIP, &rpc.IPSet{IPv4: "0.0.0.0/32", IPv6: "::/128"}) + containerIPNet, err = terwayTypes.BuildIPNet(podIP, subNet) if err != nil { return err } diff --git a/types/types.go b/types/types.go index 2119a7b9..5879690c 100644 --- a/types/types.go +++ b/types/types.go @@ -123,6 +123,16 @@ func (i *IPNetSet) ToRPC() *rpc.IPSet { IPv6: ipv6, } } +func (i *IPNetSet) String() string { + var result []string + if i.IPv4 != nil { + result = append(result, i.IPv4.String()) + } + if i.IPv6 != nil { + result = append(result, i.IPv6.String()) + } + return strings.Join(result, "-") +} // ENI aliyun ENI resource type ENI struct {