cni: fix netmask set in VPC and ipvlan mode

plugin/driver/drivers.go

@@ -14,7 +14,7 @@ type SetupConfig struct {
 	HostVETHName string
 
 	ContainerIfName string
-	ContainerIPNet  *terwayTypes.IPNetSet // ipNet type with mask /32 or /128
+	ContainerIPNet  *terwayTypes.IPNetSet
 	GatewayIP       *terwayTypes.IPSet
 	MTU             int
 	ENIIndex        int

plugin/driver/ipvlan.go

@@ -477,13 +477,13 @@ func (d *IPvlanDriver) teardownInitNamespace(parents map[int]struct{}, container
 			continue
 		}
 		if containerIP.IPv4 != nil {
-			err = exec(initLink, containerIP.IPv4)
+			err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv4))
 			if err != nil {
 				return err
 			}
 		}
 		if containerIP.IPv6 != nil {
-			err = exec(initLink, containerIP.IPv6)
+			err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv6))
 			if err != nil {
 				return err
 			}

plugin/driver/raw_nic.go

@@ -88,6 +88,7 @@ func (r *RawNicDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 		if err != nil {
 			return fmt.Errorf("error set link %s MTU %d, %w", nicLink.Attrs().Name, cfg.MTU, err)
 		}
+		IPNetToMaxMask(cfg.ContainerIPNet)
 		err = SetupLink(nicLink, cfg)
 		if err != nil {
 			return err

plugin/driver/utils.go

@@ -44,6 +44,11 @@ func NewDefaultLogger() *logrus.Logger {
 	return logger
 }
 
+func SetLogDebug() {
+	DefaultLogger.SetLevel(logrus.DebugLevel)
+	logrus.SetOutput(os.Stderr)
+}
+
 // JSONStr json to str
 func JSONStr(v interface{}) string {
 	b, err := json.Marshal(v)
@@ -328,7 +333,7 @@ func EnsureHostToContainerRoute(link netlink.Link, ipNetSet *terwayTypes.IPNetSe
 		err := exec(&netlink.Route{
 			LinkIndex: linkIndex,
 			Scope:     netlink.SCOPE_LINK,
-			Dst:       ipNetSet.IPv4,
+			Dst:       NewIPNetWithMaxMask(ipNetSet.IPv4),
 		})
 		if err != nil {
 			return changed, err
@@ -338,7 +343,7 @@ func EnsureHostToContainerRoute(link netlink.Link, ipNetSet *terwayTypes.IPNetSe
 		err := exec(&netlink.Route{
 			LinkIndex: linkIndex,
 			Scope:     netlink.SCOPE_LINK,
-			Dst:       ipNetSet.IPv6,
+			Dst:       NewIPNetWithMaxMask(ipNetSet.IPv6),
 		})
 		if err != nil {
 			return changed, err
@@ -418,6 +423,15 @@ func NewIPNetWithMaxMask(ipNet *net.IPNet) *net.IPNet {
 	}
 }
 
+func IPNetToMaxMask(ipNet *terwayTypes.IPNetSet) {
+	if ipNet.IPv4 != nil {
+		ipNet.IPv4 = NewIPNetWithMaxMask(ipNet.IPv4)
+	}
+	if ipNet.IPv6 != nil {
+		ipNet.IPv6 = NewIPNetWithMaxMask(ipNet.IPv6)
+	}
+}
+
 func FindIPRules(ipNet *net.IPNet, found func(rule *netlink.Rule) error) error {
 	var ruleList []netlink.Rule
 	var err error
@@ -499,49 +513,52 @@ func EnsurePolicyRule(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, tableID
 	}
 
 	if ipNetSet.IPv4 != nil {
+		v4 := NewIPNetWithMaxMask(ipNetSet.IPv4)
 		// 2. add host to container rule
 		toContainerRule := netlink.NewRule()
-		toContainerRule.Dst = ipNetSet.IPv4
+		toContainerRule.Dst = v4
 		toContainerRule.Table = mainRouteTable
 		toContainerRule.Priority = toContainerPriority
 
-		err := exec(ipNetSet.IPv4, toContainerRule)
+		err := exec(v4, toContainerRule)
 		if err != nil {
 			return changed, err
 		}
 
 		// 3. add from container rule
 		fromContainerRule := netlink.NewRule()
 		fromContainerRule.IifName = link.Attrs().Name
-		fromContainerRule.Src = ipNetSet.IPv4
+		fromContainerRule.Src = v4
 		fromContainerRule.Table = tableID
 		fromContainerRule.Priority = fromContainerPriority
 
-		err = exec(ipNetSet.IPv4, fromContainerRule)
+		err = exec(v4, fromContainerRule)
 		if err != nil {
 			return changed, err
 		}
 	}
 	if ipNetSet.IPv6 != nil {
+		v6 := NewIPNetWithMaxMask(ipNetSet.IPv6)
+
 		// 2. add host to container rule
 		toContainerRule := netlink.NewRule()
-		toContainerRule.Dst = ipNetSet.IPv6
+		toContainerRule.Dst = v6
 		toContainerRule.Table = mainRouteTable
 		toContainerRule.Priority = toContainerPriority
 
-		err := exec(ipNetSet.IPv6, toContainerRule)
+		err := exec(v6, toContainerRule)
 		if err != nil {
 			return changed, err
 		}
 
 		// 3. add from container rule
 		fromContainerRule := netlink.NewRule()
 		fromContainerRule.IifName = link.Attrs().Name
-		fromContainerRule.Src = ipNetSet.IPv6
+		fromContainerRule.Src = v6
 		fromContainerRule.Table = tableID
 		fromContainerRule.Priority = fromContainerPriority
 
-		err = exec(ipNetSet.IPv6, fromContainerRule)
+		err = exec(v6, fromContainerRule)
 		if err != nil {
 			return changed, err
 		}

plugin/driver/veth.go

@@ -84,7 +84,7 @@ func (d *VETHDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 		if err != nil {
 			return fmt.Errorf("error find link %s in container, %w", contVETH.Attrs().Name, err)
 		}
-
+		IPNetToMaxMask(cfg.ContainerIPNet)
 		err = SetupLink(contLink, cfg)
 		if err != nil {
 			return err

plugin/terway/cni.go

@@ -17,7 +17,6 @@ import (
 	"github.com/AliyunContainerService/terway/rpc"
 	terwayTypes "github.com/AliyunContainerService/terway/types"
 	"github.com/AliyunContainerService/terway/version"
-	"github.com/sirupsen/logrus"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
@@ -158,7 +157,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	defer cniNetns.Close()
 
 	if conf.Debug {
-		driver.DefaultLogger.SetLevel(logrus.DebugLevel)
+		driver.SetLogDebug()
 	}
 	logger = logger.WithFields(map[string]interface{}{
 		"netns":        args.Netns,
@@ -231,9 +230,6 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}()
 
 	hostVETHName, _ := link.VethNameForPod(string(k8sConfig.K8S_POD_NAME), string(k8sConfig.K8S_POD_NAMESPACE), defaultVethPrefix)
-	var (
-		allocatedIPAddr net.IPNet
-	)
 
 	var containerIPNet *terwayTypes.IPNetSet
 	var gatewayIPSet *terwayTypes.IPSet
@@ -244,13 +240,14 @@ func cmdAdd(args *skel.CmdArgs) error {
 			return fmt.Errorf("eni multi ip return result is empty: %v", allocResult)
 		}
 		podIP := allocResult.GetENIMultiIP().GetENIConfig().GetPodIP()
+		subNet := allocResult.GetENIMultiIP().GetENIConfig().GetSubnet()
 		gatewayIP := allocResult.GetENIMultiIP().GetENIConfig().GetGatewayIP()
 		eniMAC := allocResult.GetENIMultiIP().GetENIConfig().GetMAC()
 		ingress := allocResult.GetENIMultiIP().GetPodConfig().GetIngress()
 		egress := allocResult.GetENIMultiIP().GetPodConfig().GetEgress()
 		serviceCIDR := allocResult.GetENIMultiIP().GetServiceCIDR()
 
-		containerIPNet, err = terwayTypes.BuildIPNet(podIP, &rpc.IPSet{IPv4: "0.0.0.0/32", IPv6: "::/128"})
+		containerIPNet, err = terwayTypes.BuildIPNet(podIP, subNet)
 		if err != nil {
 			return err
 		}
@@ -356,6 +353,13 @@ func cmdAdd(args *skel.CmdArgs) error {
 		podIPAddr := ipamResult.IPs[0].Address
 		gateway := ipamResult.IPs[0].Gateway
 
+		containerIPNet = &terwayTypes.IPNetSet{
+			IPv4: &podIPAddr,
+		}
+		gatewayIPSet = &terwayTypes.IPSet{
+			IPv4: gateway,
+		}
+
 		ingress := allocResult.GetVPCIP().GetPodConfig().GetIngress()
 		egress := allocResult.GetVPCIP().GetPodConfig().GetEgress()
 		l, err := driver.GrabFileLock(terwayCNILock)
@@ -368,15 +372,11 @@ func cmdAdd(args *skel.CmdArgs) error {
 		setupCfg := &driver.SetupConfig{
 			HostVETHName:    hostVETHName,
 			ContainerIfName: args.IfName,
-			ContainerIPNet: &terwayTypes.IPNetSet{
-				IPv4: &podIPAddr,
-			},
-			GatewayIP: &terwayTypes.IPSet{
-				IPv4: gateway,
-			},
-			MTU:     conf.MTU,
-			Ingress: ingress,
-			Egress:  egress,
+			ContainerIPNet:  containerIPNet,
+			GatewayIP:       gatewayIPSet,
+			MTU:             conf.MTU,
+			Ingress:         ingress,
+			Egress:          egress,
 		}
 
 		err = veth.Setup(setupCfg, cniNetns)
@@ -508,7 +508,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 			K8SPodNamespace: string(k8sConfig.K8S_POD_NAMESPACE),
 			EventType:       rpc.EventType_EventTypeNormal,
 			Reason:          "AllocIPSucceed",
-			Message:         fmt.Sprintf("Alloc IP %s for Pod", allocatedIPAddr.String()),
+			Message:         fmt.Sprintf("Alloc IP %s for Pod", containerIPNet.String()),
 		})
 
 	return types.PrintResult(result, confVersion)
@@ -523,7 +523,7 @@ func cmdDel(args *skel.CmdArgs) error {
 	defer cniNetns.Close()
 
 	if conf.Debug {
-		driver.DefaultLogger.SetLevel(logrus.DebugLevel)
+		driver.SetLogDebug()
 	}
 	logger = logger.WithFields(map[string]interface{}{
 		"netns":        args.Netns,
@@ -676,7 +676,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 	defer cniNetns.Close()
 
 	if conf.Debug {
-		driver.DefaultLogger.SetLevel(logrus.DebugLevel)
+		driver.SetLogDebug()
 	}
 	logger = logger.WithFields(map[string]interface{}{
 		"netns":        args.Netns,
@@ -720,8 +720,9 @@ func cmdCheck(args *skel.CmdArgs) error {
 		}
 
 		podIP := getResult.GetENIMultiIP().GetENIConfig().GetPodIP()
+		subNet := getResult.GetENIMultiIP().GetENIConfig().GetSubnet()
 
-		containerIPNet, err = terwayTypes.BuildIPNet(podIP, &rpc.IPSet{IPv4: "0.0.0.0/32", IPv6: "::/128"})
+		containerIPNet, err = terwayTypes.BuildIPNet(podIP, subNet)
 		if err != nil {
 			return err
 		}

types/types.go

@@ -123,6 +123,16 @@ func (i *IPNetSet) ToRPC() *rpc.IPSet {
 		IPv6: ipv6,
 	}
 }
+func (i *IPNetSet) String() string {
+	var result []string
+	if i.IPv4 != nil {
+		result = append(result, i.IPv4.String())
+	}
+	if i.IPv6 != nil {
+		result = append(result, i.IPv6.String())
+	}
+	return strings.Join(result, "-")
+}
 
 // ENI aliyun ENI resource
 type ENI struct {