Merge pull request #171 from l1b0k/master

repair ip rule

daemon/daemon.go

@@ -12,15 +12,16 @@ import (
 	"time"
 
 	"github.com/AliyunContainerService/terway/pkg/aliyun"
+	"github.com/AliyunContainerService/terway/pkg/link"
 	"github.com/AliyunContainerService/terway/pkg/metric"
 	"github.com/AliyunContainerService/terway/pkg/pool"
 	"github.com/AliyunContainerService/terway/pkg/storage"
 	"github.com/AliyunContainerService/terway/pkg/tracing"
 	"github.com/AliyunContainerService/terway/rpc"
 	"github.com/AliyunContainerService/terway/types"
 	"github.com/containernetworking/cni/libcni"
-	containertypes "github.com/containernetworking/cni/pkg/types"
 
+	containertypes "github.com/containernetworking/cni/pkg/types"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
@@ -649,6 +650,31 @@ func (networkService *networkService) startGarbageCollectionLoop() {
 						}
 					} else {
 						relateExpireList = append(relateExpireList, podInfoKey(resRelate.PodInfo.Namespace, resRelate.PodInfo.Name))
+
+						// delete resource first
+						func() {
+							if networkService.daemonMode != daemonModeENIMultiIP {
+								return
+							}
+							// try clean ip rules
+							if resRelate.PodInfo.PodIP == "" {
+								return
+							}
+							_, addr, err := net.ParseCIDR(fmt.Sprintf("%s/32", resRelate.PodInfo.PodIP))
+							if err != nil {
+								log.Errorf("failed parse ip %s", resRelate.PodInfo.PodIP)
+								return
+							}
+							// try clean all
+							err = link.DeleteIPRulesByIP(addr)
+							if err != nil {
+								log.Errorf("failed release ip rules %v", err)
+							}
+							err = link.DeleteRouteByIP(addr)
+							if err != nil {
+								log.Errorf("failed delete route %v", err)
+							}
+						}()
 					}
 				}
 				for _, res := range resRelate.Resources {
@@ -671,6 +697,7 @@ func (networkService *networkService) startGarbageCollectionLoop() {
 					}
 				}
 			}
+
 			gcDone := true
 			for mgrType := range inUseSet {
 				mgr, ok := networkService.mgrForResource[mgrType]
@@ -763,7 +790,6 @@ func (networkService *networkService) startPeriodCheck() {
 			}()
 		}
 	}()
-
 }
 
 // tracing

pkg/link/interface.go

@@ -3,7 +3,11 @@
 package link
 
 import (
+	"net"
+	"os"
+
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -36,3 +40,66 @@ func GetDeviceName(mac string) (string, error) {
 	}
 	return "", errors.Wrapf(ErrNotFound, "can't found dev by mac %s", mac)
 }
+
+// DeleteIPRulesByIP delete all ip rule related to the addr
+func DeleteIPRulesByIP(addr *net.IPNet) error {
+	family := netlink.FAMILY_V4
+	if addr.IP.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+	rules, err := netlink.RuleList(family)
+	if err != nil {
+		return err
+	}
+	for _, r := range rules {
+		if ipNetEqual(addr, r.Src) || ipNetEqual(addr, r.Dst) {
+			log.Infof("del ip rule %s", r.String())
+			err := netlink.RuleDel(&r)
+			if err == nil {
+				continue
+			}
+			if os.IsNotExist(err) {
+				// keep the old behave
+				r.IifName = ""
+				_ = netlink.RuleDel(&r)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// DeleteRouteByIP delete all route related to the addr
+func DeleteRouteByIP(addr *net.IPNet) error {
+	family := netlink.FAMILY_V4
+	if addr.IP.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+	routes, err := netlink.RouteList(nil, family)
+	if err != nil {
+		return err
+	}
+	for _, r := range routes {
+		if r.Dst != nil && r.Dst.IP.Equal(addr.IP) {
+			log.Infof("del route %s", r.String())
+			err := netlink.RouteDel(&r)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// ipNetEqual returns true iff both IPNet are equal
+func ipNetEqual(ipn1 *net.IPNet, ipn2 *net.IPNet) bool {
+	if ipn1 == ipn2 {
+		return true
+	}
+	if ipn1 == nil || ipn2 == nil {
+		return false
+	}
+	m1, _ := ipn1.Mask.Size()
+	m2, _ := ipn2.Mask.Size()
+	return m1 == m2 && ipn1.IP.Equal(ipn2.IP)
+}

pkg/link/interface_unsupport.go

@@ -11,3 +11,13 @@ func GetDeviceNumber(mac string) (int32, error) {
 func GetDeviceName(mac string) (string, error) {
 	return "", ErrUnsupported
 }
+
+// DeleteIPRulesByIP delete all ip rule related to the addr
+func DeleteIPRulesByIP(addr *net.IPNet) error {
+	return ErrUnsupported
+}
+
+// DeleteRouteByIP delete all route related to the addr
+func DeleteRouteByIP(addr *net.IPNet) error {
+	return ErrUnsupported
+}

plugin/driver/drivers.go

@@ -29,7 +29,7 @@ type CheckConfig struct {
 	NetNS ns.NetNS
 
 	HostVethName string
-	DeviceID     int32
+	DeviceID     int32 // phy device
 	MTU          int
 
 	ContainerIFName string
@@ -209,17 +209,9 @@ func (d *vethDriver) Setup(
 		IP:   ipv4Addr.IP,
 		Mask: net.CIDRMask(32, 32),
 	}
-	err = deleteRoutesForAddr(containerDst, 0)
+	err = EnsureHostToContainerRoute(containerDst, hostLink.Attrs().Index)
 	if err != nil {
-		return errors.Wrap(err, "vethDriver, error set route to container veth")
-	}
-	err = netlink.RouteAdd(&netlink.Route{
-		LinkIndex: hostLink.Attrs().Index,
-		Scope:     netlink.SCOPE_LINK,
-		Dst:       containerDst,
-	})
-	if err != nil {
-		return errors.Wrap(err, "vethDriver, error setup route to container veth")
+		return err
 	}
 
 	if len(extraRoutes) != 0 {
@@ -309,11 +301,9 @@ func (d *vethDriver) setupTC(dev netlink.Link, bandwidthInBytes uint64) error {
 func (d *vethDriver) ensureEniConfig(eni netlink.Link, mtu, tableID int, gw net.IP) error {
 	var err error
 	// set link up
-	if eni.Attrs().OperState != netlink.OperUp {
-		err = netlink.LinkSetUp(eni)
-		if err != nil {
-			return errors.Wrapf(err, "error set eni parent link up")
-		}
+	_, err = EnsureLinkUp(eni)
+	if err != nil {
+		return errors.Wrapf(err, "error set eni parent link up")
 	}
 	nodeIPNet := *linkIP
 	if nodeIP, err := k8snet.ChooseBindAddress(nil); err == nil {
@@ -457,6 +447,131 @@ func (d *vethDriver) Teardown(hostIfName string,
 }
 
 func (d *vethDriver) Check(cfg *CheckConfig) error {
+	err := cfg.NetNS.Do(func(netNS ns.NetNS) error {
+		link, err := netlink.LinkByName(cfg.ContainerIFName)
+		if err != nil {
+			return err
+		}
+		if link.Type() != "veth" {
+			return fmt.Errorf("link type mismatch want veth, got %s", link.Type())
+		}
+		return nil
+	})
+	if err != nil {
+		cfg.RecordPodEvent(fmt.Sprintf("veth driver failed to check nic %#v", err))
+		return nil
+	}
+	containerDst := &net.IPNet{
+		IP:   cfg.IPv4Addr.IP,
+		Mask: net.CIDRMask(32, 32),
+	}
+
+	vethHostLink, err := netlink.LinkByName(cfg.HostVethName)
+	if err != nil {
+		Log.Debugf("can't found veth %s on host", cfg.HostVethName)
+		if os.IsNotExist(err) {
+			cfg.RecordPodEvent(fmt.Sprintf("can't found veth %s on host", cfg.HostVethName))
+		}
+		return nil
+	}
+
+	err = EnsureHostToContainerRoute(containerDst, vethHostLink.Attrs().Index)
+	if err != nil {
+		return nil
+	}
+	if cfg.DeviceID == 0 {
+		Log.Debugf("device id=0 invalid")
+		return nil
+	}
+
+	// sync policy route
+	parentLink, err := netlink.LinkByIndex(int(cfg.DeviceID))
+	if err != nil {
+		cfg.RecordPodEvent(fmt.Sprintf("failed to get nic by id %d %#v", cfg.DeviceID, err))
+		Log.Debugf("failed to get nic by id %d %#v", cfg.DeviceID, err)
+		return nil
+	}
+	tableID := getRouteTableID(parentLink.Attrs().Index)
+	// ensure eni config
+	err = d.ensureEniConfig(parentLink, cfg.MTU, tableID, cfg.Gateway)
+	if err != nil {
+		Log.Debug(errors.Wrapf(err, "vethDriver, fail ensure eni config"))
+		return nil
+	}
+
+	ruleList, err := netlink.RuleList(netlink.FAMILY_V4)
+	if err != nil {
+		Log.Debug(errors.Wrapf(err, "vethDriver, fail list rule"))
+		return nil
+	}
+
+	// to container rule
+	toContainerRule := netlink.NewRule()
+	toContainerRule.Dst = containerDst
+	toContainerRule.Table = mainRouteTable
+	toContainerRule.Priority = toContainerPriority
+
+	found := false
+	for _, got := range ruleList {
+		if !ipNetEqual(containerDst, got.Dst) {
+			continue
+		}
+		if got.Table != toContainerRule.Table ||
+			got.Priority != toContainerRule.Priority {
+			Log.Debugf("del %s", got.String())
+			err := netlink.RuleDel(&got)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				Log.Debugf("failed to del rule %s, %v", got.String(), err)
+				continue
+			}
+		}
+		found = true
+	}
+	if !found {
+		Log.Debugf("add %s", toContainerRule.String())
+		err := netlink.RuleAdd(toContainerRule)
+		if err != nil {
+			return errors.Wrapf(err, "vethDriver, fail add container add rule")
+		}
+	}
+
+	// from container rule
+	fromContainerRule := netlink.NewRule()
+	fromContainerRule.IifName = cfg.HostVethName
+	fromContainerRule.Src = containerDst
+	fromContainerRule.Table = tableID
+	fromContainerRule.Priority = fromContainerPriority
+	found = false
+	for _, got := range ruleList {
+		if !ipNetEqual(fromContainerRule.Src, got.Src) {
+			continue
+		}
+		if got.IifName != fromContainerRule.IifName ||
+			got.Table != fromContainerRule.Table ||
+			got.Priority != fromContainerRule.Priority {
+			Log.Debugf("del %s", got.String())
+			err := netlink.RuleDel(&got)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				Log.Debugf("failed to del rule %s, %v", got.String(), err)
+				continue
+			}
+		}
+		found = true
+	}
+	if !found {
+		Log.Debugf("add %s", fromContainerRule.String())
+		err := netlink.RuleAdd(fromContainerRule)
+		if err != nil {
+			return errors.Wrapf(err, "vethDriver, fail add container add rule")
+		}
+	}
+
 	return nil
 }