fix: Teardown may cause resource leak

plugin/driver/ipvlan.go

@@ -153,45 +153,13 @@ func (d *IPvlanDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 }
 
 func (d *IPvlanDriver) Teardown(cfg *TeardownCfg, netNS ns.NetNS) error {
-	parents := make(map[int]struct{})
-	link, err := netlink.LinkByName(cfg.HostVETHName)
+	err := DelLinkByName(cfg.HostVETHName)
 	if err != nil {
-		if _, ok := err.(netlink.LinkNotFoundError); !ok {
-			return fmt.Errorf("error get link %s, %w", cfg.HostVETHName, err)
-		}
-	} else {
-		parents[link.Attrs().ParentIndex] = struct{}{}
-		_ = LinkSetDown(link)
-		err = LinkDel(link)
-		if err != nil {
-			return fmt.Errorf("error del link, %w", err)
-		}
-	}
-	err = netNS.Do(func(netNS ns.NetNS) error {
-		for _, ifName := range []string{cfg.HostVETHName, cfg.ContainerIfName} {
-			link, err := netlink.LinkByName(ifName)
-			if err != nil {
-				if _, ok := err.(netlink.LinkNotFoundError); !ok {
-					return fmt.Errorf("error get link %s, %w", ifName, err)
-				}
-				continue
-			}
-
-			parents[link.Attrs().ParentIndex] = struct{}{}
-			_ = LinkSetDown(link)
-			err = LinkDel(link)
-			if err != nil {
-				return fmt.Errorf("error del link, %w", err)
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error teardown container, %w", err)
+		return err
 	}
 
-	delete(parents, 0)
-	return d.teardownInitNamespace(parents, cfg.ContainerIPNet)
+	// del route to container
+	return d.teardownInitNamespace(cfg.ContainerIPNet)
 }
 
 func (d *IPvlanDriver) Check(cfg *CheckConfig) error {
@@ -445,20 +413,17 @@ func (d *IPvlanDriver) setupInitNamespace(parentLink netlink.Link, cfg *SetupCon
 	return nil
 }
 
-func (d *IPvlanDriver) teardownInitNamespace(parents map[int]struct{}, containerIP *terwayTypes.IPNetSet) error {
+func (d *IPvlanDriver) teardownInitNamespace(containerIP *terwayTypes.IPNetSet) error {
 	if containerIP == nil {
 		return nil
 	}
 
-	exec := func(link netlink.Link, ipNet *net.IPNet) error {
-		rt := &netlink.Route{
-			LinkIndex: link.Attrs().Index,
-			Dst:       NewIPNetWithMaxMask(ipNet),
-		}
-
-		routes, err := netlink.RouteListFiltered(NetlinkFamily(ipNet.IP), rt, netlink.RT_FILTER_DST|netlink.RT_FILTER_OIF)
+	exec := func(ipNet *net.IPNet) error {
+		routes, err := FoundRoutes(&netlink.Route{
+			Dst: ipNet,
+		})
 		if err != nil {
-			return fmt.Errorf("error get route by filter %s, %w", rt.String(), err)
+			return err
 		}
 		for _, route := range routes {
 			err = RouteDel(&route)
@@ -468,39 +433,27 @@ func (d *IPvlanDriver) teardownInitNamespace(parents map[int]struct{}, container
 		}
 		return nil
 	}
-	// get slave link
-	for index := range parents {
-		initLink, err := d.initSlaveLink(index)
+
+	if containerIP.IPv4 != nil {
+		err := exec(NewIPNetWithMaxMask(containerIP.IPv4))
 		if err != nil {
-			if _, ok := err.(netlink.LinkNotFoundError); !ok {
-				return fmt.Errorf("error get link by index %d, %w", index, err)
-			}
-			continue
-		}
-		if containerIP.IPv4 != nil {
-			err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv4))
-			if err != nil {
-				return err
-			}
+			return err
 		}
-		if containerIP.IPv6 != nil {
-			err = exec(initLink, NewIPNetWithMaxMask(containerIP.IPv6))
-			if err != nil {
-				return err
-			}
+	}
+	if containerIP.IPv6 != nil {
+		err := exec(NewIPNetWithMaxMask(containerIP.IPv6))
+		if err != nil {
+			return err
 		}
 	}
+
 	return nil
 }
 
 func (d *IPvlanDriver) initSlaveName(parentIndex int) string {
 	return fmt.Sprintf("ipvl_%d", parentIndex)
 }
 
-func (d *IPvlanDriver) initSlaveLink(parentIndex int) (netlink.Link, error) {
-	return netlink.LinkByName(d.initSlaveName(parentIndex))
-}
-
 type redirectRule struct {
 	index    int
 	proto    uint16

plugin/driver/netlink.go

@@ -179,7 +179,13 @@ func RuleDel(rule *netlink.Rule) error {
 	Log.Infof(cmd)
 	err := netlink.RuleDel(rule)
 	if err != nil {
-		return fmt.Errorf("error %s, %w", cmd, err)
+		rule.IifName = ""
+		rule.OifName = ""
+
+		err = netlink.RuleDel(rule)
+		if err != nil {
+			return fmt.Errorf("error %s, %w", cmd, err)
+		}
 	}
 	return nil
 }

plugin/driver/raw_nic.go

@@ -119,37 +119,6 @@ func (r *RawNicDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 }
 
 func (r *RawNicDriver) Teardown(cfg *TeardownCfg, netNS ns.NetNS) error {
-	// 1. move link out
-	hostCurrentNs, err := ns.GetCurrentNS()
-	defer func() {
-		err = hostCurrentNs.Close()
-	}()
-	if err != nil {
-		return fmt.Errorf("error get host net ns, %w", err)
-	}
-	err = netNS.Do(func(netNS ns.NetNS) error {
-		var nicLink netlink.Link
-		nicLink, err = netlink.LinkByName(cfg.ContainerIfName)
-		if err == nil {
-			nicName, err1 := r.randomNicName()
-			if err1 != nil {
-				return fmt.Errorf("error generate random nic name, %w", err)
-			}
-			err = netlink.LinkSetDown(nicLink)
-			if err != nil {
-				return fmt.Errorf("error set link %s down, %w", nicLink.Attrs().Name, err)
-			}
-			err = netlink.LinkSetName(nicLink, nicName)
-			if err != nil {
-				return fmt.Errorf("error set link %s name %s, %w", nicLink.Attrs().Name, nicName, err)
-			}
-			return netlink.LinkSetNsFd(nicLink, int(hostCurrentNs.Fd()))
-		}
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("error move eni to host net ns, %w", err)
-	}
 	return nil
 }
 

plugin/driver/utils.go

@@ -9,7 +9,10 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containernetworking/plugins/pkg/ip"
+	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/pkg/errors"
+	k8sErr "k8s.io/apimachinery/pkg/util/errors"
 
 	terwayIP "github.com/AliyunContainerService/terway/pkg/ip"
 	terwaySysctl "github.com/AliyunContainerService/terway/pkg/sysctl"
@@ -171,6 +174,17 @@ func EnsureLinkName(link netlink.Link, name string) (bool, error) {
 	return true, LinkSetName(link, name)
 }
 
+// DelLinkByName del by name and ignore if link not present
+func DelLinkByName(ifName string) error {
+	contLink, err := netlink.LinkByName(ifName)
+	if err != nil {
+		if _, ok := err.(netlink.LinkNotFoundError); ok { //nolint
+			return nil
+		}
+	}
+	return LinkDel(contLink)
+}
+
 // EnsureAddrWithPrefix take the ipNet set and ensure only one IP for each family is present on link
 // it will remove other unmatched IPs
 func EnsureAddrWithPrefix(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, prefixRoute bool) (bool, error) {
@@ -562,30 +576,6 @@ func EnsureIPRule(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, tableID int
 	return changed, nil
 }
 
-func DelIPRulesByIP(ipNet *net.IPNet) error {
-	var ruleList []netlink.Rule
-	var err error
-	if terwayIP.IPv6(ipNet.IP) {
-		ruleList, err = netlink.RuleList(netlink.FAMILY_V6)
-	} else {
-		ruleList, err = netlink.RuleList(netlink.FAMILY_V4)
-	}
-	if err != nil {
-		return fmt.Errorf("error get ip rule, %w", err)
-	}
-
-	for _, rule := range ruleList {
-		if terwayIP.NetEqual(ipNet, rule.Src) || terwayIP.NetEqual(ipNet, rule.Dst) {
-			innerErr := RuleDel(&rule)
-			if innerErr != nil {
-				rule.IifName = ""
-				err = errors.Wrap(RuleDel(&rule), "error de")
-			}
-		}
-	}
-	return err
-}
-
 func EnableIPv6() error {
 	err := terwaySysctl.EnsureConf("/proc/sys/net/ipv6/conf/all/disable_ipv6", "0")
 	if err != nil {
@@ -782,3 +772,118 @@ func EnsureClsActQdsic(link netlink.Link) error {
 	}
 	return nil
 }
+
+// GenericTearDown target to clean all related resource as much as possible
+func GenericTearDown(netNS ns.NetNS) error {
+	var errList []error
+	hostNetNS, err := ns.GetCurrentNS()
+	if err != nil {
+		return fmt.Errorf("err get host net ns, %w", err)
+	}
+	err = netNS.Do(func(netNS ns.NetNS) error {
+		linkList, err := netlink.LinkList()
+		if err != nil {
+			return fmt.Errorf("error get link list from netlink, %w", err)
+		}
+		for _, l := range linkList {
+			_ = LinkSetDown(l)
+			switch l.(type) {
+			case *netlink.IPVlan, *netlink.Vlan, *netlink.Veth, *netlink.Ifb, *netlink.Dummy:
+				errList = append(errList, LinkDel(l))
+			case *netlink.Device:
+				name, err := ip.RandomVethName()
+				if err != nil {
+					errList = append(errList, err)
+					continue
+				}
+				errList = append(errList, LinkSetName(l, name))
+				errList = append(errList, LinkSetNsFd(l, hostNetNS))
+			default:
+				continue
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		if _, ok := err.(ns.NSPathNotExistErr); !ok {
+			errList = append(errList, err)
+		}
+	}
+	errList = append(errList, CleanIPRules())
+	return k8sErr.NewAggregate(errList)
+}
+
+// CleanIPRules del ip rule for detached devs
+func CleanIPRules() (err error) {
+	var rules []netlink.Rule
+	rules, err = netlink.RuleList(netlink.FAMILY_ALL)
+	if err != nil {
+		return err
+	}
+
+	var ipNets []*net.IPNet
+	defer func() {
+		for _, r := range rules {
+			if r.Priority != 512 && r.Priority != 2048 {
+				continue
+			}
+			if r.IifName != "" || r.OifName != "" {
+				continue
+			}
+			found := false
+
+			for _, ipNet := range ipNets {
+				if r.Dst != nil {
+					if r.Dst.String() == ipNet.String() {
+						found = true
+						break
+					}
+				}
+				if r.Src != nil {
+					if r.Src.String() == ipNet.String() {
+						found = true
+						break
+					}
+				}
+			}
+			if !found {
+				continue
+			}
+			_ = RuleDel(&r)
+		}
+	}()
+	for _, r := range rules {
+		if r.Priority != 512 && r.Priority != 2048 {
+			continue
+		}
+		name := r.IifName
+		if name == "" {
+			name = r.OifName
+		}
+		if name == "" {
+			continue
+		}
+		_, err = netlink.LinkByName(name)
+		if err != nil {
+			if _, ok := err.(netlink.LinkNotFoundError); !ok {
+				return err
+			}
+			err = RuleDel(&r)
+			if err != nil {
+				return err
+			}
+			var ipNet *net.IPNet
+			if r.Dst != nil {
+				ipNet = r.Dst
+			}
+			if r.Src != nil {
+				ipNet = r.Src
+			}
+			if ipNet != nil {
+				ipNets = append(ipNets, ipNet)
+			}
+		}
+	}
+
+	return nil
+}