Merge pull request #207 from xh4n3/features/fix-ipvlan-pod-to-node-da…

…ta-path let pod talk to node directly in ipvlan mode
AliyunContainerService · May 20, 2021 · 7ffa6e5 · 7ffa6e5
2 parents 1159252 + 7c60855
commit 7ffa6e5
Show file tree

Hide file tree

Showing 2 changed files with 108 additions and 28 deletions.
diff --git a/plugin/driver/ipvlan.go b/plugin/driver/ipvlan.go
@@ -79,6 +79,11 @@ func (d *IPvlanDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 		return err
 	}
 
+	hostIPSet, err := GetHostIP(d.ipv4, d.ipv6)
+	if err != nil {
+		return err
+	}
+
 	// 2. setup addr and default route
 	err = netNS.Do(func(netNS ns.NetNS) error {
 		if d.ipv6 {
@@ -102,9 +107,24 @@ func (d *IPvlanDriver) Setup(cfg *SetupConfig, netNS ns.NetNS) error {
 				return err
 			}
 			_, err = EnsureDefaultRoute(link, cfg.GatewayIP)
+			if err != nil {
+				return err
+			}
+
+			// setup route to host ipvlan interface
+			_, err = EnsureRoute(link, hostIPSet)
+			if err != nil {
+				return fmt.Errorf("add route to host %s %s error, %w", hostIPSet.IPv4, hostIPSet.IPv6, err)
+			}
+
+			// set host ipvlan interface mac in ARP table
+			_, err = EnsureNeighbor(link, hostIPSet)
+			if err != nil {
+				return err
+			}
 			return err
 		}
-		return nil
+		return err
 	})
 
 	if err != nil {

diff --git a/plugin/driver/utils.go b/plugin/driver/utils.go
@@ -208,45 +208,40 @@ func EnsureAddr(link netlink.Link, ipNetSet *terwayTypes.IPNetSet, equal func(a
 
 func EnsureDefaultRoute(link netlink.Link, gw *terwayTypes.IPSet) (bool, error) {
 	var changed bool
-
-	exec := func(dst *net.IPNet, gw net.IP) (bool, error) {
-		err := ip.ValidateExpectedRoute([]*types.Route{
-			{
-				Dst: *dst,
-				GW:  gw,
-			},
-		})
-		if err == nil {
-			return false, nil
-		}
-		if !strings.Contains(err.Error(), "not found") {
-			return false, err
+	if gw.IPv4 != nil {
+		ok, err := ensureRoute(link, defaultRoute, netlink.SCOPE_UNIVERSE, int(netlink.FLAG_ONLINK), gw.IPv4)
+		if err != nil {
+			return changed, err
 		}
-		r := &netlink.Route{
-			LinkIndex: link.Attrs().Index,
-			Scope:     netlink.SCOPE_UNIVERSE,
-			Flags:     int(netlink.FLAG_ONLINK),
-			Dst:       dst,
-			Gw:        gw,
+		if ok {
+			changed = true
 		}
-
-		err = RouteReplace(r)
+	}
+	if gw.IPv6 != nil {
+		ok, err := ensureRoute(link, defaultRouteIPv6, netlink.SCOPE_UNIVERSE, int(netlink.FLAG_ONLINK), gw.IPv6)
 		if err != nil {
-			return false, err
+			return changed, err
+		}
+		if ok {
+			changed = true
 		}
-		return true, nil
 	}
-	if gw.IPv4 != nil {
-		ok, err := exec(defaultRoute, gw.IPv4)
+	return changed, nil
+}
+
+func EnsureRoute(link netlink.Link, hostIPSet *terwayTypes.IPNetSet) (bool, error) {
+	var changed bool
+	if hostIPSet.IPv4 != nil {
+		ok, err := ensureRoute(link, hostIPSet.IPv4, netlink.SCOPE_LINK, 0, nil)
 		if err != nil {
 			return changed, err
 		}
 		if ok {
 			changed = true
 		}
 	}
-	if gw.IPv6 != nil {
-		ok, err := exec(defaultRouteIPv6, gw.IPv6)
+	if hostIPSet.IPv6 != nil {
+		ok, err := ensureRoute(link, hostIPSet.IPv6, netlink.SCOPE_LINK, 0, nil)
 		if err != nil {
 			return changed, err
 		}
@@ -257,6 +252,36 @@ func EnsureDefaultRoute(link netlink.Link, gw *terwayTypes.IPSet) (bool, error)
 	return changed, nil
 }
 
+func ensureRoute(link netlink.Link, dst *net.IPNet, scope netlink.Scope, flags int, gw net.IP) (bool, error) {
+	var err error
+	if gw != nil {
+		err = ip.ValidateExpectedRoute([]*types.Route{
+			{
+				Dst: *dst,
+				GW:  gw,
+			},
+		})
+		if err == nil {
+			return false, nil
+		}
+		if !strings.Contains(err.Error(), "not found") {
+			return false, err
+		}
+	}
+	r := &netlink.Route{
+		LinkIndex: link.Attrs().Index,
+		Scope:     scope,
+		Flags:     flags,
+		Dst:       dst,
+		Gw:        gw,
+	}
+	err = RouteReplace(r)
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 // EnsureHostToContainerRoute create host to container route
 func EnsureHostToContainerRoute(link netlink.Link, ipNetSet *terwayTypes.IPNetSet) (bool, error) {
 	var changed bool
@@ -571,3 +596,38 @@ func GetHostIP(ipv4, ipv6 bool) (*terwayTypes.IPNetSet, error) {
 		IPv6: nodeIPv6,
 	}, nil
 }
+
+func EnsureNeighbor(link netlink.Link, hostIPSet *terwayTypes.IPNetSet) (bool, error) {
+	var changed bool
+	var err error
+
+	if hostIPSet.IPv4 != nil {
+		err = netlink.NeighSet(&netlink.Neigh{
+			IP:           hostIPSet.IPv4.IP,
+			Family:       netlink.FAMILY_V4,
+			LinkIndex:    link.Attrs().Index,
+			HardwareAddr: link.Attrs().HardwareAddr,
+			Type:         netlink.NDA_DST,
+			State:        netlink.NUD_PERMANENT,
+		})
+		if err != nil {
+			return false, fmt.Errorf("add host ipvlan interface %s mac %s to ARP table error, %w", hostIPSet.IPv4, link.Attrs().HardwareAddr, err)
+		}
+		changed = true
+	}
+	if hostIPSet.IPv6 != nil {
+		err = netlink.NeighSet(&netlink.Neigh{
+			IP:           hostIPSet.IPv6.IP,
+			Family:       netlink.FAMILY_V6,
+			LinkIndex:    link.Attrs().Index,
+			HardwareAddr: link.Attrs().HardwareAddr,
+			Type:         netlink.NDA_DST,
+			State:        netlink.NUD_PERMANENT,
+		})
+		if err != nil {
+			return false, fmt.Errorf("add host ipvlan interface %s mac %s to ARP table error, %w", hostIPSet.IPv4, link.Attrs().HardwareAddr, err)
+		}
+		changed = true
+	}
+	return changed, nil
+}