From 780e0bdee3ff7713e43cb01e00f3da1e7f45370f Mon Sep 17 00:00:00 2001 From: rakelkar Date: Tue, 17 Oct 2017 23:30:30 -0700 Subject: [PATCH] Add logic to support vxlan on Windows (#2) Flannel: Add support for Windows Overlay (vxlan) mode Windows Overlay mode requires the network to be set to the cluster CIDR. So for overlay setup host-local as IPAM with the pod CIDR as the ip range --- plugins/meta/flannel/README.md | 56 +++++++++++++++++++++++ plugins/meta/flannel/flannel.go | 79 ++++++++++++++++++++++----------- 2 files changed, 109 insertions(+), 26 deletions(-) diff --git a/plugins/meta/flannel/README.md b/plugins/meta/flannel/README.md index 0efb69059..b624cecd4 100644 --- a/plugins/meta/flannel/README.md +++ b/plugins/meta/flannel/README.md @@ -86,3 +86,59 @@ flannel plugin will set the following fields in the delegated plugin configurati * `mtu`: `$FLANNEL_MTU` Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present. + +## Windows Support (Experimental) +This plugin supports delegating to the windows CNI plugin (wincni.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). +Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-networking) in L2Bridge mode for host-gw and +in Overlay mode for vxlan. + +The following fields must be set in the delegated plugin configuration: +* `name` (string, required): the name of the network (must match the name in Flannel config / name of the HNS network) +* `type` (string, optional): for custom scenarios can be set to delegate to a plugin other than WINCNI +* `backendType` (string, optional): set to the Flannel backend mode being used, "host-gw" (default) or "vxlan" +* `endpointMacPrefix` (string, optional): required for vxlan mode, set to the MAC prefix configured for Flannel + +For host-gw, the Flannel CNI plugin will set: +* `ipam` (string, required): subnet to `$FLANNEL_SUBNET` and GW to the .2 address in the `$FLANNEL_SUBNET` (this is required by HNS). IPAM type is left empty to allow Windows HNS to do IPAM + +For vxlan, the Flannel CNI plugin will set: +* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_NETWORK` but limited to a range per `$FLANNEL_SUBNET` and gateway as the .1 address in `$FLANNEL_NETWORK` + +If IPMASQ is true, the Flannel CNI plugin will setup an OutBoundNAT policy and add FLANNEL_SUBNET to any existing exclusions. + +All other delegate config e.g. other HNS endpoint policis in AdditionalArgs will be passed to WINCNI as-is. + +Example VXLAN Flannel CNI config +``` +{ + "name": "vxlan0", + "type": "flannel", + "delegate": { + "backendType": "vxlan", + "endpointMacPrefix": "0E-2A" + } +} +``` + +For this example, Flannel CNI would generate the following config to delegate to WINCNI when FLANNEL_NETWORK=10.244.0.0/16, FLANNEL_SUBNET=10.244.1.0/24 and IPMASQ=true +``` +{ + "name": "vxlan0", + "type": "wincni.exe", + "endpointMacPrefix": "0E-2A", + "ipam": { + "gateway": "10.244.0.1", + "rangeEnd": "10.244.1.254", + "rangeStart": "10.244.1.2", + "subnet": "10.244.0.0/16", + "type": "host-local" + }, + "AdditionalArgs": [{ + "Name": "EndpointPolicy", + "Value": { + "ExceptionList": ["10.244.0.0/16"], + "Type": "OutBoundNAT" + } + }] +} +``` diff --git a/plugins/meta/flannel/flannel.go b/plugins/meta/flannel/flannel.go index 22e053c00..73cf7e6c4 100644 --- a/plugins/meta/flannel/flannel.go +++ b/plugins/meta/flannel/flannel.go @@ -20,6 +20,7 @@ package main import ( "bufio" + "encoding/binary" "encoding/json" "fmt" "io/ioutil" @@ -30,11 +31,11 @@ import ( "strconv" "strings" + "errors" "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/version" - "math/big" ) const ( @@ -262,19 +263,63 @@ func cmdAddWindows(containerID string, n *NetConf, fenv *subnetEnv) error { n.Delegate["cniVersion"] = n.CNIVersion } - // for now get Windows HNS to do IPAM - n.Delegate["ipam"] = map[string]interface{}{ - "subnet": fenv.sn.String(), - "routes": []interface{}{ - map[string]interface{}{ - "GW": calcGatewayIPforWindows(fenv.sn), + backendType := "host-gw" + if hasKey(n.Delegate, "backendType") { + backendType = n.Delegate["backendType"].(string) + } + + switch backendType { + case "host-gw": + // let HNS do IPAM for hostgw (L2 bridge) mode + gw := fenv.sn.IP.Mask(fenv.sn.Mask) + gw[len(gw)-1] += 2 + + n.Delegate["ipam"] = map[string]interface{}{ + "subnet": fenv.sn.String(), + "routes": []interface{}{ + map[string]interface{}{ + "GW": gw.String(), + }, }, - }, + } + case "vxlan": + // for vxlan (Overlay) mode the gw is on the cluster CIDR + gw := fenv.nw.IP.Mask(fenv.nw.Mask) + gw[len(gw)-1] += 1 + + // but restrict allocation to the node's pod CIDR + rs := fenv.sn.IP.Mask(fenv.sn.Mask).To4() + rs[len(rs)-1] += 2 + re, err := lastAddr(fenv.sn) + if err != nil { + return err + } + re[len(re)-1] -= 1 + n.Delegate["ipam"] = map[string]interface{}{ + "type": "host-local", + "subnet": fenv.nw.String(), + "rangeStart": rs.String(), + "rangeEnd": re.String(), + "gateway": gw.String(), + } + + default: + return fmt.Errorf("backendType [%v] is not supported on windows", backendType) } return delegateAdd(containerID, n.DataDir, n.Delegate) } +// https://stackoverflow.com/questions/36166791/how-to-get-broadcast-address-of-ipv4-net-ipnet +func lastAddr(n *net.IPNet) (net.IP, error) { // works when the n is a prefix, otherwise... + if n.IP.To4() == nil { + return net.IP{}, errors.New("does not support IPv6 addresses.") + } + ip := make(net.IP, len(n.IP.To4())) + binary.BigEndian.PutUint32(ip, binary.BigEndian.Uint32(n.IP.To4())|^binary.BigEndian.Uint32(net.IP(n.Mask).To4())) + return ip, nil +} + func updateOutboundNat(delegate map[string]interface{}, fenv *subnetEnv) { if !*fenv.ipmasq { return @@ -332,24 +377,6 @@ func updateOutboundNat(delegate map[string]interface{}, fenv *subnetEnv) { delegate["AdditionalArgs"] = append(addlArgs, natEntry) } -func calcGatewayIPforWindows(ipn *net.IPNet) net.IP { - // HNS currently requires x.x.x.2 - nid := ipn.IP.Mask(ipn.Mask) - nid[len(nid)-1] += 2 - return nid -} - -func ipToInt(ip net.IP) *big.Int { - if v := ip.To4(); v != nil { - return big.NewInt(0).SetBytes(v) - } - return big.NewInt(0).SetBytes(ip.To16()) -} - -func intToIP(i *big.Int) net.IP { - return net.IP(i.Bytes()) -} - func cmdDel(args *skel.CmdArgs) error { nc, err := loadFlannelNetConf(args.StdinData) if err != nil {