Swarm to support node-local networks

- regardless of the network driver datascope

Signed-off-by: Alessandro Boch <[email protected]>

api/specs.proto

@@ -341,6 +341,11 @@ message NetworkSpec {
 	// swarm internally created only and it was identified by the name
 	// "ingress" and the label "com.docker.swarm.internal": "true".
 	bool ingress = 7;
+
+	// ConfigFrom indicates that the network specific configuration
+	// for this network will be provided via another network, locally
+	// on the node where this network is being plumbed.
+	string config_from = 8;
 }
 
 // ClusterSpec specifies global cluster settings.

manager/allocator/networkallocator/drivers_network_linux.go

@@ -0,0 +1,19 @@
+package networkallocator
+
+import (
+	"github.com/docker/libnetwork/drivers/bridge/brmanager"
+	"github.com/docker/libnetwork/drivers/ipvlan/ivmanager"
+	"github.com/docker/libnetwork/drivers/macvlan/mvmanager"
+	"github.com/docker/libnetwork/drivers/overlay/ovmanager"
+	"github.com/docker/libnetwork/drivers/remote"
+)
+
+func getInitializers() []initializer {
+	return []initializer{
+		{remote.Init, "remote"},
+		{ovmanager.Init, "overlay"},
+		{mvmanager.Init, "macvlan"},
+		{brmanager.Init, "bridge"},
+		{ivmanager.Init, "ipvlan"},
+	}
+}

manager/allocator/networkallocator/drivers_network.go → manager/allocator/networkallocator/drivers_network_windows.go

@@ -1,5 +1,3 @@
-// +build linux windows
-
 package networkallocator
 
 import (

manager/allocator/networkallocator/networkallocator.go

@@ -3,6 +3,7 @@ package networkallocator
 import (
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/libnetwork/datastore"
@@ -62,6 +63,18 @@ type network struct {
 	// endpoints is a map of endpoint IP to the poolID from which it
 	// was allocated.
 	endpoints map[string]string
+
+	// isNodeLocal indicates whether the scope of the network's resources
+	// is local to the node. If true, it means the resources can only be
+	// allocated locally by the node where the network will be deployed.
+	// In this the swarm manager will skip the allocations.
+	isNodeLocal bool
+}
+
+type networkDriver struct {
+	driver     driverapi.Driver
+	name       string
+	capability *driverapi.Capability
 }
 
 type initializer struct {
@@ -110,22 +123,38 @@ func (na *NetworkAllocator) Allocate(n *api.Network) error {
 		return fmt.Errorf("network %s already allocated", n.ID)
 	}
 
-	pools, err := na.allocatePools(n)
+	d, err := na.resolveDriver(n)
 	if err != nil {
-		return errors.Wrapf(err, "failed allocating pools and gateway IP for network %s", n.ID)
+		return err
 	}
 
-	if err := na.allocateDriverState(n); err != nil {
-		na.freePools(n, pools)
-		return errors.Wrapf(err, "failed while allocating driver state for network %s", n.ID)
+	nw := &network{
+		nw:          n,
+		endpoints:   make(map[string]string),
+		isNodeLocal: d.capability.DataScope == datastore.LocalScope,
 	}
 
-	na.networks[n.ID] = &network{
-		nw:        n,
-		pools:     pools,
-		endpoints: make(map[string]string),
+	// No swarm-level allocation can be provided by the network driver for
+	// node-local networks. Only thing needed is populating the driver's name
+	// in the driver's state.
+	if nw.isNodeLocal {
+		n.DriverState = &api.Driver{
+			Name: d.name,
+		}
+	} else {
+		nw.pools, err = na.allocatePools(n)
+		if err != nil {
+			return errors.Wrapf(err, "failed allocating pools and gateway IP for network %s", n.ID)
+		}
+
+		if err := na.allocateDriverState(n); err != nil {
+			na.freePools(n, nw.pools)
+			return errors.Wrapf(err, "failed while allocating driver state for network %s", n.ID)
+		}
 	}
 
+	na.networks[n.ID] = nw
+
 	return nil
 }
 
@@ -141,11 +170,18 @@ func (na *NetworkAllocator) Deallocate(n *api.Network) error {
 		return fmt.Errorf("could not get networker state for network %s", n.ID)
 	}
 
+	// No swarm-level resource deallocation needed for node-local networks
+	if localNet.isNodeLocal {
+		delete(na.networks, n.ID)
+		return nil
+	}
+
 	if err := na.freeDriverState(n); err != nil {
 		return errors.Wrapf(err, "failed to free driver state for network %s", n.ID)
 	}
 
 	delete(na.networks, n.ID)
+
 	return na.freePools(n, localNet.pools)
 }
 
@@ -282,24 +318,32 @@ func (na *NetworkAllocator) IsTaskAllocated(t *api.Task) bool {
 	}
 
 	// To determine whether the task has its resources allocated,
-	// we just need to look at one network(in case of
+	// we just need to look at one global scope network (in case of
 	// multi-network attachment).  This is because we make sure we
 	// allocate for every network or we allocate for none.
 
-	// If the network is not allocated, the task cannot be allocated.
-	localNet, ok := na.networks[t.Networks[0].Network.ID]
-	if !ok {
-		return false
-	}
+	// Find the first global scope network
+	for _, nAttach := range t.Networks {
+		// If the network is not allocated, the task cannot be allocated.
+		localNet, ok := na.networks[nAttach.Network.ID]
+		if !ok {
+			return false
+		}
 
-	// Addresses empty. Task is not allocated.
-	if len(t.Networks[0].Addresses) == 0 {
-		return false
-	}
+		// Nothing else to check for local scope network
+		if localNet.isNodeLocal {
+			continue
+		}
 
-	// The allocated IP address not found in local endpoint state. Not allocated.
-	if _, ok := localNet.endpoints[t.Networks[0].Addresses[0]]; !ok {
-		return false
+		// Addresses empty. Task is not allocated.
+		if len(nAttach.Addresses) == 0 {
+			return false
+		}
+
+		// The allocated IP address not found in local endpoint state. Not allocated.
+		if _, ok := localNet.endpoints[nAttach.Addresses[0]]; !ok {
+			return false
+		}
 	}
 
 	return true
@@ -445,6 +489,9 @@ func (na *NetworkAllocator) DeallocateNode(node *api.Node) error {
 // networks that a task is attached to.
 func (na *NetworkAllocator) AllocateTask(t *api.Task) error {
 	for i, nAttach := range t.Networks {
+		if localNet := na.getNetwork(nAttach.Network.ID); localNet != nil && localNet.isNodeLocal {
+			continue
+		}
 		if err := na.allocateNetworkIPs(nAttach); err != nil {
 			if err := na.releaseEndpoints(t.Networks[:i]); err != nil {
 				log.G(context.TODO()).WithError(err).Errorf("Failed to release IP addresses while rolling back allocation for task %s network %s", t.ID, nAttach.Network.ID)
@@ -467,16 +514,20 @@ func (na *NetworkAllocator) DeallocateTask(t *api.Task) error {
 
 func (na *NetworkAllocator) releaseEndpoints(networks []*api.NetworkAttachment) error {
 	for _, nAttach := range networks {
-		ipam, _, _, err := na.resolveIPAM(nAttach.Network)
-		if err != nil {
-			return errors.Wrapf(err, "failed to resolve IPAM while allocating")
-		}
-
 		localNet := na.getNetwork(nAttach.Network.ID)
 		if localNet == nil {
 			return fmt.Errorf("could not find network allocator state for network %s", nAttach.Network.ID)
 		}
 
+		if localNet.isNodeLocal {
+			continue
+		}
+
+		ipam, _, _, err := na.resolveIPAM(nAttach.Network)
+		if err != nil {
+			return errors.Wrapf(err, "failed to resolve IPAM while releasing")
+		}
+
 		// Do not fail and bail out if we fail to release IP
 		// address here. Keep going and try releasing as many
 		// addresses as possible.
@@ -512,6 +563,10 @@ func (na *NetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error {
 		return errors.New("networkallocator: could not find local network state")
 	}
 
+	if localNet.isNodeLocal {
+		return nil
+	}
+
 	// If this IP is already allocated in memory we don't need to
 	// do anything.
 	if _, ok := localNet.endpoints[vip.Addr]; ok {
@@ -556,7 +611,9 @@ func (na *NetworkAllocator) deallocateVIP(vip *api.Endpoint_VirtualIP) error {
 	if localNet == nil {
 		return errors.New("networkallocator: could not find local network state")
 	}
-
+	if localNet.isNodeLocal {
+		return nil
+	}
 	ipam, _, _, err := na.resolveIPAM(localNet.nw)
 	if err != nil {
 		return errors.Wrap(err, "failed to resolve IPAM while allocating")
@@ -637,16 +694,16 @@ func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) e
 }
 
 func (na *NetworkAllocator) freeDriverState(n *api.Network) error {
-	d, _, err := na.resolveDriver(n)
+	d, err := na.resolveDriver(n)
 	if err != nil {
 		return err
 	}
 
-	return d.NetworkFree(n.ID)
+	return d.driver.NetworkFree(n.ID)
 }
 
 func (na *NetworkAllocator) allocateDriverState(n *api.Network) error {
-	d, dName, err := na.resolveDriver(n)
+	d, err := na.resolveDriver(n)
 	if err != nil {
 		return err
 	}
@@ -691,22 +748,22 @@ func (na *NetworkAllocator) allocateDriverState(n *api.Network) error {
 		ipv4Data = append(ipv4Data, data)
 	}
 
-	ds, err := d.NetworkAllocate(n.ID, options, ipv4Data, nil)
+	ds, err := d.driver.NetworkAllocate(n.ID, options, ipv4Data, nil)
 	if err != nil {
 		return err
 	}
 
 	// Update network object with the obtained driver state.
 	n.DriverState = &api.Driver{
-		Name:    dName,
+		Name:    d.name,
 		Options: ds,
 	}
 
 	return nil
 }
 
 // Resolve network driver
-func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, string, error) {
+func (na *NetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, error) {
 	dName := DefaultDriver
 	if n.Spec.DriverConfig != nil && n.Spec.DriverConfig.Name != "" {
 		dName = n.Spec.DriverConfig.Name
@@ -717,21 +774,16 @@ func (na *NetworkAllocator) resolveDriver(n *api.Network) (driverapi.Driver, str
 		var err error
 		err = na.loadDriver(dName)
 		if err != nil {
-			return nil, "", err
+			return nil, err
 		}
 
 		d, drvcap = na.drvRegistry.Driver(dName)
 		if d == nil {
-			return nil, "", fmt.Errorf("could not resolve network driver %s", dName)
+			return nil, fmt.Errorf("could not resolve network driver %s", dName)
 		}
-
-	}
-
-	if drvcap.DataScope != datastore.GlobalScope {
-		return nil, "", fmt.Errorf("swarm can allocate network resources only for global scoped networks. network driver (%s) is scoped %s", dName, drvcap.DataScope)
 	}
 
-	return d, dName, nil
+	return &networkDriver{driver: d, capability: drvcap, name: dName}, nil
 }
 
 func (na *NetworkAllocator) loadDriver(name string) error {
@@ -934,3 +986,14 @@ func IsIngressNetworkNeeded(s *api.Service) bool {
 
 	return false
 }
+
+// IsBuiltInDriver returns whether the passed driver is an internal network driver
+func IsBuiltInDriver(name string) bool {
+	n := strings.ToLower(name)
+	for _, d := range getInitializers() {
+		if n == d.ntype {
+			return true
+		}
+	}
+	return false
+}

manager/controlapi/common.go

@@ -104,8 +104,16 @@ func validateDriver(driver *api.Driver, pg plugingetter.PluginGetter, pluginType
 		return grpc.Errorf(codes.InvalidArgument, "driver name: if driver is specified name is required")
 	}
 
-	if strings.ToLower(driver.Name) == networkallocator.DefaultDriver || strings.ToLower(driver.Name) == ipamapi.DefaultIPAM {
-		return nil
+	// First check against the known drivers
+	switch pluginType {
+	case ipamapi.PluginEndpointType:
+		if strings.ToLower(driver.Name) == ipamapi.DefaultIPAM {
+			return nil
+		}
+	default:
+		if networkallocator.IsBuiltInDriver(driver.Name) {
+			return nil
+		}
 	}
 
 	if pg == nil {