diff --git a/pkg/network/master/master.go b/pkg/network/master/master.go index e30b577fd..97d5d5514 100644 --- a/pkg/network/master/master.go +++ b/pkg/network/master/master.go @@ -20,6 +20,7 @@ import ( networkinformers "github.com/openshift/client-go/network/informers/externalversions/network/v1" "github.com/openshift/library-go/pkg/network/networkutils" "github.com/openshift/sdn/pkg/network/common" + masterutil "github.com/openshift/sdn/pkg/network/master/util" ) const ( @@ -38,9 +39,7 @@ type OsdnMaster struct { netNamespaceInformer networkinformers.NetNamespaceInformer // Used for allocating subnets in order - subnetAllocatorList []*SubnetAllocator - // Used for clusterNetwork --> subnetAllocator lookup - subnetAllocatorMap map[common.ParsedClusterNetworkEntry]*SubnetAllocator + subnetAllocator *masterutil.SubnetAllocator // Holds Node IP used in creating host subnet for a node hostSubnetNodeIPs map[ktypes.UID]string @@ -66,8 +65,7 @@ func Start(networkClient networkclient.Interface, kClient kclientset.Interface, hostSubnetInformer: networkInformers.Network().V1().HostSubnets(), netNamespaceInformer: networkInformers.Network().V1().NetNamespaces(), - subnetAllocatorMap: map[common.ParsedClusterNetworkEntry]*SubnetAllocator{}, - hostSubnetNodeIPs: map[ktypes.UID]string{}, + hostSubnetNodeIPs: map[ktypes.UID]string{}, } if err = master.checkClusterNetworkAgainstLocalNetworks(); err != nil { diff --git a/pkg/network/master/subnet_allocator.go b/pkg/network/master/subnet_allocator.go deleted file mode 100644 index 876a89fff..000000000 --- a/pkg/network/master/subnet_allocator.go +++ /dev/null @@ -1,234 +0,0 @@ -package master - -import ( - "encoding/binary" - "fmt" - "net" - "sync" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -var ErrSubnetAllocatorFull = fmt.Errorf("No subnets available.") - -type SubnetAllocator struct { - network *net.IPNet - hostBits uint32 - leftShift uint32 - leftMask uint32 - rightShift uint32 - rightMask uint32 - next uint32 - allocMap map[string]bool - mutex sync.Mutex -} - -func newSubnetAllocator(network string, hostBits uint32) (*SubnetAllocator, error) { - _, netIP, err := net.ParseCIDR(network) - if err != nil { - return nil, fmt.Errorf("failed to parse network address: %q", network) - } - - netMaskSize, _ := netIP.Mask.Size() - if hostBits == 0 { - return nil, fmt.Errorf("host capacity cannot be zero.") - } else if hostBits > (32 - uint32(netMaskSize)) { - return nil, fmt.Errorf("subnet capacity cannot be larger than number of networks available.") - } - subnetBits := 32 - uint32(netMaskSize) - hostBits - - // In the simple case, the subnet part of the 32-bit IP address is just the subnet - // number shifted hostBits to the left. However, if hostBits isn't a multiple of - // 8, then it can be difficult to distinguish the subnet part and the host part - // visually. (Eg, given network="10.1.0.0/16" and hostBits=6, then "10.1.0.50" and - // "10.1.0.70" are on different networks.) - // - // To try to avoid this confusion, if the subnet extends into the next higher - // octet, we rotate the bits of the subnet number so that we use the subnets with - // all 0s in the shared octet first. So again given network="10.1.0.0/16", - // hostBits=6, we first allocate 10.1.0.0/26, 10.1.1.0/26, etc, through - // 10.1.255.0/26 (just like we would with /24s in the hostBits=8 case), and only - // if we use up all of those subnets do we start allocating 10.1.0.64/26, - // 10.1.1.64/26, etc. - var leftShift, rightShift uint32 - var leftMask, rightMask uint32 - if hostBits%8 != 0 && ((hostBits-1)/8 != (hostBits+subnetBits-1)/8) { - leftShift = 8 - (hostBits % 8) - leftMask = uint32(1)<<(32-uint32(netMaskSize)) - 1 - rightShift = subnetBits - leftShift - rightMask = (uint32(1)<> sna.rightShift) & sna.rightMask) - genIp := Uint32ToIP(ipu) - genSubnet := &net.IPNet{IP: genIp, Mask: net.CIDRMask(int(numSubnetBits)+netMaskSize, 32)} - if !sna.allocMap[genSubnet.String()] { - sna.allocMap[genSubnet.String()] = true - sna.next = n + 1 - return genSubnet, nil - } - } - - sna.next = 0 - return nil, ErrSubnetAllocatorFull -} - -func (sna *SubnetAllocator) releaseNetwork(ipnet *net.IPNet) error { - sna.mutex.Lock() - defer sna.mutex.Unlock() - - if !sna.network.Contains(ipnet.IP) { - return fmt.Errorf("provided subnet %v doesn't belong to the network %v.", ipnet, sna.network) - } - - ipnetStr := ipnet.String() - if !sna.allocMap[ipnetStr] { - return fmt.Errorf("provided subnet %v is already available.", ipnet) - } else { - sna.allocMap[ipnetStr] = false - } - return nil -} - -func IPToUint32(ip net.IP) uint32 { - return binary.BigEndian.Uint32(ip.To4()) -} - -func Uint32ToIP(u uint32) net.IP { - ip := make([]byte, 4) - binary.BigEndian.PutUint32(ip, u) - return net.IPv4(ip[0], ip[1], ip[2], ip[3]) -} - -//--------------------- Master methods ---------------------- - -func (master *OsdnMaster) initSubnetAllocators() error { - for _, cn := range master.networkInfo.ClusterNetworks { - sa, err := newSubnetAllocator(cn.ClusterCIDR.String(), cn.HostSubnetLength) - if err != nil { - return err - } - master.subnetAllocatorList = append(master.subnetAllocatorList, sa) - master.subnetAllocatorMap[cn] = sa - } - - // Populate subnet allocator - subnets, err := master.networkClient.NetworkV1().HostSubnets().List(metav1.ListOptions{}) - if err != nil { - return err - } - for _, sn := range subnets.Items { - if err := master.markAllocatedNetwork(sn.Subnet); err != nil { - utilruntime.HandleError(err) - } - } - - return nil -} - -func (master *OsdnMaster) markAllocatedNetwork(subnet string) error { - sa, ipnet, err := master.getSubnetAllocator(subnet) - if err != nil { - return err - } - if err = sa.markAllocatedNetwork(ipnet); err != nil { - return err - } - return nil -} - -func (master *OsdnMaster) allocateNetwork(nodeName string) (string, error) { - var sn *net.IPNet - var err error - - for _, possibleSubnet := range master.subnetAllocatorList { - sn, err = possibleSubnet.allocateNetwork() - if err == ErrSubnetAllocatorFull { - // Current subnet exhausted, check the next one - continue - } else if err != nil { - utilruntime.HandleError(fmt.Errorf("Error allocating network from subnet: %v", possibleSubnet)) - continue - } else { - return sn.String(), nil - } - } - return "", fmt.Errorf("error allocating network for node %s: %v", nodeName, err) -} - -func (master *OsdnMaster) releaseNetwork(subnet string) error { - sa, ipnet, err := master.getSubnetAllocator(subnet) - if err != nil { - return err - } - if err = sa.releaseNetwork(ipnet); err != nil { - return err - } - return nil -} - -func (master *OsdnMaster) getSubnetAllocator(subnet string) (*SubnetAllocator, *net.IPNet, error) { - _, ipnet, err := net.ParseCIDR(subnet) - if err != nil { - return nil, nil, fmt.Errorf("error parsing subnet %q: %v", subnet, err) - } - - for _, cn := range master.networkInfo.ClusterNetworks { - if cn.ClusterCIDR.Contains(ipnet.IP) { - sa, ok := master.subnetAllocatorMap[cn] - if !ok || sa == nil { - return nil, nil, fmt.Errorf("subnet allocator not found for cluster network: %v", cn) - } - return sa, ipnet, nil - } - } - return nil, nil, fmt.Errorf("subnet %q not found in the cluster networks: %v", subnet, master.networkInfo.ClusterNetworks) -} diff --git a/pkg/network/master/subnet_allocator_test.go b/pkg/network/master/subnet_allocator_test.go deleted file mode 100644 index 1cf64d577..000000000 --- a/pkg/network/master/subnet_allocator_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package master - -import ( - "fmt" - "net" - "testing" -) - -func TestAllocateSubnet(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 8) - if err != nil { - t.Fatal("Failed to initialize subnet allocator: ", err) - } - - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.0.0/24" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 0, sn.String()) - } - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.1.0/24" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 1, sn.String()) - } - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.2.0/24" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 2, sn.String()) - } -} - -// 10.1.SSSSSSHH.HHHHHHHH -func TestAllocateSubnetLargeHostBits(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 10) - if err != nil { - t.Fatal("Failed to initialize subnet allocator: ", err) - } - - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.0.0/22" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 0, sn.String()) - } - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.4.0/22" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 1, sn.String()) - } - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.8.0/22" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 2, sn.String()) - } - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.12.0/22" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 3, sn.String()) - } -} - -// 10.1.SSSSSSSS.SSHHHHHH -func TestAllocateSubnetLargeSubnetBits(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 6) - if err != nil { - t.Fatal("Failed to initialize subnet allocator: ", err) - } - - for n := 0; n < 256; n++ { - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != fmt.Sprintf("10.1.%d.0/26", n) { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", n, sn.String()) - } - } - - for n := 0; n < 256; n++ { - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != fmt.Sprintf("10.1.%d.64/26", n) { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", n+256, sn.String()) - } - } - - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.0.128/26" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 512, sn.String()) - } -} - -// 10.000000SS.SSSSSSHH.HHHHHHHH -func TestAllocateSubnetOverlapping(t *testing.T) { - sna, err := newSubnetAllocator("10.0.0.0/14", 10) - if err != nil { - t.Fatal("Failed to initialize subnet allocator: ", err) - } - - for n := 0; n < 4; n++ { - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != fmt.Sprintf("10.%d.0.0/22", n) { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", n, sn.String()) - } - } - - for n := 0; n < 4; n++ { - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != fmt.Sprintf("10.%d.4.0/22", n) { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", n+4, sn.String()) - } - } - - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.0.8.0/22" { - t.Fatalf("Did not get expected subnet (n=%d, sn=%s)", 8, sn.String()) - } -} - -// 10.1.HHHHHHHH.HHHHHHHH -func TestAllocateSubnetNoSubnetBits(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 16) - if err != nil { - t.Fatal("Failed to initialize subnet allocator: ", err) - } - - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != "10.1.0.0/16" { - t.Fatalf("Did not get expected subnet (sn=%s)", sn.String()) - } - - sn, err = sna.allocateNetwork() - if err == nil { - t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn.String()) - } -} - -func TestAllocateSubnetInvalidHostBitsOrCIDR(t *testing.T) { - _, err := newSubnetAllocator("10.1.0.0/16", 18) - if err == nil { - t.Fatal("Unexpectedly succeeded in initializing subnet allocator") - } - - _, err = newSubnetAllocator("10.1.0.0/16", 0) - if err == nil { - t.Fatal("Unexpectedly succeeded in initializing subnet allocator") - } - - _, err = newSubnetAllocator("10.1.0.0/33", 16) - if err == nil { - t.Fatal("Unexpectedly succeeded in initializing subnet allocator") - } -} - -func TestMarkAllocatedNetwork(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 14) - if err != nil { - t.Fatal("Failed to initialize IP allocator: ", err) - } - - allocSubnets := make([]*net.IPNet, 4) - for i := 0; i < 4; i++ { - if allocSubnets[i], err = sna.allocateNetwork(); err != nil { - t.Fatal("Failed to allocate network: ", err) - } - } - - if sn, err := sna.allocateNetwork(); err == nil { - t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn.String()) - } - if err := sna.releaseNetwork(allocSubnets[2]); err != nil { - t.Fatalf("Failed to release the subnet (allocSubnets[2]=%s): %v", allocSubnets[2].String(), err) - } - for i := 0; i < 2; i++ { - if err := sna.markAllocatedNetwork(allocSubnets[2]); err != nil { - t.Fatalf("Failed to mark allocated subnet (allocSubnets[2]=%s): %v", allocSubnets[2].String(), err) - } - } - if sn, err := sna.allocateNetwork(); err == nil { - t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn.String()) - } - - // Test subnet that does not belong to network - var sn *net.IPNet - _, sn, err = net.ParseCIDR("10.2.3.4/24") - if err != nil { - t.Fatal("Failed to parse given network: ", err) - } - if err := sna.markAllocatedNetwork(sn); err == nil { - t.Fatalf("Unexpectedly succeeded in marking allocated subnet that doesn't belong to network (sn=%s)", sn.String()) - } -} - -func TestAllocateReleaseSubnet(t *testing.T) { - sna, err := newSubnetAllocator("10.1.0.0/16", 14) - if err != nil { - t.Fatal("Failed to initialize IP allocator: ", err) - } - - var releaseSn *net.IPNet - - for i := 0; i < 4; i++ { - sn, err := sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != fmt.Sprintf("10.1.%d.0/18", i*64) { - t.Fatalf("Did not get expected subnet (i=%d, sn=%s)", i, sn.String()) - } - if i == 2 { - releaseSn = sn - } - } - - sn, err := sna.allocateNetwork() - if err == nil { - t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn.String()) - } - - if err := sna.releaseNetwork(releaseSn); err != nil { - t.Fatalf("Failed to release the subnet (releaseSn=%s): %v", releaseSn.String(), err) - } - - sn, err = sna.allocateNetwork() - if err != nil { - t.Fatal("Failed to allocate network: ", err) - } - if sn.String() != releaseSn.String() { - t.Fatalf("Did not get expected subnet (sn=%s)", sn.String()) - } - - sn, err = sna.allocateNetwork() - if err == nil { - t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn.String()) - } -} - -func TestIPUint32Conversion(t *testing.T) { - ip := net.ParseIP("10.1.2.3") - if ip == nil { - t.Fatal("Failed to parse IP") - } - - u := IPToUint32(ip) - t.Log(u) - ip2 := Uint32ToIP(u) - t.Log(ip2) - - if !ip2.Equal(ip) { - t.Fatal("Conversion back and forth failed") - } -} diff --git a/pkg/network/master/subnets.go b/pkg/network/master/subnets.go index 48241d28d..b23d61965 100644 --- a/pkg/network/master/subnets.go +++ b/pkg/network/master/subnets.go @@ -16,12 +16,28 @@ import ( networkapi "github.com/openshift/api/network/v1" "github.com/openshift/sdn/pkg/network" "github.com/openshift/sdn/pkg/network/common" + masterutil "github.com/openshift/sdn/pkg/network/master/util" ) func (master *OsdnMaster) startSubnetMaster() error { - if err := master.initSubnetAllocators(); err != nil { + master.subnetAllocator = masterutil.NewSubnetAllocator() + for _, cn := range master.networkInfo.ClusterNetworks { + err := master.subnetAllocator.AddNetworkRange(cn.ClusterCIDR.String(), cn.HostSubnetLength) + if err != nil { + return err + } + } + + // Populate subnet allocator + subnets, err := master.networkClient.NetworkV1().HostSubnets().List(metav1.ListOptions{}) + if err != nil { return err } + for _, sn := range subnets.Items { + if err := master.subnetAllocator.MarkAllocatedNetwork(sn.Subnet); err != nil { + utilruntime.HandleError(err) + } + } master.watchNodes() master.watchSubnets() @@ -112,9 +128,9 @@ func (master *OsdnMaster) addNode(nodeName string, nodeUID string, nodeIP string } hsAnnotations[networkapi.NodeUIDAnnotation] = nodeUID } - network, err := master.allocateNetwork(nodeName) + network, err := master.subnetAllocator.AllocateNetwork() if err != nil { - return "", err + return "", fmt.Errorf("error allocating network for node %s: %v", nodeName, err) } sub = &networkapi.HostSubnet{ TypeMeta: metav1.TypeMeta{Kind: "HostSubnet"}, @@ -125,7 +141,7 @@ func (master *OsdnMaster) addNode(nodeName string, nodeUID string, nodeIP string } sub, err = master.networkClient.NetworkV1().HostSubnets().Create(sub) if err != nil { - if er := master.releaseNetwork(network); er != nil { + if er := master.subnetAllocator.ReleaseNetwork(network); er != nil { utilruntime.HandleError(er) } return "", fmt.Errorf("error allocating subnet for node %q: %v", nodeName, err) @@ -245,7 +261,7 @@ func (master *OsdnMaster) handleDeleteSubnet(obj interface{}) { return } - if err := master.releaseNetwork(hs.Subnet); err != nil { + if err := master.subnetAllocator.ReleaseNetwork(hs.Subnet); err != nil { utilruntime.HandleError(err) } } diff --git a/pkg/network/master/util/subnet_allocator.go b/pkg/network/master/util/subnet_allocator.go new file mode 100644 index 000000000..56439557b --- /dev/null +++ b/pkg/network/master/util/subnet_allocator.go @@ -0,0 +1,213 @@ +package util + +import ( + "fmt" + "net" + "sync" +) + +var ErrSubnetAllocatorFull = fmt.Errorf("no subnets available.") + +type SubnetAllocator struct { + sync.Mutex + + ranges []*subnetAllocatorRange +} + +func NewSubnetAllocator() *SubnetAllocator { + return &SubnetAllocator{} +} + +func (sna *SubnetAllocator) AddNetworkRange(network string, hostBits uint32) error { + sna.Lock() + defer sna.Unlock() + + _, ipnet, err := net.ParseCIDR(network) + if err != nil { + return err + } + snr, err := newSubnetAllocatorRange(ipnet, hostBits) + if err != nil { + return err + } + sna.ranges = append(sna.ranges, snr) + return nil +} + +func (sna *SubnetAllocator) MarkAllocatedNetwork(subnet string) error { + sna.Lock() + defer sna.Unlock() + + _, ipnet, err := net.ParseCIDR(subnet) + if err != nil { + return err + } + for _, snr := range sna.ranges { + if snr.markAllocatedNetwork(ipnet) { + return nil + } + } + return fmt.Errorf("network %s does not belong to any known range", subnet) +} + +func (sna *SubnetAllocator) AllocateNetwork() (string, error) { + sna.Lock() + defer sna.Unlock() + + for _, snr := range sna.ranges { + sn := snr.allocateNetwork() + if sn != nil { + return sn.String(), nil + } + } + return "", ErrSubnetAllocatorFull +} + +func (sna *SubnetAllocator) ReleaseNetwork(subnet string) error { + sna.Lock() + defer sna.Unlock() + + _, ipnet, err := net.ParseCIDR(subnet) + if err != nil { + return err + } + for _, snr := range sna.ranges { + if snr.releaseNetwork(ipnet) { + return nil + } + } + return fmt.Errorf("network %s does not belong to any known range", subnet) +} + +// subnetAllocatorRange handles allocating subnets out of a single CIDR +type subnetAllocatorRange struct { + network *net.IPNet + hostBits uint32 + subnetBits uint32 + next uint32 + allocMap map[string]bool + + // IPv4-only address-alignment hackery; see below + leftShift uint32 + leftMask uint32 + rightShift uint32 + rightMask uint32 +} + +func newSubnetAllocatorRange(network *net.IPNet, hostBits uint32) (*subnetAllocatorRange, error) { + netMaskSize, addrLen := network.Mask.Size() + if hostBits == 0 { + return nil, fmt.Errorf("host capacity cannot be zero.") + } else if hostBits > uint32(addrLen-netMaskSize) { + return nil, fmt.Errorf("subnet capacity cannot be larger than number of networks available.") + } + subnetBits := uint32(addrLen-netMaskSize) - hostBits + + snr := &subnetAllocatorRange{ + network: network, + hostBits: hostBits, + subnetBits: subnetBits, + next: 0, + allocMap: make(map[string]bool), + } + + // In the simple case, the subnet part of the 32-bit IP address is just the subnet + // number shifted hostBits to the left. However, if hostBits isn't a multiple of + // 8, then it can be difficult to distinguish the subnet part and the host part + // visually. (Eg, given network="10.1.0.0/16" and hostBits=6, then "10.1.0.50" and + // "10.1.0.70" are on different networks.) + // + // To try to avoid this confusion, if the subnet extends into the next higher + // octet, we rotate the bits of the subnet number so that we use the subnets with + // all 0s in the shared octet first. So again given network="10.1.0.0/16", + // hostBits=6, we first allocate 10.1.0.0/26, 10.1.1.0/26, etc, through + // 10.1.255.0/26 (just like we would with /24s in the hostBits=8 case), and only + // if we use up all of those subnets do we start allocating 10.1.0.64/26, + // 10.1.1.64/26, etc. + // + // (For IPv6 we just don't bother worrying about this.) + if addrLen == 32 && hostBits%8 != 0 && ((hostBits-1)/8 != (hostBits+subnetBits-1)/8) { + // leftShift is used to move the subnet id left by the number of bits that + // the subnet part extends into the overlap octet (which is to say, the + // number of bits that the host part ISN'T using in that octet). leftMask + // masks out the bits that get shifted left out of the subnet part + snr.leftShift = 8 - (hostBits % 8) + snr.leftMask = 1< 24 { + // We need to make sure that the uint32 math below won't overflow. If + // snr.subnetBits > 32 then numSubnets has already overflowed, but also if + // numSubnets is between 1<<24 and 1<<32 then "base << (snr.hostBits % 8)" + // below could overflow if snr.hostBits%8 is non-0. So we cap numSubnets + // at 1<<24. "16M subnets ought to be enough for anybody." + numSubnets = 1 << 24 + } + + var i uint32 + for i = 0; i < numSubnets; i++ { + n := (i + snr.next) % numSubnets + base := n + if snr.leftShift != 0 { + base = ((base << snr.leftShift) & snr.leftMask) | ((base >> snr.rightShift) & snr.rightMask) + } else if addrLen == 128 && snr.subnetBits >= 16 { + // Skip the 0 subnet (and other subnets with all 0s in the low word) + // since the extra 0 word will get compressed out and make the address + // look different from addresses on other subnets. + if (base & 0xFFFF) == 0 { + continue + } + } + + genIP := append([]byte{}, []byte(snr.network.IP)...) + subnetBits := base << (snr.hostBits % 8) + b := (uint32(addrLen) - snr.hostBits - 1) / 8 + for subnetBits != 0 { + genIP[b] |= byte(subnetBits) + subnetBits >>= 8 + b-- + } + + genSubnet := &net.IPNet{IP: genIP, Mask: net.CIDRMask(int(snr.subnetBits)+netMaskSize, addrLen)} + if !snr.allocMap[genSubnet.String()] { + snr.allocMap[genSubnet.String()] = true + snr.next = n + 1 + return genSubnet + } + } + + snr.next = 0 + return nil +} + +// releaseNetwork marks network as being not in use, if it is part of snr's range. +// It returns whether the network was in snr's range. +func (snr *subnetAllocatorRange) releaseNetwork(network *net.IPNet) bool { + if !snr.network.Contains(network.IP) { + return false + } + + snr.allocMap[network.String()] = false + return true +} diff --git a/pkg/network/master/util/subnet_allocator_test.go b/pkg/network/master/util/subnet_allocator_test.go new file mode 100644 index 000000000..cee2522f6 --- /dev/null +++ b/pkg/network/master/util/subnet_allocator_test.go @@ -0,0 +1,406 @@ +package util + +import ( + "fmt" + "net" + "testing" +) + +func newSubnetAllocator(clusterCIDR string, hostBits uint32) (*SubnetAllocator, error) { + sna := NewSubnetAllocator() + err := sna.AddNetworkRange(clusterCIDR, hostBits) + return sna, err +} + +func networkID(n int) string { + if n == -1 { + return "network" + } else { + return fmt.Sprintf("network %d", n) + } +} + +func allocateExpected(sna *SubnetAllocator, n int, expected string) error { + // Canonicalize expected; eg "fd01:0:0:0::/64" -> "fd01::/64" + _, expectedCIDR, _ := net.ParseCIDR(expected) + expected = expectedCIDR.String() + + sn, err := sna.AllocateNetwork() + if err != nil { + return fmt.Errorf("failed to allocate %s (%s): %v", networkID(n), expected, err) + } + if sn != expected { + return fmt.Errorf("failed to allocate %s: expected %s, got %s", networkID(n), expected, sn) + } + return nil +} + +func allocateNotExpected(sna *SubnetAllocator, n int) error { + if sn, err := sna.AllocateNetwork(); err == nil { + return fmt.Errorf("unexpectedly succeeded in allocating %s (sn=%s)", networkID(n), sn) + } else if err != ErrSubnetAllocatorFull { + return fmt.Errorf("returned error was not ErrSubnetAllocatorFull (%v)", err) + } + return nil +} + +// 10.1.ssssssss.hhhhhhhh +func TestAllocateSubnetIPv4(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 8) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + for n := 0; n < 256; n++ { + if err := allocateExpected(sna, n, fmt.Sprintf("10.1.%d.0/24", n)); err != nil { + t.Fatal(err) + } + } + if err := allocateNotExpected(sna, 256); err != nil { + t.Fatal(err) + } +} + +// fd01:0:0:SSSS:HHHH:HHHH:HHHH:HHHH +func TestAllocateSubnetIPv6(t *testing.T) { + sna, err := newSubnetAllocator("fd01::/48", 64) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + // IPv6 allocation skips the 0 subnet, so we start with n=1 + for n := 1; n < 256; n++ { + if err := allocateExpected(sna, n, fmt.Sprintf("fd01:0:0:%x::/64", n)); err != nil { + t.Fatal(err) + } + } + if err := allocateExpected(sna, 256, "fd01:0:0:100::/64"); err != nil { + t.Fatal(err) + } + + // We have 16 bits for subnet, after which it will wrap around (and then allocate + // the next previously-unallocated value, skipping the 0 subnet again). + sna.ranges[0].next = 0xFFFF + if err := allocateExpected(sna, -1, "fd01:0:0:ffff::/64"); err != nil { + t.Fatal(err) + } + if err := allocateExpected(sna, -1, "fd01:0:0:101::/64"); err != nil { + t.Fatal(err) + } +} + +// 10.1.sssssshh.hhhhhhhh +func TestAllocateSubnetLargeHostBitsIPv4(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 10) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + for n := 0; n < 64; n++ { + if err := allocateExpected(sna, n, fmt.Sprintf("10.1.%d.0/22", n*4)); err != nil { + t.Fatal(err) + } + } + if err := allocateNotExpected(sna, 64); err != nil { + t.Fatal(err) + } +} + +// fd01:0:0:SSSH:HHHH:HHHH:HHHH:HHHH +func TestAllocateSubnetLargeHostBitsIPv6(t *testing.T) { + sna, err := newSubnetAllocator("fd01::/48", 68) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + // (Because of the small subnet size we won't skip the 0 subnet like in the + // other IPv6 cases.) + for n := 0; n < 256; n++ { + if err := allocateExpected(sna, n, fmt.Sprintf("fd01:0:0:%x::/60", n<<4)); err != nil { + t.Fatal(err) + } + } +} + +// 10.1.ssssssss.sshhhhhh +func TestAllocateSubnetLargeSubnetBitsIPv4(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 6) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + // for IPv4, we tweak the allocation order and expect to see all of the ".0" + // networks before any non-".0" network + for n := 0; n < 256; n++ { + if err = allocateExpected(sna, n, fmt.Sprintf("10.1.%d.0/26", n)); err != nil { + t.Fatal(err) + } + } + for n := 0; n < 256; n++ { + if err = allocateExpected(sna, n+256, fmt.Sprintf("10.1.%d.64/26", n)); err != nil { + t.Fatal(err) + } + } + if err = allocateExpected(sna, 512, "10.1.0.128/26"); err != nil { + t.Fatal(err) + } + + sna.ranges[0].next = 1023 + if err = allocateExpected(sna, -1, "10.1.255.192/26"); err != nil { + t.Fatal(err) + } + // Next allocation should wrap around and get the next unallocated network (513) + if err = allocateExpected(sna, -1, "10.1.1.128/26"); err != nil { + t.Fatalf("After wraparound: %v", err) + } +} + +// fd01:0:0:SSSS:SSSS:SHHH:HHHH:HHHH +func TestAllocateSubnetLargeSubnetBitsIPv6(t *testing.T) { + sna, err := newSubnetAllocator("fd01::/48", 44) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + // For IPv6 we expect to see the networks just get allocated in order + for n := 1; n < 256; n++ { + // (Many of the IPv6 strings we Sprintf here won't be in canonical + // format, but allocateExpecting() will fix them for us.) + if err := allocateExpected(sna, n, fmt.Sprintf("fd01:0:0:0:%x:%x::/84", n>>4, (n<<12)&0xFFFF)); err != nil { + t.Fatal(err) + } + } + if err := allocateExpected(sna, -1, "fd01:0:0:0:10:0::/84"); err != nil { + t.Fatal(err) + } + + // Even though we theoretically have 36 bits of subnets, SubnetAllocator will only + // use the lower 24 bits before looping around and then allocating the next + // previously-unallocated subnet. + sna.ranges[0].next = 0x00FFFFFF + if err := allocateExpected(sna, -1, "fd01:0:0:000f:ffff:f000::/84"); err != nil { + t.Fatal(err) + } + if err := allocateExpected(sna, -1, "fd01:0:0:0:10:1000::/84"); err != nil { + t.Fatal(err) + } +} + +// 10.000000ss.sssssshh.hhhhhhhh +func TestAllocateSubnetOverlappingIPv4(t *testing.T) { + sna, err := newSubnetAllocator("10.0.0.0/14", 10) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + for n := 0; n < 4; n++ { + if err = allocateExpected(sna, n, fmt.Sprintf("10.%d.0.0/22", n)); err != nil { + t.Fatal(err) + } + } + for n := 0; n < 4; n++ { + if err = allocateExpected(sna, n+4, fmt.Sprintf("10.%d.4.0/22", n)); err != nil { + t.Fatal(err) + } + } + if err := allocateExpected(sna, 8, "10.0.8.0/22"); err != nil { + t.Fatal(err) + } + + sna.ranges[0].next = 255 + if err := allocateExpected(sna, -1, "10.3.252.0/22"); err != nil { + t.Fatal(err) + } + if err := allocateExpected(sna, -1, "10.1.8.0/22"); err != nil { + t.Fatalf("After wraparound: %v", err) + } +} + +// There's no TestAllocateSubnetOverlappingIPv6 because it wouldn't be any different from +// TestAllocateSubnetLargeSubnetBitsIPv6. + +// 10.1.hhhhhhhh.hhhhhhhh +func TestAllocateSubnetNoSubnetBitsIPv4(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 16) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + if err := allocateExpected(sna, 0, "10.1.0.0/16"); err != nil { + t.Fatal(err) + } + if err := allocateNotExpected(sna, 1); err != nil { + t.Fatal(err) + } +} + +func TestAllocateSubnetNoSubnetBitsIPv6(t *testing.T) { + // fd01:0:0:0:HHHH:HHHH:HHHH:HHHH + sna, err := newSubnetAllocator("fd01::/64", 64) + if err != nil { + t.Fatal("Failed to initialize subnet allocator: ", err) + } + + if err := allocateExpected(sna, 0, "fd01::/64"); err != nil { + t.Fatal(err) + } + if err := allocateNotExpected(sna, 1); err != nil { + t.Fatal(err) + } +} + +func TestAllocateSubnetInvalidHostBitsOrCIDR(t *testing.T) { + _, err := newSubnetAllocator("10.1.0.0/16", 18) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } + + _, err = newSubnetAllocator("10.1.0.0/16", 0) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } + + _, err = newSubnetAllocator("10.1.0.0/33", 16) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } + + _, err = newSubnetAllocator("fd01::/64", 66) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } + + _, err = newSubnetAllocator("fd01::/64", 0) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } + + _, err = newSubnetAllocator("fd01::/129", 64) + if err == nil { + t.Fatal("Unexpectedly succeeded in initializing subnet allocator") + } +} + +func TestMarkAllocatedNetwork(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 14) + if err != nil { + t.Fatal("Failed to initialize IP allocator: ", err) + } + + allocSubnets := make([]string, 4) + for i := 0; i < 4; i++ { + if allocSubnets[i], err = sna.AllocateNetwork(); err != nil { + t.Fatal("Failed to allocate network: ", err) + } + } + + if sn, err := sna.AllocateNetwork(); err == nil { + t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn) + } + if err := sna.ReleaseNetwork(allocSubnets[2]); err != nil { + t.Fatalf("Failed to release the subnet (allocSubnets[2]=%s): %v", allocSubnets[2], err) + } + for i := 0; i < 2; i++ { + if err := sna.MarkAllocatedNetwork(allocSubnets[2]); err != nil { + t.Fatalf("Failed to mark allocated subnet (allocSubnets[2]=%s): %v", allocSubnets[2], err) + } + } + if sn, err := sna.AllocateNetwork(); err == nil { + t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn) + } + + // Test subnet that does not belong to network + sn := "10.2.3.4/24" + if err := sna.MarkAllocatedNetwork(sn); err == nil { + t.Fatalf("Unexpectedly succeeded in marking allocated subnet that doesn't belong to network (sn=%s)", sn) + } +} + +func TestAllocateReleaseSubnet(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 14) + if err != nil { + t.Fatal("Failed to initialize IP allocator: ", err) + } + + var releaseSn string + + for i := 0; i < 4; i++ { + sn, err := sna.AllocateNetwork() + if err != nil { + t.Fatal("Failed to allocate network: ", err) + } + if sn != fmt.Sprintf("10.1.%d.0/18", i*64) { + t.Fatalf("Did not get expected subnet (i=%d, sn=%s)", i, sn) + } + if i == 2 { + releaseSn = sn + } + } + + sn, err := sna.AllocateNetwork() + if err == nil { + t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn) + } + + if err := sna.ReleaseNetwork(releaseSn); err != nil { + t.Fatalf("Failed to release the subnet (releaseSn=%s): %v", releaseSn, err) + } + + sn, err = sna.AllocateNetwork() + if err != nil { + t.Fatal("Failed to allocate network: ", err) + } + if sn != releaseSn { + t.Fatalf("Did not get expected subnet (sn=%s)", sn) + } + + sn, err = sna.AllocateNetwork() + if err == nil { + t.Fatalf("Unexpectedly succeeded in allocating network (sn=%s)", sn) + } +} + +func TestMultipleSubnets(t *testing.T) { + sna, err := newSubnetAllocator("10.1.0.0/16", 14) + if err != nil { + t.Fatal("Failed to initialize IP allocator: ", err) + } + err = sna.AddNetworkRange("10.2.0.0/16", 14) + if err != nil { + t.Fatal("Failed to add network range: ", err) + } + + for i := 0; i < 4; i++ { + if err := allocateExpected(sna, i, fmt.Sprintf("10.1.%d.0/18", i*64)); err != nil { + t.Fatal(err) + } + } + + for i := 0; i < 4; i++ { + if err := allocateExpected(sna, i+4, fmt.Sprintf("10.2.%d.0/18", i*64)); err != nil { + t.Fatal(err) + } + } + + if err := allocateNotExpected(sna, 8); err != nil { + t.Fatal(err) + } + + if err := sna.ReleaseNetwork("10.1.128.0/18"); err != nil { + t.Fatalf("Failed to release the subnet 10.1.128.0/18: %v", err) + } + if err := sna.ReleaseNetwork("10.2.128.0/18"); err != nil { + t.Fatalf("Failed to release the subnet 10.2.128.0/18: %v", err) + } + + if err := allocateExpected(sna, -1, "10.1.128.0/18"); err != nil { + t.Fatal(err) + } + if err := allocateExpected(sna, -1, "10.2.128.0/18"); err != nil { + t.Fatal(err) + } + + if err := allocateNotExpected(sna, -1); err != nil { + t.Fatal(err) + } +}