Skip to content

Commit fb27ab7

Browse files
authored
Merge pull request #10439 from prezha/prevent-oci-overlapping-net
prevent oci try to create overlapping network
2 parents 77e9751 + 7727b85 commit fb27ab7

File tree

4 files changed

+252
-28
lines changed

4 files changed

+252
-28
lines changed

pkg/drivers/kic/oci/network_create.go

+10-23
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/pkg/errors"
2929

3030
"k8s.io/klog/v2"
31+
"k8s.io/minikube/pkg/network"
3132
)
3233

3334
// firstSubnetAddr subnet to be used on first kic cluster
@@ -70,32 +71,18 @@ func CreateNetwork(ociBin string, networkName string) (net.IP, error) {
7071
if err != nil {
7172
klog.Warningf("failed to get mtu information from the %s's default network %q: %v", ociBin, defaultBridgeName, err)
7273
}
73-
attempts := 0
74-
subnetAddr := firstSubnetAddr
7574
// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
7675
// will be like 192.168.49.0/24 ,...,192.168.239.0/24
77-
for attempts < 20 {
78-
info.gateway, err = tryCreateDockerNetwork(ociBin, subnetAddr, defaultSubnetMask, info.mtu, networkName)
79-
if err == nil {
80-
return info.gateway, nil
81-
}
82-
83-
// don't retry if error is not adddress is taken
84-
if !(errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken)) {
85-
klog.Errorf("error while trying to create network %v", err)
86-
return nil, errors.Wrap(err, "un-retryable")
87-
}
88-
attempts++
89-
// Find an open subnet by incrementing the 3rd octet by 10 for each try
90-
// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
91-
// at most it will add up to 169 which is still less than max allowed 255
92-
// this is large enough to try more and not too small to not try enough
93-
// can be tuned in the next iterations
94-
newSubnet := net.ParseIP(subnetAddr).To4()
95-
newSubnet[2] += byte(9 + attempts)
96-
subnetAddr = newSubnet.String()
76+
subnet, err := network.FreeSubnet(firstSubnetAddr, 10, 20)
77+
if err != nil {
78+
klog.Errorf("error while trying to create network: %v", err)
79+
return nil, errors.Wrap(err, "un-retryable")
80+
}
81+
info.gateway, err = tryCreateDockerNetwork(ociBin, subnet.IP, defaultSubnetMask, info.mtu, networkName)
82+
if err != nil {
83+
return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
9784
}
98-
return info.gateway, fmt.Errorf("failed to create network after 20 attempts")
85+
return info.gateway, nil
9986
}
10087

10188
func tryCreateDockerNetwork(ociBin string, subnetAddr string, subnetMask int, mtu int, name string) (net.IP, error) {

pkg/drivers/kvm/network.go

+27-4
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,35 @@ import (
3131
"github.com/docker/machine/libmachine/log"
3232
libvirt "github.com/libvirt/libvirt-go"
3333
"github.com/pkg/errors"
34+
"k8s.io/minikube/pkg/network"
3435
"k8s.io/minikube/pkg/util/retry"
3536
)
3637

3738
// Replace with hardcoded range with CIDR
3839
// https://play.golang.org/p/m8TNTtygK0
3940
const networkTmpl = `
4041
<network>
41-
<name>{{.PrivateNetwork}}</name>
42+
<name>{{.Name}}</name>
4243
<dns enable='no'/>
43-
<ip address='192.168.39.1' netmask='255.255.255.0'>
44+
{{with .Parameters}}
45+
<ip address='{{.Gateway}}' netmask='{{.Netmask}}'>
4446
<dhcp>
45-
<range start='192.168.39.2' end='192.168.39.254'/>
47+
<range start='{{.ClientMin}}' end='{{.ClientMax}}'/>
4648
</dhcp>
4749
</ip>
50+
{{end}}
4851
</network>
4952
`
5053

54+
type kvmNetwork struct {
55+
Name string
56+
network.Parameters
57+
}
58+
59+
// firstSubnetAddr is starting subnet to try for new KVM cluster,
60+
// avoiding possible conflict with other local networks by further incrementing it up to 20 times by 10.
61+
const firstSubnetAddr = "192.168.39.0"
62+
5163
// setupNetwork ensures that the network with `name` is started (active)
5264
// and has the autostart feature set.
5365
func setupNetwork(conn *libvirt.Connect, name string) error {
@@ -145,10 +157,20 @@ func (d *Driver) createNetwork() error {
145157
// Only create the private network if it does not already exist
146158
netp, err := conn.LookupNetworkByName(d.PrivateNetwork)
147159
if err != nil {
160+
subnet, err := network.FreeSubnet(firstSubnetAddr, 10, 20)
161+
if err != nil {
162+
log.Debugf("error while trying to create network: %v", err)
163+
return errors.Wrap(err, "un-retryable")
164+
}
165+
tryNet := kvmNetwork{
166+
Name: d.PrivateNetwork,
167+
Parameters: *subnet,
168+
}
169+
148170
// create the XML for the private network from our networkTmpl
149171
tmpl := template.Must(template.New("network").Parse(networkTmpl))
150172
var networkXML bytes.Buffer
151-
if err := tmpl.Execute(&networkXML, d); err != nil {
173+
if err := tmpl.Execute(&networkXML, tryNet); err != nil {
152174
return errors.Wrap(err, "executing network template")
153175
}
154176

@@ -173,6 +195,7 @@ func (d *Driver) createNetwork() error {
173195
if err := retry.Local(create, 10*time.Second); err != nil {
174196
return errors.Wrapf(err, "creating network %s", d.PrivateNetwork)
175197
}
198+
log.Debugf("Network %s created", d.PrivateNetwork)
176199
}
177200
defer func() {
178201
if netp != nil {

pkg/minikube/cluster/ip.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) {
4747
}
4848
return net.ParseIP(ip), nil
4949
case driver.KVM2:
50-
return net.ParseIP("192.168.39.1"), nil
50+
ip, err := host.Driver.GetIP()
51+
if err != nil {
52+
return []byte{}, errors.Wrap(err, "Error getting VM/Host IP address")
53+
}
54+
return net.ParseIP(ip), nil
5155
case driver.HyperV:
5256
v := reflect.ValueOf(host.Driver).Elem()
5357
var hypervVirtualSwitch string

pkg/network/network.go

+210
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package network
18+
19+
import (
20+
"encoding/binary"
21+
"fmt"
22+
"net"
23+
24+
"github.com/pkg/errors"
25+
"k8s.io/klog/v2"
26+
)
27+
28+
var (
29+
// valid private network subnets (RFC1918)
30+
privateSubnets = []net.IPNet{
31+
// 10.0.0.0/8
32+
{
33+
IP: []byte{10, 0, 0, 0},
34+
Mask: []byte{255, 0, 0, 0},
35+
},
36+
// 172.16.0.0/12
37+
{
38+
IP: []byte{172, 16, 0, 0},
39+
Mask: []byte{255, 240, 0, 0},
40+
},
41+
// 192.168.0.0/16
42+
{
43+
IP: []byte{192, 168, 0, 0},
44+
Mask: []byte{255, 255, 0, 0},
45+
},
46+
}
47+
)
48+
49+
// Parameters contains main network parameters.
50+
type Parameters struct {
51+
IP string // IP address of the network
52+
Netmask string // form: 4-byte ('a.b.c.d')
53+
CIDR string // form: CIDR
54+
Gateway string // first IP address (assumed, not checked !)
55+
ClientMin string // second IP address
56+
ClientMax string // last IP address before broadcastS
57+
Broadcast string // last IP address
58+
Interface
59+
}
60+
61+
// Interface contains main network interface parameters.
62+
type Interface struct {
63+
IfaceName string
64+
IfaceIPv4 string
65+
IfaceMTU int
66+
IfaceMAC string
67+
}
68+
69+
// inspect initialises IPv4 network parameters struct from given address.
70+
// address can be single address (like "192.168.17.42"), network address (like "192.168.17.0"), or in cidr form (like "192.168.17.42/24 or "192.168.17.0/24").
71+
// If addr is valid existsing interface address, network struct will also contain info about the respective interface.
72+
func inspect(addr string) (*Parameters, error) {
73+
n := &Parameters{}
74+
75+
// extract ip from addr
76+
ip, network, err := net.ParseCIDR(addr)
77+
if err != nil {
78+
ip = net.ParseIP(addr)
79+
if ip == nil {
80+
return nil, errors.Wrapf(err, "parsing address %q", addr)
81+
}
82+
}
83+
84+
// check local interfaces
85+
ifaces, _ := net.Interfaces()
86+
for _, iface := range ifaces {
87+
ifAddrs, err := iface.Addrs()
88+
if err != nil {
89+
return nil, errors.Wrapf(err, "listing addresses of network interface %+v", iface)
90+
}
91+
for _, ifAddr := range ifAddrs {
92+
ifip, lan, err := net.ParseCIDR(ifAddr.String())
93+
if err != nil {
94+
return nil, errors.Wrapf(err, "parsing address of network iface %+v", ifAddr)
95+
}
96+
if lan.Contains(ip) {
97+
n.IfaceName = iface.Name
98+
n.IfaceIPv4 = ifip.To4().String()
99+
n.IfaceMTU = iface.MTU
100+
n.IfaceMAC = iface.HardwareAddr.String()
101+
n.Gateway = n.IfaceIPv4
102+
network = lan
103+
break
104+
}
105+
}
106+
}
107+
108+
if network == nil {
109+
ipnet := &net.IPNet{
110+
IP: ip,
111+
Mask: ip.DefaultMask(), // assume default network mask
112+
}
113+
_, network, err = net.ParseCIDR(ipnet.String())
114+
if err != nil {
115+
return nil, errors.Wrapf(err, "determining network address from %q", addr)
116+
}
117+
}
118+
119+
n.IP = network.IP.String()
120+
n.Netmask = net.IP(network.Mask).String() // form: 4-byte ('a.b.c.d')
121+
n.CIDR = network.String()
122+
123+
networkIP := binary.BigEndian.Uint32(network.IP) // IP address of the network
124+
networkMask := binary.BigEndian.Uint32(network.Mask) // network mask
125+
broadcastIP := (networkIP & networkMask) | (networkMask ^ 0xffffffff) // last network IP address
126+
127+
broadcast := make(net.IP, 4)
128+
binary.BigEndian.PutUint32(broadcast, broadcastIP)
129+
n.Broadcast = broadcast.String()
130+
131+
gateway := net.ParseIP(n.Gateway).To4() // has to be converted to 4-byte representation!
132+
if gateway == nil {
133+
gateway = make(net.IP, 4)
134+
binary.BigEndian.PutUint32(gateway, networkIP+1) // assume first network IP address
135+
n.Gateway = gateway.String()
136+
}
137+
gatewayIP := binary.BigEndian.Uint32(gateway)
138+
139+
min := make(net.IP, 4)
140+
binary.BigEndian.PutUint32(min, gatewayIP+1) // clients-from: first network IP address after gateway
141+
n.ClientMin = min.String()
142+
143+
max := make(net.IP, 4)
144+
binary.BigEndian.PutUint32(max, broadcastIP-1) // clients-from: last network IP address before broadcast
145+
n.ClientMax = max.String()
146+
147+
return n, nil
148+
}
149+
150+
// isSubnetTaken returns if local network subnet exists and any error occurred.
151+
// If will return false in case of an error.
152+
func isSubnetTaken(subnet string) (bool, error) {
153+
ips, err := net.InterfaceAddrs()
154+
if err != nil {
155+
return false, errors.Wrap(err, "listing local networks")
156+
}
157+
for _, ip := range ips {
158+
_, lan, err := net.ParseCIDR(ip.String())
159+
if err != nil {
160+
return false, errors.Wrapf(err, "parsing network iface address %q", ip)
161+
}
162+
if lan.Contains(net.ParseIP(subnet)) {
163+
return true, nil
164+
}
165+
}
166+
return false, nil
167+
}
168+
169+
// isSubnetPrivate returns if subnet is a private network.
170+
func isSubnetPrivate(subnet string) bool {
171+
for _, ipnet := range privateSubnets {
172+
if ipnet.Contains(net.ParseIP(subnet)) {
173+
return true
174+
}
175+
}
176+
return false
177+
}
178+
179+
// FreeSubnet will try to find free private network beginning with startSubnet, incrementing it in steps up to number of tries.
180+
func FreeSubnet(startSubnet string, step, tries int) (*Parameters, error) {
181+
for try := 0; try < tries; try++ {
182+
n, err := inspect(startSubnet)
183+
if err != nil {
184+
return nil, err
185+
}
186+
startSubnet = n.IP
187+
if isSubnetPrivate(startSubnet) {
188+
taken, err := isSubnetTaken(startSubnet)
189+
if err != nil {
190+
return nil, err
191+
}
192+
if !taken {
193+
klog.Infof("using free private subnet %s: %+v", n.CIDR, n)
194+
return n, nil
195+
}
196+
klog.Infof("skipping subnet %s that is taken: %+v", n.CIDR, n)
197+
} else {
198+
klog.Infof("skipping subnet %s that is not private", n.CIDR)
199+
}
200+
ones, _ := net.ParseIP(n.IP).DefaultMask().Size()
201+
nextSubnet := net.ParseIP(startSubnet).To4()
202+
if ones <= 16 {
203+
nextSubnet[1] += byte(step)
204+
} else {
205+
nextSubnet[2] += byte(step)
206+
}
207+
startSubnet = nextSubnet.String()
208+
}
209+
return nil, fmt.Errorf("no free private network subnets found with given parameters (start: %q, step: %d, tries: %d)", startSubnet, step, tries)
210+
}

0 commit comments

Comments
 (0)