From 9469ee6e50b4142c1092ba2d04cabbc627dccd3d Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Aug 2020 17:26:10 -0700 Subject: [PATCH 01/10] First try --- pkg/drivers/kic/kic.go | 14 +++++++-- pkg/drivers/kic/oci/errors.go | 3 ++ pkg/drivers/kic/oci/network.go | 56 +++++++++++++++++++++++++++------- pkg/drivers/kic/oci/oci.go | 4 +-- pkg/drivers/kic/oci/types.go | 6 ---- 5 files changed, 60 insertions(+), 23 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 3adcd5b51fd9..db2d8ccea669 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -81,11 +81,19 @@ func (d *Driver) Create() error { } defaultNetwork := d.MachineName - if err := oci.CreateNetwork(defaultNetwork, oci.DefaultIPRange, oci.DefaultGateway); err != nil { - glog.Warningf("unable to create docker network; node ip may not be stable: %v", err) + if subnet, err := oci.CreateNetwork(defaultNetwork); err != nil { + glog.Errorf("unable to create docker network; node ip may not be stable: %v", err) } else { params.Network = defaultNetwork - params.IP = oci.DefaultIP + fmt.Println("subnet is", subnet) + ip := net.ParseIP(strings.Split(subnet, "/")[0]) + fmt.Println("the ip is", ip) + // make sure it's only 4 bytes + ip = ip.To4() + // check ip != nil + ip[3]++ + ip[3]++ + params.IP = ip.String() } // control plane specific options diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 8c30201f978c..e02e319856ff 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -45,6 +45,9 @@ var ErrExitedUnexpectedly = errors.New("container exited unexpectedly") // ErrDaemonInfo is thrown when docker/podman info is failing or not responding var ErrDaemonInfo = errors.New("daemon info not responding") +// ErrNetworkSubnetTaken is thrown when a subnet is taken by another network +var ErrNetworkSubnetTaken = errors.New("subnet is taken") + // LogContainerDebug will print relevant docker/podman infos after a container fails func LogContainerDebug(ociBin string, name string) string { rr, err := containerInspect(ociBin, name) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index dc905a277797..761014c4e9f7 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -61,11 +61,6 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine // gets the ip from user's host docker func dockerGatewayIP(profile string) (net.IP, error) { - // check if using custom network first - if networkExists(profile) { - ip := net.ParseIP(DefaultGateway) - return ip, nil - } rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { return nil, errors.Wrapf(err, "get network bridge") @@ -168,32 +163,71 @@ func dockerContainerIP(name string) (string, string, error) { return ips[0], ips[1], nil } -// CreateNetwork creates a network -func CreateNetwork(name, ipRange, gateway string) error { +// CreateNetwork creates a network returns subnet and error +func CreateNetwork(name string) (string, error) { // check if the network already exists if networkExists(name) { - return nil + return "", nil } - subnet := fmt.Sprintf("--subnet=%s", ipRange) - _, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", subnet, "--gateway", gateway, name)) + // simple way to create networks, subnet is taken, try one bigger + attempt := 0 + subnet := fmt.Sprintf("192.168.%d.0/24", 39) + fmt.Println("about to create network with", subnet) + err := attemptCreateNework(subnet, name) if err != nil { - return errors.Wrapf(err, "error creating network") + if err == ErrNetworkSubnetTaken { + for attempt < 10 { + attempt++ + glog.Warningf("Couldn't create network %q at %q subnet will try a new subnet", name, subnet) + // increase by 10 each time + subnet = fmt.Sprintf(fmt.Sprintf("192.168.%d.0/24", 39+10*attempt)) + err := attemptCreateNework(subnet, name) + if err == nil { + return subnet, nil + } + if err == ErrNetworkSubnetTaken { + continue + } + } + } + return "", errors.Wrapf(err, "error creating network") + } + fmt.Println("created newrok", name, subnet) + + return subnet, nil +} +func attemptCreateNework(subnet string, name string) error { + fmt.Println("inside attempt", subnet, name) + rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) + if err != nil { + fmt.Printf("failed to created newrok: %v", err) + if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { + return ErrNetworkSubnetTaken + } + return errors.Wrapf(err, "error creating network") + } return nil } // removeNetwork removes a network func removeNetwork(name string) error { + fmt.Println("inside remove container", name) if !networkExists(name) { return nil } _, err := runCmd(exec.Command(Docker, "network", "remove", name)) + fmt.Printf("ran : %v\n", err) + + // TODO: handel this error for multinode med@xmac:~/workspace/minikube (nework_delete)$ docker network rm mynet123 + // Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints return err } func networkExists(name string) bool { + fmt.Println("inside networkExists", name) rr, err := runCmd(exec.Command(Docker, "network", "ls", "--format", "{{.Name}}")) if err != nil { glog.Warningf("error listing networks: %v", err) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index f8c0eb2b6aed..eb9ea1321777 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -153,10 +153,8 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name)) } if p.OCIBinary == Docker { - // on linux, we can provide a static IP for docker - if runtime.GOOS == "linux" && p.Network != "" && p.IP != "" { + if p.Network != "" { runArgs = append(runArgs, "--network", p.Network) - runArgs = append(runArgs, "--ip", p.IP) } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 9ede5b703278..7e9487cef77c 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -31,12 +31,6 @@ const ( nodeRoleLabelKey = "role.minikube.sigs.k8s.io" // CreatedByLabelKey is applied to any container/volume that is created by minikube created_by.minikube.sigs.k8s.io=true CreatedByLabelKey = "created_by.minikube.sigs.k8s.io" - // DefaultGateway is the default gateway for the docker network created by the kic driver on linux - DefaultGateway = "192.168.39.1" - // DefaultIPRange is the default IP range for the docker network created by the kic driver on linux - DefaultIPRange = "192.168.39.0/24" - // DefaultIP is the default IP for the docker network created by the kic driver on linux - DefaultIP = "192.168.39.2" ) // CreateParams are parameters needed to create a container From 264f9c99b75a79790dd0c0033d05a3ae2f0715a3 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Aug 2020 20:43:56 -0700 Subject: [PATCH 02/10] fix docker static ip --- cmd/minikube/cmd/delete.go | 10 +++ pkg/drivers/kic/kic.go | 12 +-- pkg/drivers/kic/oci/errors.go | 9 ++ pkg/drivers/kic/oci/network.go | 145 +++++++++++++++++++++++++-------- pkg/drivers/kic/oci/oci.go | 6 +- pkg/drivers/kic/oci/types.go | 2 + pkg/minikube/machine/delete.go | 10 ++- 7 files changed, 148 insertions(+), 46 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 98ed5df2e3bb..60ca2bc9b3b7 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -119,6 +119,11 @@ func deleteContainersAndVolumes(ociBin string) { if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs) } + + errs = oci.DeleteAllNetworksByKIC() + if errs != nil { + glog.Warningf("error deleting left over networks (might be okay).\nTo see the list of netowrks: 'docker network ls'\n:%v", errs) + } } // runDelete handles the executes the flow of "minikube delete" @@ -258,6 +263,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) { glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } + errs = oci.DeleteAllNetworksByKIC() + if errs != nil { + glog.Warningf("error deleting left over networks (might be okay).\nTo see the list of netowrks: 'docker network ls'\n:%v", errs) + } + if bin == oci.Podman { // podman prune does not support --filter return diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index db2d8ccea669..a646286bb5fb 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -85,14 +85,10 @@ func (d *Driver) Create() error { glog.Errorf("unable to create docker network; node ip may not be stable: %v", err) } else { params.Network = defaultNetwork - fmt.Println("subnet is", subnet) - ip := net.ParseIP(strings.Split(subnet, "/")[0]) - fmt.Println("the ip is", ip) - // make sure it's only 4 bytes - ip = ip.To4() - // check ip != nil - ip[3]++ - ip[3]++ + ip := subnet.IP.To4() + if ip != nil { + ip[3]++ + } params.IP = ip.String() } diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index e02e319856ff..c5efa3bda9e5 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -48,6 +48,15 @@ var ErrDaemonInfo = errors.New("daemon info not responding") // ErrNetworkSubnetTaken is thrown when a subnet is taken by another network var ErrNetworkSubnetTaken = errors.New("subnet is taken") +// ErrNetworkNotFound is when given network was not found +var ErrNetworkNotFound = errors.New("kic network not found") + +// ErrNetworkGatewayTaken is when given network gatway is taken +var ErrNetworkGatewayTaken = errors.New("network gateway is taken") + +// ErrNetworkInUse is when trying to delete a network which is attached to another container +var ErrNetworkInUse = errors.New("can't delete network attached to a running container") + // LogContainerDebug will print relevant docker/podman infos after a container fails func LogContainerDebug(ociBin string, name string) string { rr, err := containerInspect(ociBin, name) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 761014c4e9f7..889ef4e89936 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -17,6 +17,8 @@ limitations under the License. package oci import ( + "bufio" + "bytes" "fmt" "net" "os/exec" @@ -33,7 +35,11 @@ import ( func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - return dockerGatewayIP(containerName) + _, gateway, err := dockerNetworkInspect(containerName) + if err != nil { + return gateway, errors.Wrap(err, "routable gateway") + } + return gateway, nil } // for windows and mac, the gateway ip is not routable so we use dns trick. return digDNS(ociBin, containerName, "host.docker.internal") @@ -164,24 +170,29 @@ func dockerContainerIP(name string) (string, string, error) { } // CreateNetwork creates a network returns subnet and error -func CreateNetwork(name string) (string, error) { +func CreateNetwork(name string) (*net.IPNet, error) { // check if the network already exists - if networkExists(name) { - return "", nil + subnet, _, err := dockerNetworkInspect(name) + if err == nil { + return subnet, nil } // simple way to create networks, subnet is taken, try one bigger attempt := 0 - subnet := fmt.Sprintf("192.168.%d.0/24", 39) - fmt.Println("about to create network with", subnet) - err := attemptCreateNework(subnet, name) + _, subnet, err = net.ParseCIDR(defaultSubnet) + if err != nil { + return nil, errors.Wrapf(err, "parse default subnet %s", defaultSubnet) + } + + err = attemptCreateNework(subnet, name) if err != nil { if err == ErrNetworkSubnetTaken { - for attempt < 10 { + // try up tp 13 times + for attempt < 13 { attempt++ - glog.Warningf("Couldn't create network %q at %q subnet will try a new subnet", name, subnet) - // increase by 10 each time - subnet = fmt.Sprintf(fmt.Sprintf("192.168.%d.0/24", 39+10*attempt)) + glog.Infof("Couldn't create network %q at %q subnet will try again a new subnet ...", name, subnet) + // increase 2nd digit by 10 each time + subnet.IP.To4()[2] += 10 err := attemptCreateNework(subnet, name) if err == nil { return subnet, nil @@ -191,53 +202,121 @@ func CreateNetwork(name string) (string, error) { } } } - return "", errors.Wrapf(err, "error creating network") + return nil, errors.Wrapf(err, "error creating network") } - fmt.Println("created newrok", name, subnet) - return subnet, nil } -func attemptCreateNework(subnet string, name string) error { - fmt.Println("inside attempt", subnet, name) - rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) +func attemptCreateNework(subnet *net.IPNet, name string) error { + gateway := subnet.IP.To4() + gateway[3]++ // first ip for gateway + glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway) + rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) if err != nil { fmt.Printf("failed to created newrok: %v", err) if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { return ErrNetworkSubnetTaken } + if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") { + return ErrNetworkGatewayTaken + } return errors.Wrapf(err, "error creating network") } return nil } -// removeNetwork removes a network -func removeNetwork(name string) error { - fmt.Println("inside remove container", name) +// RemoveNetwork removes a network +func RemoveNetwork(name string) error { if !networkExists(name) { return nil } - _, err := runCmd(exec.Command(Docker, "network", "remove", name)) - fmt.Printf("ran : %v\n", err) + rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + if err != nil { + if strings.Contains(rr.Output(), "No such network:") { + return ErrNetworkNotFound + } + // Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints + if strings.Contains(rr.Output(), "has active endpoints") { + return ErrNetworkInUse + } + } - // TODO: handel this error for multinode med@xmac:~/workspace/minikube (nework_delete)$ docker network rm mynet123 - // Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints return err } func networkExists(name string) bool { - fmt.Println("inside networkExists", name) - rr, err := runCmd(exec.Command(Docker, "network", "ls", "--format", "{{.Name}}")) - if err != nil { - glog.Warningf("error listing networks: %v", err) + if _, _, err := dockerNetworkInspect(name); err != nil { + if err == ErrNetworkNotFound { + return false + } + glog.Warningf("error inspecting network %s: %v", name, err) return false } - networks := strings.Split(rr.Output(), "\n") - for _, n := range networks { - if strings.Trim(n, "\n") == name { - return true + return true +} + +// returns subnet and gate if exists +func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) { + rr, err := runCmd(exec.Command(Docker, "network", "inspect", name, "--format", "{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}")) + if err != nil { + if strings.Contains(rr.Output(), "No such network:") { + return nil, nil, ErrNetworkNotFound } + return nil, nil, err + } + // results looks like 172.17.0.0/16,172.17.0.1 + ips := strings.Split(rr.Stdout.String(), ",") + if len(ips) == 0 { + return nil, nil, fmt.Errorf("invalid network info") + } + + _, subnet, err := net.ParseCIDR(ips[0]) + if err != nil { + return nil, nil, errors.Wrapf(err, "parse subnet for %s", name) } - return false + if len(ips) == 1 { + return subnet, nil, nil + } + gateway := net.ParseIP(ips[1]) + return subnet, gateway, nil +} + +// returns all network names created by a label +func allNetworkByLabel(ociBin string, label string) ([]string, error) { + if ociBin != Docker { + return nil, fmt.Errorf("%s not supported", ociBin) + } + + // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}} + rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + if err != nil { + return nil, err + } + var lines []string + scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) + for scanner.Scan() { + lines = append(lines, strings.TrimSpace(scanner.Text())) + } + + return lines, nil +} + +// DeleteAllNetworksByKIC delets all networks created by kic +func DeleteAllNetworksByKIC() []error { + var errs []error + ns, err := allNetworkByLabel(Docker, CreatedByLabelKey+"=true") + if err != nil { + return []error{errors.Wrap(err, "list all volume")} + } + for _, n := range ns { + err := RemoveNetwork(n) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errs + } + return nil } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index eb9ea1321777..fed862896814 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -88,7 +88,7 @@ func DeleteContainer(ociBin string, name string) error { if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil { return errors.Wrapf(err, "delete %s", name) } - if err := removeNetwork(name); err != nil { + if err := RemoveNetwork(name); err != nil { return errors.Wrap(err, "removing network") } return nil @@ -153,8 +153,10 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name)) } if p.OCIBinary == Docker { - if p.Network != "" { + // to provide a static IP for docker + if p.Network != "" && p.IP != "" { runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 7e9487cef77c..1bc0ec00adc8 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -31,6 +31,8 @@ const ( nodeRoleLabelKey = "role.minikube.sigs.k8s.io" // CreatedByLabelKey is applied to any container/volume that is created by minikube created_by.minikube.sigs.k8s.io=true CreatedByLabelKey = "created_by.minikube.sigs.k8s.io" + // DefaultSubnet subnet to be used on first cluster + defaultSubnet = "192.168.39.0/24" ) // CreateParams are parameters needed to create a container diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 3c0a2d6f1288..6685de7df7fa 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -45,18 +45,22 @@ func deleteOrphanedKIC(ociBin string, name string) { glog.Infof("couldn't inspect container %q before deleting: %v", name, err) return } - // allow no more than 5 seconds for delting the container - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() if err := oci.ShutDown(ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } + // allow no more than 10 seconds for deleting the container + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() cmd := exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name) err = cmd.Run() if err == nil { glog.Infof("Found stale kic container and successfully cleaned it up!") } + if err := oci.RemoveNetwork(name); err != nil { + glog.Infof("couldn't delete network ", name, err) + } + } // DeleteHost deletes the host VM. From c48aae6b710629c94f2ce98d8ca7057f821d5211 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Tue, 25 Aug 2020 21:50:28 -0700 Subject: [PATCH 03/10] fix mount with static ip --- cmd/minikube/cmd/mount.go | 2 ++ pkg/drivers/kic/oci/network.go | 38 +++++++--------------------------- pkg/minikube/machine/delete.go | 13 +++++++----- 3 files changed, 18 insertions(+), 35 deletions(-) diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index 28ea923e7cdc..c41c1633eee2 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -117,6 +117,7 @@ var mountCmd = &cobra.Command{ } } port, err := getPort() + fmt.Println("port is ", ip) if err != nil { exit.WithError("Error finding port for mount", err) } @@ -147,6 +148,7 @@ var mountCmd = &cobra.Command{ } bindIP := ip.String() // the ip to listen on the user's host machine + fmt.Println("bind ip is", bindIP) if driver.IsKIC(co.CP.Host.Driver.DriverName()) && runtime.GOOS != "linux" { bindIP = "127.0.0.1" } diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 889ef4e89936..b1308b869115 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -46,7 +46,7 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro } if runtime.GOOS == "linux" { - return containerGatewayIP(ociBin, containerName) + return podmanGatewayIP(containerName) } return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux") @@ -64,29 +64,9 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { return ip, nil } -// dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine -// gets the ip from user's host docker -func dockerGatewayIP(profile string) (net.IP, error) { - rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) - if err != nil { - return nil, errors.Wrapf(err, "get network bridge") - } - - bridgeID := strings.TrimSpace(rr.Stdout.String()) - rr, err = runCmd(exec.Command(Docker, "network", "inspect", - "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) - if err != nil { - return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID) - } - - ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) - glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String()) - return ip, nil -} - -// containerGatewayIP gets the default gateway ip for the container -func containerGatewayIP(ociBin, containerName string) (net.IP, error) { - rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName)) +// podmanGatewayIP gets the default gateway ip for the container +func podmanGatewayIP(containerName string) (net.IP, error) { + rr, err := runCmd(exec.Command(Podman, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName)) if err != nil { return nil, errors.Wrapf(err, "inspect gateway") } @@ -214,7 +194,6 @@ func attemptCreateNework(subnet *net.IPNet, name string) error { glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway) rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) if err != nil { - fmt.Printf("failed to created newrok: %v", err) if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { return ErrNetworkSubnetTaken } @@ -266,19 +245,18 @@ func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) { return nil, nil, err } // results looks like 172.17.0.0/16,172.17.0.1 - ips := strings.Split(rr.Stdout.String(), ",") + ips := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",") if len(ips) == 0 { return nil, nil, fmt.Errorf("invalid network info") } - _, subnet, err := net.ParseCIDR(ips[0]) if err != nil { return nil, nil, errors.Wrapf(err, "parse subnet for %s", name) } - if len(ips) == 1 { - return subnet, nil, nil + var gateway net.IP + if len(ips) > 0 { + gateway = net.ParseIP(ips[1]) } - gateway := net.ParseIP(ips[1]) return subnet, gateway, nil } diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 6685de7df7fa..6770c4abe0e7 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -40,6 +40,12 @@ func deleteOrphanedKIC(ociBin string, name string) { return } + defer func() { // networks should be deleted after containers + if err := oci.RemoveNetwork(name); err != nil { + glog.Infof("couldn't delete network %s (might be okay): %v", name, err) + } + }() + _, err := oci.ContainerStatus(ociBin, name) if err != nil { glog.Infof("couldn't inspect container %q before deleting: %v", name, err) @@ -49,17 +55,14 @@ func deleteOrphanedKIC(ociBin string, name string) { if err := oci.ShutDown(ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - // allow no more than 10 seconds for deleting the container - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + // allow no more than 5 seconds for deleting the container + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name) err = cmd.Run() if err == nil { glog.Infof("Found stale kic container and successfully cleaned it up!") } - if err := oci.RemoveNetwork(name); err != nil { - glog.Infof("couldn't delete network ", name, err) - } } From 8c3a18fce8f49a6de643b000fb65f41adb68e208 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 00:01:21 -0700 Subject: [PATCH 04/10] address review comments --- hack/jenkins/common.sh | 3 ++ hack/jenkins/cron/cleanup_and_reboot_Linux.sh | 2 + pkg/drivers/kic/oci/network.go | 40 +++++++++++-------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 3742bfd0a430..1037b111ccbd 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -35,8 +35,11 @@ sudo ./installers/check_install_golang.sh "1.14.6" "/usr/local" || true docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true +docker network prune -f || true +docker system prune -f || true docker system df || true + echo ">> Starting at $(date)" echo "" echo "arch: ${OS_ARCH}" diff --git a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh index e03e6cd06b67..8a84e25077fc 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh @@ -39,6 +39,8 @@ killall java # clean docker left overs docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true +docker system prune -f || true +docker network prune -f || true docker volume ls || true docker system df || true diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index b1308b869115..e574f3a8f5ff 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -37,7 +37,7 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro if runtime.GOOS == "linux" { _, gateway, err := dockerNetworkInspect(containerName) if err != nil { - return gateway, errors.Wrap(err, "routable gateway") + return gateway, errors.Wrap(err, "network inspect") } return gateway, nil } @@ -166,23 +166,28 @@ func CreateNetwork(name string) (*net.IPNet, error) { err = attemptCreateNework(subnet, name) if err != nil { - if err == ErrNetworkSubnetTaken { - // try up tp 13 times - for attempt < 13 { - attempt++ - glog.Infof("Couldn't create network %q at %q subnet will try again a new subnet ...", name, subnet) - // increase 2nd digit by 10 each time - subnet.IP.To4()[2] += 10 - err := attemptCreateNework(subnet, name) - if err == nil { - return subnet, nil - } - if err == ErrNetworkSubnetTaken { - continue - } + if err != ErrNetworkSubnetTaken { + return nil, errors.Wrapf(err, "error creating network") + } + // try up to 13 times + // we can try up to 255 + for attempt < 13 { + attempt++ + glog.Infof("Couldn't create network %q at %q subnet will try again with a new subnet ...", name, subnet) + // increase 3nd digit by 10 each time + // 13 times adding 10 defaultSubnet "192.168.39.0/24" + // at most it will add up to 169 which is still less than max allowed 255 + // this is large enough to try more and not too small to not try enough + // can be tuned in the next iterations + subnet.IP.To4()[2] += 10 + err := attemptCreateNework(subnet, name) + if err == nil { + return subnet, nil + } + if err == ErrNetworkSubnetTaken { + continue } } - return nil, errors.Wrapf(err, "error creating network") } return subnet, nil @@ -192,7 +197,8 @@ func attemptCreateNework(subnet *net.IPNet, name string) error { gateway := subnet.IP.To4() gateway[3]++ // first ip for gateway glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway) - rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) + // options documenation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options + rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", "-o", "com.docker.network.bridge.enable_icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) if err != nil { if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { return ErrNetworkSubnetTaken From 06b9a35a2e9254bc41000a546380638cca1ff72b Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 00:04:02 -0700 Subject: [PATCH 05/10] remove debugging info --- cmd/minikube/cmd/mount.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index c41c1633eee2..28ea923e7cdc 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -117,7 +117,6 @@ var mountCmd = &cobra.Command{ } } port, err := getPort() - fmt.Println("port is ", ip) if err != nil { exit.WithError("Error finding port for mount", err) } @@ -148,7 +147,6 @@ var mountCmd = &cobra.Command{ } bindIP := ip.String() // the ip to listen on the user's host machine - fmt.Println("bind ip is", bindIP) if driver.IsKIC(co.CP.Host.Driver.DriverName()) && runtime.GOOS != "linux" { bindIP = "127.0.0.1" } From cd0bcb15def90f7401568dccaf94af1e2d20e774 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 00:04:44 -0700 Subject: [PATCH 06/10] add more cleanup --- hack/jenkins/cron/cleanup_and_reboot_Darwin.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh index 75c7dfedb456..a20beed1c6c1 100755 --- a/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh @@ -44,6 +44,8 @@ killall java # clean docker left overs docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true docker volume prune -f || true +docker system prune -f || true +docker network prune -f || true docker volume ls || true docker system df || true From 15323f8dcae60833c92bf40c86f5d4227d58d9c1 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 00:17:48 -0700 Subject: [PATCH 07/10] shorter option --- pkg/drivers/kic/oci/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index e574f3a8f5ff..522d8f088e7b 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -198,7 +198,7 @@ func attemptCreateNework(subnet *net.IPNet, name string) error { gateway[3]++ // first ip for gateway glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway) // options documenation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options - rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "com.docker.network.bridge.enable_ip_masquerade=true", "-o", "com.docker.network.bridge.enable_icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) + rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) if err != nil { if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { return ErrNetworkSubnetTaken From b7a256fb4d0efc88278f83b9a8ab0e6432175617 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 02:36:37 -0700 Subject: [PATCH 08/10] create one network per cluster --- cmd/minikube/cmd/mount.go | 2 +- hack/preload-images/generate.go | 1 + pkg/drivers/kic/kic.go | 29 +++++++--- pkg/drivers/kic/kic_test.go | 62 +++++++++++++++++++++ pkg/drivers/kic/oci/network.go | 36 ++++++------ pkg/drivers/kic/types.go | 1 + pkg/minikube/cluster/ip.go | 6 +- pkg/minikube/node/start.go | 2 +- pkg/minikube/registry/drvs/docker/docker.go | 1 + pkg/minikube/registry/drvs/podman/podman.go | 1 + 10 files changed, 113 insertions(+), 28 deletions(-) create mode 100644 pkg/drivers/kic/kic_test.go diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index 28ea923e7cdc..f984efe1598b 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -106,7 +106,7 @@ var mountCmd = &cobra.Command{ var ip net.IP var err error if mountIP == "" { - ip, err = cluster.HostIP(co.CP.Host) + ip, err = cluster.HostIP(co.CP.Host, co.Config.Name) if err != nil { exit.WithError("Error getting the host IP address to use from within the VM", err) } diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index ec3de2ece506..e1d6061d8bd6 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -39,6 +39,7 @@ import ( func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { driver := kic.NewDriver(kic.Config{ + ClusterName: profile, KubernetesVersion: kubernetesVersion, ContainerRuntime: containerRuntime, OCIBinary: oci.Docker, diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index a646286bb5fb..45d4b34aabd5 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,6 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util/retry" ) @@ -65,6 +66,18 @@ func NewDriver(c Config) *Driver { return d } +// machineOrder returns the order of the container based on it is name +func machineOrder(machineName string) int { + // minikube-m02 + sp := strings.Split(machineName, "-") + m := strings.Trim(sp[len(sp)-1], "m") // m02 + i, err := strconv.Atoi(m) + if err != nil { + return 1 + } + return i +} + // Create a host using the driver's config func (d *Driver) Create() error { params := oci.CreateParams{ @@ -80,15 +93,17 @@ func (d *Driver) Create() error { APIServerPort: d.NodeConfig.APIServerPort, } - defaultNetwork := d.MachineName - if subnet, err := oci.CreateNetwork(defaultNetwork); err != nil { - glog.Errorf("unable to create docker network; node ip may not be stable: %v", err) + // one network bridge per cluster. + defaultNetwork := d.NodeConfig.ClusterName + if gateway, err := oci.CreateNetwork(d.OCIBinary, defaultNetwork); err != nil { + glog.Warningf("failed to create network: %v", err) + out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart.") } else { params.Network = defaultNetwork - ip := subnet.IP.To4() - if ip != nil { - ip[3]++ - } + ip := gateway.To4() + i := machineOrder(d.NodeConfig.MachineName) + ip[3] = ip[3] + byte(i) + glog.Infof("calculated static IP %q for the %q container ", ip.String(), d.NodeConfig.MachineName) params.IP = ip.String() } diff --git a/pkg/drivers/kic/kic_test.go b/pkg/drivers/kic/kic_test.go new file mode 100644 index 000000000000..d607bbc7f2dd --- /dev/null +++ b/pkg/drivers/kic/kic_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kic + +import ( + "testing" +) + +func TestMachineOrder(t *testing.T) { + testCases := []struct { + Name string + MachineName string + Want int + }{ + { + Name: "default", + MachineName: "minikube", + Want: 1}, + { + Name: "second-node", + MachineName: "minikube-m02", + Want: 2}, + { + Name: "dash-profile", + MachineName: "my-dashy-minikube", + Want: 1}, + + { + Name: "dash-profile-second-node", + MachineName: "my-dashy-minikube-m02", + Want: 2}, + { + Name: "michivious-user", + MachineName: "michivious-user-m02-m03", + Want: 3}, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + got := machineOrder(tc.MachineName) + if got != tc.Want { + t.Errorf("want order %q but got %q", tc.Want, got) + + } + }) + + } +} diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 522d8f088e7b..93a43d5dd60e 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -32,10 +32,10 @@ import ( // RoutableHostIPFromInside returns the ip/dns of the host that container lives on // is routable from inside the container -func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) { +func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - _, gateway, err := dockerNetworkInspect(containerName) + _, gateway, err := dockerNetworkInspect(clusterName) if err != nil { return gateway, errors.Wrap(err, "network inspect") } @@ -149,14 +149,17 @@ func dockerContainerIP(name string) (string, string, error) { return ips[0], ips[1], nil } -// CreateNetwork creates a network returns subnet and error -func CreateNetwork(name string) (*net.IPNet, error) { +// CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster +func CreateNetwork(ociBin string, name string) (net.IP, error) { + if ociBin != Docker { + return nil, fmt.Errorf("Podman network not implemented yet", ociBin) + } // check if the network already exists - subnet, _, err := dockerNetworkInspect(name) + subnet, gateway, err := dockerNetworkInspect(name) if err == nil { - return subnet, nil + glog.Info("Found existing network with subnet %q and gatway %q.", subnet, gateway) + return gateway, nil } - // simple way to create networks, subnet is taken, try one bigger attempt := 0 _, subnet, err = net.ParseCIDR(defaultSubnet) @@ -164,7 +167,7 @@ func CreateNetwork(name string) (*net.IPNet, error) { return nil, errors.Wrapf(err, "parse default subnet %s", defaultSubnet) } - err = attemptCreateNework(subnet, name) + gateway, err = attemptCreateNework(subnet, name) if err != nil { if err != ErrNetworkSubnetTaken { return nil, errors.Wrapf(err, "error creating network") @@ -180,9 +183,9 @@ func CreateNetwork(name string) (*net.IPNet, error) { // this is large enough to try more and not too small to not try enough // can be tuned in the next iterations subnet.IP.To4()[2] += 10 - err := attemptCreateNework(subnet, name) + gateway, err := attemptCreateNework(subnet, name) if err == nil { - return subnet, nil + return gateway, nil } if err == ErrNetworkSubnetTaken { continue @@ -190,10 +193,10 @@ func CreateNetwork(name string) (*net.IPNet, error) { } } - return subnet, nil + return gateway, nil } -func attemptCreateNework(subnet *net.IPNet, name string) error { +func attemptCreateNework(subnet *net.IPNet, name string) (net.IP, error) { gateway := subnet.IP.To4() gateway[3]++ // first ip for gateway glog.Infof("attempt to create network %q with subnet: %s and gateway %s...", subnet, name, gateway) @@ -201,14 +204,14 @@ func attemptCreateNework(subnet *net.IPNet, name string) error { rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", subnet), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name)) if err != nil { if strings.Contains(rr.Output(), "Pool overlaps with other one on this address space") { - return ErrNetworkSubnetTaken + return nil, ErrNetworkSubnetTaken } if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") { - return ErrNetworkGatewayTaken + return nil, ErrNetworkGatewayTaken } - return errors.Wrapf(err, "error creating network") + return nil, errors.Wrapf(err, "error creating network") } - return nil + return gateway, nil } // RemoveNetwork removes a network @@ -255,6 +258,7 @@ func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) { if len(ips) == 0 { return nil, nil, fmt.Errorf("invalid network info") } + _, subnet, err := net.ParseCIDR(ips[0]) if err != nil { return nil, nil, errors.Wrapf(err, "parse subnet for %s", name) diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index a374da2a135b..0acfb49f21ec 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -48,6 +48,7 @@ var ( // Config is configuration for the kic driver used by registry type Config struct { + ClusterName string // profile name or cluster name MachineName string // maps to the container name being created CPU int // Number of CPU cores assigned to the container Memory int // max memory in MB diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index f853848399ab..4c731e3689aa 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -34,12 +34,12 @@ import ( ) // HostIP gets the ip address to be used for mapping host -> VM and VM -> host -func HostIP(host *host.Host) (net.IP, error) { +func HostIP(host *host.Host, clusterName string) (net.IP, error) { switch host.DriverName { case driver.Docker: - return oci.RoutableHostIPFromInside(oci.Docker, host.Name) + return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name) case driver.Podman: - return oci.RoutableHostIPFromInside(oci.Podman, host.Name) + return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name) case driver.KVM2: return net.ParseIP("192.168.39.1"), nil case driver.HyperV: diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index ba4b7f7e2aed..9e13389f8704 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -92,7 +92,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { showVersionInfo(starter.Node.KubernetesVersion, cr) // Add "host.minikube.internal" DNS alias (intentionally non-fatal) - hostIP, err := cluster.HostIP(starter.Host) + hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { glog.Errorf("Unable to get host IP: %v", err) } else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index fbbf66cfe07c..2e761db961aa 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -50,6 +50,7 @@ func init() { func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { return kic.NewDriver(kic.Config{ + ClusterName: cc.Name, MachineName: driver.MachineName(cc, n), StorePath: localpath.MiniPath(), ImageDigest: cc.KicBaseImage, diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index ad16ae468143..1f9c1245aea4 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -64,6 +64,7 @@ func init() { func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { return kic.NewDriver(kic.Config{ + ClusterName: cc.Name, MachineName: driver.MachineName(cc, n), StorePath: localpath.MiniPath(), ImageDigest: strings.Split(cc.KicBaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. From 7572812861e3fe770d792b30a148ee9683c11ef8 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 02:38:11 -0700 Subject: [PATCH 09/10] simplify --- pkg/drivers/kic/kic.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 45d4b34aabd5..509792b7a2e7 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -101,8 +101,8 @@ func (d *Driver) Create() error { } else { params.Network = defaultNetwork ip := gateway.To4() - i := machineOrder(d.NodeConfig.MachineName) - ip[3] = ip[3] + byte(i) + // calculate the container IP based on its machine order + ip[3] = ip[3] + byte(machineOrder(d.NodeConfig.MachineName)) glog.Infof("calculated static IP %q for the %q container ", ip.String(), d.NodeConfig.MachineName) params.IP = ip.String() } From ce77d0547b88630af83e8adadb974d4e5561999e Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 26 Aug 2020 02:43:50 -0700 Subject: [PATCH 10/10] fail fast if IP is in use --- pkg/drivers/kic/oci/errors.go | 3 +++ pkg/drivers/kic/oci/oci.go | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index c5efa3bda9e5..0889358bc0e9 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -39,6 +39,9 @@ var ErrWindowsContainers = &FailFastError{errors.New("docker container type is w // ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container var ErrCPUCountLimit = &FailFastError{errors.New("not enough CPUs is available for container")} +// ErrIPinUse is thrown when the container been given an IP used by another container +var ErrIPinUse = &FailFastError{errors.New("can't create with that IP, address already in use")} + // ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore. var ErrExitedUnexpectedly = errors.New("container exited unexpectedly") diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index fed862896814..1ca8abc86fa9 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -276,6 +276,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error { if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available return ErrCPUCountLimit } + // example: docker: Error response from daemon: Address already in use. + if strings.Contains(rr.Output(), "Address already in use") { + return ErrIPinUse + } return err }