Merge pull request #6695 from medyagh/vol_fix

improve docker volume clean up

cmd/minikube/cmd/delete.go

@@ -113,13 +113,23 @@ func runDelete(cmd *cobra.Command, args []string) {
 		if profileFlag != constants.DefaultMachineName {
 			exit.UsageT("usage: minikube delete --all")
 		}
+		delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true")
+		errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel)
+		if len(errs) > 0 { // it will error if there is no container to delete
+			glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, err)
+		}
+
+		errs = oci.DeleteAllVolumesByLabel(oci.Docker, delLabel)
+		if len(errs) > 0 { // it will not error if there is nothing to delete
+			glog.Warningf("error delete volumes by label %q (might be okay): %+v", delLabel, errs)
+		}
 
-		err := oci.DeleteAllVolumesByLabel(oci.Docker, fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "=true"))
-		if err != nil { // if there is no volume there won't be any error
-			glog.Warningf("error deleting left docker volumes. \n%v:\nfor more information please refer to docker documentation: https://docs.docker.com/engine/reference/commandline/volume_prune/", err)
+		errs = oci.PruneAllVolumesByLabel(oci.Docker, delLabel)
+		if len(errs) > 0 { // it will not error if there is nothing to delete
+			glog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs)
 		}
 
-		errs := DeleteProfiles(profilesToDelete)
+		errs = DeleteProfiles(profilesToDelete)
 		if len(errs) > 0 {
 			HandleDeletionErrors(errs)
 		} else {
@@ -180,9 +190,20 @@ func DeleteProfiles(profiles []*pkg_config.Profile) []error {
 
 func deleteProfile(profile *pkg_config.Profile) error {
 	viper.Set(pkg_config.MachineProfile, profile.Name)
-	err := oci.DeleteAllVolumesByLabel(oci.Docker, fmt.Sprintf("%s=%s", oci.ProfileLabelKey, profile.Name))
-	if err != nil { // if there is no volume there wont be any error
-		glog.Warningf("error deleting left docker volumes. To see the list of volumes run: 'docker volume ls' \n:%v", err)
+
+	delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, profile.Name)
+	errs := oci.DeleteAllContainersByLabel(oci.Docker, delLabel)
+	if errs != nil { // it will error if there is no container to delete
+		glog.Infof("error deleting containers for %s (might be okay):\n%v", profile.Name, errs)
+	}
+	errs = oci.DeleteAllVolumesByLabel(oci.Docker, delLabel)
+	if errs != nil { // it will not error if there is nothing to delete
+		glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
+	}
+
+	errs = oci.PruneAllVolumesByLabel(oci.Docker, delLabel)
+	if len(errs) > 0 { // it will not error if there is nothing to delete
+		glog.Warningf("error pruning volume (might be okay):\n%v", errs)
 	}
 
 	api, err := machine.NewAPIClient()

pkg/drivers/kic/oci/oci.go

@@ -34,6 +34,31 @@ import (
 	"strings"
 )
 
+// DeleteAllContainersByLabel deletes all containers that have a specific label
+// if there no containers found with the given 	label, it will return nil
+func DeleteAllContainersByLabel(ociBin string, label string) []error {
+	var deleteErrs []error
+	if ociBin == Docker {
+		if err := PointToHostDockerDaemon(); err != nil {
+			return []error{errors.Wrap(err, "point host docker-daemon")}
+		}
+	}
+	cs, err := listContainersByLabel(ociBin, label)
+	if err != nil {
+		return []error{fmt.Errorf("listing containers by label %q", label)}
+	}
+	if len(cs) == 0 {
+		return nil
+	}
+	for _, c := range cs {
+		cmd := exec.Command(ociBin, "rm", "-f", "-v", c)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out))
+		}
+	}
+	return deleteErrs
+}
+
 // CreateContainerNode creates a new container node
 func CreateContainerNode(p CreateParams) error {
 	if err := PointToHostDockerDaemon(); err != nil {
@@ -399,23 +424,23 @@ func withPortMappings(portMappings []PortMapping) createOpt {
 	}
 }
 
-// listContainersByLabel lists all the containres that kic driver created on user's machine using a label
-// io.x-k8s.kic.cluster
+// listContainersByLabel returns all the container names with a specified label
 func listContainersByLabel(ociBinary string, label string) ([]string, error) {
 	if err := PointToHostDockerDaemon(); err != nil {
 		return nil, errors.Wrap(err, "point host docker-daemon")
 	}
 	cmd := exec.Command(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}")
-	var b bytes.Buffer
-	cmd.Stdout = &b
-	cmd.Stderr = &b
-	err := cmd.Run()
-	var lines []string
-	sc := bufio.NewScanner(&b)
-	for sc.Scan() {
-		lines = append(lines, sc.Text())
+	stdout, err := cmd.Output()
+	s := bufio.NewScanner(bytes.NewReader(stdout))
+	var names []string
+	for s.Scan() {
+		n := strings.TrimSpace(s.Text())
+		if n != "" {
+			names = append(names, n)
+		}
 	}
-	return lines, err
+
+	return names, err
 }
 
 // PointToHostDockerDaemon will unset env variables that point to docker inside minikube

pkg/drivers/kic/oci/volumes.go

@@ -17,27 +17,75 @@ limitations under the License.
 package oci
 
 import (
+	"bufio"
+	"bytes"
+	"fmt"
 	"os/exec"
+	"strings"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 )
 
 // DeleteAllVolumesByLabel deletes all volumes that have a specific label
 // if there is no volume to delete it will return nil
+func DeleteAllVolumesByLabel(ociBin string, label string) []error {
+	var deleteErrs []error
+	glog.Infof("trying to delete all %s volumes with label %s", ociBin, label)
+	if ociBin == Docker {
+		if err := PointToHostDockerDaemon(); err != nil {
+			return []error{errors.Wrap(err, "point host docker-daemon")}
+		}
+	}
+
+	vs, err := allVolumesByLabel(ociBin, label)
+	if err != nil {
+		return []error{fmt.Errorf("listing volumes by label %q: %v", label, err)}
+	}
+
+	for _, v := range vs {
+		cmd := exec.Command(ociBin, "volume", "rm", "--force", v)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			deleteErrs = append(deleteErrs, fmt.Errorf("deleting volume %s: output: %s", v, string(out)))
+		}
+	}
+	return deleteErrs
+}
+
+// PruneAllVolumesByLabel deletes all volumes that have a specific label
+// if there is no volume to delete it will return nil
 // example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube
-func DeleteAllVolumesByLabel(ociBin string, label string) error {
+func PruneAllVolumesByLabel(ociBin string, label string) []error {
+	var deleteErrs []error
 	glog.Infof("trying to prune all %s volumes with label %s", ociBin, label)
 	if ociBin == Docker {
 		if err := PointToHostDockerDaemon(); err != nil {
-			return errors.Wrap(err, "point host docker-daemon")
+			return []error{errors.Wrap(err, "point host docker-daemon")}
 		}
 	}
-	cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", label)
+
+	// try to prune afterwards just in case delete didn't go through
+	cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label)
 	if out, err := cmd.CombinedOutput(); err != nil {
-		return errors.Wrapf(err, "output %s", string(out))
+		deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s: %s", label, string(out)))
 	}
-	return nil
+	return deleteErrs
+}
+
+// allVolumesByLabel returns name of all docker volumes by a specific label
+// will not return error if there is no volume found.
+func allVolumesByLabel(ociBin string, label string) ([]string, error) {
+	cmd := exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")
+	stdout, err := cmd.Output()
+	s := bufio.NewScanner(bytes.NewReader(stdout))
+	var vols []string
+	for s.Scan() {
+		v := strings.TrimSpace(s.Text())
+		if v != "" {
+			vols = append(vols, v)
+		}
+	}
+	return vols, err
 }
 
 // createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
@@ -47,7 +95,7 @@ func createDockerVolume(name string) error {
 	if err := PointToHostDockerDaemon(); err != nil {
 		return errors.Wrap(err, "point host docker-daemon")
 	}
-	cmd := exec.Command(Docker, "volume", "create", name, "--label", "name.minikube.sigs.k8s.io="+name, "--label", "craeted_by_minikube.minikube.sigs.k8s.io=true")
+	cmd := exec.Command(Docker, "volume", "create", name, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, name), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "output %s", string(out))
 	}