Merge pull request #6487 from medyagh/fix_denv

fix docker-env for kic drivers

Makefile

@@ -500,8 +500,8 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.4-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.4-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.5-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.5-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 

hack/images/kicbase.Dockerfile

@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y \
   && apt-get clean -y 
 # disable containerd by default
 RUN systemctl disable containerd
+RUN rm /etc/crictl.yaml
 # enable docker which is default
 RUN systemctl enable docker
 # making SSH work for docker container 

pkg/drivers/kic/types.go

@@ -26,7 +26,7 @@ const (
 	// DefaultBindIPV4 is The default IP the container will bind to.
 	DefaultBindIPV4 = "127.0.0.1"
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.4@sha256:01552b7ffc7cb0c49c6106796ed73f69647691d8fbae89fed8ee317dc7f1015a"
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.5@sha256:3ddd8461dfb5c3e452ccc44d87750b87a574ec23fc425da67dccc1f0c57d428a"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.3"

pkg/minikube/machine/client.go

@@ -36,6 +36,7 @@ import (
 	"github.com/docker/machine/libmachine/host"
 	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/persist"
+	lib_provision "github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/docker/machine/libmachine/swarm"
@@ -207,10 +208,15 @@ func (api *LocalClient) Create(h *host.Host) error {
 		{
 			"provisioning",
 			func() error {
-				if driver.BareMetal(h.Driver.DriverName()) || driver.IsKIC(h.Driver.DriverName()) {
+				if driver.BareMetal(h.Driver.DriverName()) {
 					return nil
 				}
-				pv := provision.NewBuildrootProvisioner(h.Driver)
+				var pv lib_provision.Provisioner
+				if driver.IsKIC(h.Driver.DriverName()) {
+					pv = provision.NewUbuntuProvisioner(h.Driver)
+				} else {
+					pv = provision.NewBuildrootProvisioner(h.Driver)
+				}
 				return pv.Provision(*h.HostOptions.SwarmOptions, *h.HostOptions.AuthOptions, *h.HostOptions.EngineOptions)
 			},
 		},

pkg/provision/buildroot.go

@@ -19,29 +19,18 @@ package provision
 import (
 	"bytes"
 	"fmt"
-	"os/exec"
 	"path"
-	"path/filepath"
-	"strings"
 	"text/template"
 	"time"
 
 	"github.com/docker/machine/libmachine/auth"
-	"github.com/docker/machine/libmachine/cert"
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/engine"
 	"github.com/docker/machine/libmachine/log"
-	"github.com/docker/machine/libmachine/mcnutils"
 	"github.com/docker/machine/libmachine/provision"
 	"github.com/docker/machine/libmachine/provision/pkgaction"
 	"github.com/docker/machine/libmachine/provision/serviceaction"
 	"github.com/docker/machine/libmachine/swarm"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/sshutil"
-	"k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -50,15 +39,6 @@ type BuildrootProvisioner struct {
 	provision.SystemdProvisioner
 }
 
-// for escaping systemd template specifiers (e.g. '%i'), which are not supported by minikube
-var systemdSpecifierEscaper = strings.NewReplacer("%", "%%")
-
-func init() {
-	provision.Register("Buildroot", &provision.RegisteredProvisioner{
-		New: NewBuildrootProvisioner,
-	})
-}
-
 // NewBuildrootProvisioner creates a new BuildrootProvisioner
 func NewBuildrootProvisioner(d drivers.Driver) provision.Provisioner {
 	return &BuildrootProvisioner{
@@ -75,17 +55,6 @@ func (p *BuildrootProvisioner) CompatibleWithHost() bool {
 	return p.OsReleaseInfo.ID == "buildroot"
 }
 
-// escapeSystemdDirectives escapes special characters in the input variables used to create the
-// systemd unit file, which would otherwise be interpreted as systemd directives. An example
-// are template specifiers (e.g. '%i') which are predefined variables that get evaluated dynamically
-// (see systemd man pages for more info). This is not supported by minikube, thus needs to be escaped.
-func escapeSystemdDirectives(engineConfigContext *provision.EngineConfigContext) {
-	// escape '%' in Environment option so that it does not evaluate into a template specifier
-	engineConfigContext.EngineOptions.Env = util.ReplaceChars(engineConfigContext.EngineOptions.Env, systemdSpecifierEscaper)
-	// input might contain whitespaces, wrap it in quotes
-	engineConfigContext.EngineOptions.Env = util.ConcatStrings(engineConfigContext.EngineOptions.Env, "\"", "\"")
-}
-
 // GenerateDockerOptions generates the *provision.DockerOptions for this provisioner
 func (p *BuildrootProvisioner) GenerateDockerOptions(dockerPort int) (*provision.DockerOptions, error) {
 	var engineCfg bytes.Buffer
@@ -192,14 +161,6 @@ WantedBy=multi-user.target
 	return dockerCfg, nil
 }
 
-func rootFileSystemType(p *BuildrootProvisioner) (string, error) {
-	fs, err := p.SSHCommand("df --output=fstype / | tail -n 1")
-	if err != nil {
-		return "", err
-	}
-	return strings.TrimSpace(fs), nil
-}
-
 // Package installs a package
 func (p *BuildrootProvisioner) Package(name string, action pkgaction.PackageAction) error {
 	return nil
@@ -235,176 +196,10 @@ func (p *BuildrootProvisioner) Provision(swarmOptions swarm.Options, authOptions
 	}
 
 	log.Debugf("setting minikube options for container-runtime")
-	if err := setContainerRuntimeOptions(p); err != nil {
+	if err := setContainerRuntimeOptions(p.Driver.GetMachineName(), p); err != nil {
 		log.Debugf("Error setting container-runtime options during provisioning %v", err)
 		return err
 	}
 
 	return nil
 }
-
-func setRemoteAuthOptions(p provision.Provisioner) auth.Options {
-	dockerDir := p.GetDockerOptionsDir()
-	authOptions := p.GetAuthOptions()
-
-	// due to windows clients, we cannot use filepath.Join as the paths
-	// will be mucked on the linux hosts
-	authOptions.CaCertRemotePath = path.Join(dockerDir, "ca.pem")
-	authOptions.ServerCertRemotePath = path.Join(dockerDir, "server.pem")
-	authOptions.ServerKeyRemotePath = path.Join(dockerDir, "server-key.pem")
-
-	return authOptions
-}
-
-func setContainerRuntimeOptions(p *BuildrootProvisioner) error {
-	c, err := config.Load(p.Driver.GetMachineName())
-	if err != nil {
-		return errors.Wrap(err, "getting cluster config")
-	}
-
-	switch c.KubernetesConfig.ContainerRuntime {
-	case "crio", "cri-o":
-		return p.setCrioOptions()
-	case "containerd":
-		return nil
-	default:
-		_, err := p.GenerateDockerOptions(engine.DefaultPort)
-		return err
-	}
-}
-
-func (p *BuildrootProvisioner) setCrioOptions() error {
-	// pass through --insecure-registry
-	var (
-		crioOptsTmpl = `
-CRIO_MINIKUBE_OPTIONS='{{ range .EngineOptions.InsecureRegistry }}--insecure-registry {{.}} {{ end }}'
-`
-		crioOptsPath = "/etc/sysconfig/crio.minikube"
-	)
-	t, err := template.New("crioOpts").Parse(crioOptsTmpl)
-	if err != nil {
-		return err
-	}
-	var crioOptsBuf bytes.Buffer
-	if err := t.Execute(&crioOptsBuf, p); err != nil {
-		return err
-	}
-
-	if _, err = p.SSHCommand(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(crioOptsPath), crioOptsBuf.String(), crioOptsPath)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func configureAuth(p *BuildrootProvisioner) error {
-	log.Infof("configureAuth start")
-	start := time.Now()
-	defer func() {
-		log.Infof("configureAuth took %s", time.Since(start))
-	}()
-
-	driver := p.GetDriver()
-	machineName := driver.GetMachineName()
-	authOptions := p.GetAuthOptions()
-	org := mcnutils.GetUsername() + "." + machineName
-	bits := 2048
-
-	ip, err := driver.GetIP()
-	if err != nil {
-		return errors.Wrap(err, "error getting ip during provisioning")
-	}
-
-	if err := copyHostCerts(authOptions); err != nil {
-		return err
-	}
-
-	// The Host IP is always added to the certificate's SANs list
-	hosts := append(authOptions.ServerCertSANs, ip, "localhost")
-	log.Debugf("generating server cert: %s ca-key=%s private-key=%s org=%s san=%s",
-		authOptions.ServerCertPath,
-		authOptions.CaCertPath,
-		authOptions.CaPrivateKeyPath,
-		org,
-		hosts,
-	)
-
-	err = cert.GenerateCert(&cert.Options{
-		Hosts:     hosts,
-		CertFile:  authOptions.ServerCertPath,
-		KeyFile:   authOptions.ServerKeyPath,
-		CAFile:    authOptions.CaCertPath,
-		CAKeyFile: authOptions.CaPrivateKeyPath,
-		Org:       org,
-		Bits:      bits,
-	})
-
-	if err != nil {
-		return fmt.Errorf("error generating server cert: %v", err)
-	}
-
-	return copyRemoteCerts(authOptions, driver)
-}
-
-func copyHostCerts(authOptions auth.Options) error {
-	log.Infof("copyHostCerts")
-	execRunner := &command.ExecRunner{}
-	hostCerts := map[string]string{
-		authOptions.CaCertPath:     path.Join(authOptions.StorePath, "ca.pem"),
-		authOptions.ClientCertPath: path.Join(authOptions.StorePath, "cert.pem"),
-		authOptions.ClientKeyPath:  path.Join(authOptions.StorePath, "key.pem"),
-	}
-
-	if _, err := execRunner.RunCmd(exec.Command("mkdir", "-p", authOptions.StorePath)); err != nil {
-		return err
-	}
-	for src, dst := range hostCerts {
-		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0777")
-		if err != nil {
-			return errors.Wrapf(err, "open cert file: %s", src)
-		}
-		if err := execRunner.Copy(f); err != nil {
-			return errors.Wrapf(err, "transferring file: %+v", f)
-		}
-	}
-
-	return nil
-}
-
-func copyRemoteCerts(authOptions auth.Options, driver drivers.Driver) error {
-	log.Infof("copyRemoteCerts")
-
-	remoteCerts := map[string]string{
-		authOptions.CaCertPath:     authOptions.CaCertRemotePath,
-		authOptions.ServerCertPath: authOptions.ServerCertRemotePath,
-		authOptions.ServerKeyPath:  authOptions.ServerKeyRemotePath,
-	}
-
-	sshClient, err := sshutil.NewSSHClient(driver)
-	if err != nil {
-		return errors.Wrap(err, "provisioning: error getting ssh client")
-	}
-	sshRunner := command.NewSSHRunner(sshClient)
-
-	dirs := []string{}
-	for _, dst := range remoteCerts {
-		dirs = append(dirs, path.Dir(dst))
-	}
-
-	args := append([]string{"mkdir", "-p"}, dirs...)
-	if _, err = sshRunner.RunCmd(exec.Command("sudo", args...)); err != nil {
-		return err
-	}
-
-	for src, dst := range remoteCerts {
-		f, err := assets.NewFileAsset(src, path.Dir(dst), filepath.Base(dst), "0640")
-		if err != nil {
-			return errors.Wrapf(err, "error copying %s to %s", src, dst)
-		}
-		if err := sshRunner.Copy(f); err != nil {
-			return errors.Wrapf(err, "transferring file to machine %v", f)
-		}
-	}
-
-	return nil
-}