Merge pull request #2137 from priyawadhwa/storage-provisioner-as-pod

Run storage provisioner as pod

Author: r2d4

Makefile

@@ -54,7 +54,7 @@ LOCALKUBE_LDFLAGS := "$(K8S_VERSION_LDFLAGS) $(MINIKUBE_LDFLAGS) -s -w -extldfla
 LOCALKUBEFILES := GOPATH=$(GOPATH) go list  -f '{{join .Deps "\n"}}' ./cmd/localkube/ | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}'
 MINIKUBEFILES := GOPATH=$(GOPATH) go list  -f '{{join .Deps "\n"}}' ./cmd/minikube/ | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}'
 HYPERKIT_FILES :=  GOPATH=$(GOPATH) go list  -f '{{join .Deps "\n"}}' k8s.io/minikube/cmd/drivers/hyperkit | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}'
-
+STORAGE_PROVISIONER_FILES := GOPATH=$(GOPATH) go list -f '{{join .Deps "\n"}}' k8s.io/minikube/cmd/storage-provisioner | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}'
 MINIKUBE_TEST_FILES := go list -f '{{ if .TestGoFiles }} {{.ImportPath}} {{end}}' ./... | grep k8s.io | GOPATH=$(GOPATH) xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}'
 
 KVM_DRIVER_FILES := $(shell go list -f '{{join .Deps "\n"}}' ./cmd/drivers/kvm/ | grep k8s.io | xargs go list -f '{{ range $$file := .GoFiles }} {{$$.Dir}}/{{$$file}}{{"\n"}}{{end}}')
@@ -300,6 +300,14 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	@echo ""
 	@echo "$(@) successfully built"
 
+out/storage-provisioner: $(shell $(STORAGE_PROVISIONER_FILES))
+	go build -o $(BUILD_DIR)/storage-provisioner cmd/storage-provisioner/main.go
+
+.PHONY: storage-provisioner-image
+storage-provisioner-image: out/storage-provisioner
+	docker build -t $(REGISTRY)/storage-provisioner:$(TAG) -f deploy/storage-provisioner/Dockerfile .
+	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(TAG)
+
 .PHONY: release-iso
 release-iso: minikube_iso checksum
 	gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso

cmd/localkube/cmd/start.go

@@ -139,6 +139,4 @@ func SetupServer(s *localkube.LocalkubeServer) {
 	proxy := s.NewProxyServer()
 	s.AddServer(proxy)
 
-	storageProvisioner := s.NewStorageProvisionerServer()
-	s.AddServer(storageProvisioner)
 }

cmd/storage-provisioner/main.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/localkube"
+)
+
+func main() {
+	flag.Parse()
+
+	if err := localkube.StartStorageProvisioner(); err != nil {
+		glog.Exit(err)
+	}
+
+}

deploy/addons/storage-provisioner/storage-provisioner.yaml

@@ -0,0 +1,28 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: storage-provisioner
+  namespace: kube-system
+  labels:
+    integration-test: storage-provisioner
+    addonmanager.kubernetes.io/mode: EnsureExists
+spec:
+  hostNetwork: true
+  containers:
+  - name: storage-provisioner
+    image: gcr.io/k8s-minikube/storage-provisioner:v1.8.0
+    imagePullPolicy: IfNotPresent

deploy/storage-provisioner/Dockerfile

@@ -0,0 +1,17 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM scratch
+COPY out/storage-provisioner storage-provisioner
+CMD ["/storage-provisioner"]

pkg/localkube/storage_provisioner.go

@@ -30,8 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/minikube/pkg/util"
+	restclient "k8s.io/client-go/rest"
 )
 
 const provisionerName = "k8s.io/minikube-hostpath"
@@ -110,38 +109,33 @@ func (p *hostPathProvisioner) Delete(volume *v1.PersistentVolume) error {
 	return nil
 }
 
-func (lk LocalkubeServer) NewStorageProvisionerServer() Server {
-	return NewSimpleServer("storage-provisioner", serverInterval, StartStorageProvisioner(lk), noop)
-}
+// Start storage provisioner server
+func StartStorageProvisioner() error {
+	config, err := restclient.InClusterConfig()
+	if err != nil {
+		return err
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		glog.Fatalf("Failed to create client: %v", err)
+	}
 
-func StartStorageProvisioner(lk LocalkubeServer) func() error {
-
-	return func() error {
-		config, err := clientcmd.BuildConfigFromFlags("", util.DefaultKubeConfigPath)
-		if err != nil {
-			return err
-		}
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			glog.Fatalf("Failed to create client: %v", err)
-		}
-
-		// The controller needs to know what the server version is because out-of-tree
-		// provisioners aren't officially supported until 1.5
-		serverVersion, err := clientset.Discovery().ServerVersion()
-		if err != nil {
-			return fmt.Errorf("Error getting server version: %v", err)
-		}
-
-		// Create the provisioner: it implements the Provisioner interface expected by
-		// the controller
-		hostPathProvisioner := NewHostPathProvisioner()
-
-		// Start the provision controller which will dynamically provision hostPath
-		// PVs
-		pc := controller.NewProvisionController(clientset, provisionerName, hostPathProvisioner, serverVersion.GitVersion)
-
-		pc.Run(wait.NeverStop)
-		return nil
+	// The controller needs to know what the server version is because out-of-tree
+	// provisioners aren't officially supported until 1.5
+	serverVersion, err := clientset.Discovery().ServerVersion()
+	if err != nil {
+		return fmt.Errorf("Error getting server version: %v", err)
 	}
+
+	// Create the provisioner: it implements the Provisioner interface expected by
+	// the controller
+	hostPathProvisioner := NewHostPathProvisioner()
+
+	// Start the provision controller which will dynamically provision hostPath
+	// PVs
+	pc := controller.NewProvisionController(clientset, provisionerName, hostPathProvisioner, serverVersion.GitVersion)
+
+	glog.Info("Starting storage provisioner server")
+	pc.Run(wait.NeverStop)
+	return nil
 }

pkg/minikube/assets/addons.go

@@ -82,6 +82,13 @@ var Addons = map[string]*Addon{
 			"storageclass.yaml",
 			"0640"),
 	}, true, "default-storageclass"),
+	"storage-provisioner": NewAddon([]*BinDataAsset{
+		NewBinDataAsset(
+			"deploy/addons/storage-provisioner/storage-provisioner.yaml",
+			constants.AddonsPath,
+			"storage-provisioner.yaml",
+			"0640"),
+	}, true, "storage-provisioner"),
 	"coredns": NewAddon([]*BinDataAsset{
 		NewBinDataAsset(
 			"deploy/addons/coredns/coreDNS-controller.yaml",

pkg/minikube/constants/constants.go

@@ -176,6 +176,9 @@ var LocalkubeCachedImages = []string{
 
 	// Pause
 	"gcr.io/google_containers/pause-amd64:3.0",
+
+	//Storage Provisioner
+	"gcr.io/k8s-minikube/storage-provisioner:v1.8.0",
 }
 
 func GetKubeadmCachedImages(version string) []string {
@@ -201,6 +204,9 @@ func GetKubeadmCachedImages(version string) []string {
 		"gcr.io/google_containers/kube-scheduler-amd64:" + version,
 		"gcr.io/google_containers/kube-controller-manager-amd64:" + version,
 		"gcr.io/google_containers/kube-apiserver-amd64:" + version,
+
+		//Storage Provisioner
+		"gcr.io/k8s-minikube/storage-provisioner:v1.8.0",
 	}
 }
 

test/integration/functional_test.go

@@ -35,11 +35,7 @@ func TestFunctional(t *testing.T) {
 	t.Run("Addons", testAddons)
 	t.Run("Dashboard", testDashboard)
 	t.Run("ServicesList", testServicesList)
-
-	// Don't run this test on kubeadm bootstrapper for now.
-	if !strings.Contains(*args, "--bootstrapper=kubeadm") {
-		t.Run("Provisioning", testProvisioning)
-	}
+	t.Run("Provisioning", testProvisioning)
 
 	if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") {
 		t.Run("EnvVars", testClusterEnv)

test/integration/pv_test.go

@@ -20,12 +20,15 @@ package integration
 
 import (
 	"fmt"
+	"github.com/pkg/errors"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/storage"
+	commonutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/test/integration/util"
 )
 
@@ -58,6 +61,25 @@ func testProvisioning(t *testing.T) {
 		t.Fatalf("No default storage class: %s", err)
 	}
 
+	// Check that the storage provisioner pod is running
+
+	checkPodRunning := func() error {
+		client, err := commonutil.GetClient()
+		if err != nil {
+			return errors.Wrap(err, "getting kubernetes client")
+		}
+		selector := labels.SelectorFromSet(labels.Set(map[string]string{"integration-test": "storage-provisioner"}))
+
+		if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if err := checkPodRunning(); err != nil {
+		t.Fatal("Check storage-provisioner pod running failed with error: ", err)
+	}
+
 	// Now create the PVC
 	pvcPath := filepath.Join(*testdataDir, "pvc.yaml")
 	if _, err := kubectlRunner.RunCommand([]string{"create", "-f", pvcPath}); err != nil {