Skip to content

Commit 3336c24

Browse files
author
dlorenc
committed
Stop using insecure serving.
1 parent 45eabf6 commit 3336c24

18 files changed

+212
-79
lines changed

‎deploy/addons/addon-manager.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ spec:
2626
containers:
2727
- name: kube-addon-manager
2828
image: gcr.io/google-containers/kube-addon-manager:v6.4-beta.2
29+
env:
30+
- name: KUBECONFIG
31+
value: /var/lib/localkube/kubeconfig
2932
imagePullPolicy: IfNotPresent
3033
resources:
3134
requests:
@@ -35,7 +38,13 @@ spec:
3538
- mountPath: /etc/kubernetes/
3639
name: addons
3740
readOnly: true
41+
- mountPath: /var/lib/localkube
42+
name: kubeconfig
43+
readOnly: true
3844
volumes:
3945
- hostPath:
4046
path: /etc/kubernetes/
4147
name: addons
48+
- hostPath:
49+
path: /var/lib/localkube
50+
name: kubeconfig

‎hack/jenkins/common.sh

+3
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ ${SUDO_PREFIX}out/e2e-${OS_ARCH} -minikube-args="--vm-driver=${VM_DRIVER} --v=10
7575
result=$?
7676
set -e
7777

78+
# See the KUBECONFIG file for debugging
79+
sudo cat $KUBECONFIG
80+
7881
MINIKUBE_WANTREPORTERRORPROMPT=False sudo ./out/minikube-${OS_ARCH} delete \
7982
|| MINIKUBE_WANTREPORTERRORPROMPT=False ./out/minikube-${OS_ARCH} delete \
8083
|| true

‎hack/jenkins/linux_integration_tests_none.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ OS_ARCH="linux-amd64"
3030
VM_DRIVER="none"
3131
JOB_NAME="Linux-None"
3232
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --use-vendored-driver"
33-
SUDO_PREFIX="sudo "
33+
SUDO_PREFIX="sudo -E "
34+
export KUBECONFIG="/root/.kube/config"
3435

3536
# Download files and set permissions
3637
source common.sh

‎hack/jenkins/print-debug-info.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
# the cluster state.
2121
set +e
2222

23+
env
24+
${SUDO_PREFIX} cat $KUBECONFIG
25+
2326
kubectl get pods --all-namespaces
2427
kubectl cluster-info dump
2528

2629
# For the none driver
27-
journalctl -u localkube
30+
journalctl -u localkube -n 500
2831
${SUDO_PREFIX}cat $KUBECONFIG
2932

3033
cat $HOME/.kube/config

‎pkg/localkube/apiserver.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ func StartAPIServer(lk LocalkubeServer) func() error {
4141
config.SecureServing.BindAddress = lk.APIServerAddress
4242
config.SecureServing.BindPort = lk.APIServerPort
4343

44-
config.InsecureServing.BindAddress = lk.APIServerInsecureAddress
45-
config.InsecureServing.BindPort = lk.APIServerInsecurePort
44+
// 0 turns off insecure serving.
45+
config.InsecureServing.BindPort = 0
4646

4747
config.Authentication.ClientCert.ClientCA = lk.GetCAPublicKeyCertPath()
4848

@@ -86,7 +86,7 @@ func StartAPIServer(lk LocalkubeServer) func() error {
8686
}
8787

8888
func readyFunc(lk LocalkubeServer) HealthCheck {
89-
hostport := net.JoinHostPort(lk.APIServerInsecureAddress.String(), strconv.Itoa(lk.APIServerInsecurePort))
90-
addr := "http://" + path.Join(hostport, "healthz")
91-
return healthCheck(addr)
89+
hostport := net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerPort))
90+
addr := "https://" + path.Join(hostport, "healthz")
91+
return healthCheck(addr, lk)
9292
}

‎pkg/localkube/controller-manager.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package localkube
1919
import (
2020
controllerManager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
2121
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
22+
"k8s.io/minikube/pkg/util"
2223
)
2324

2425
func (lk LocalkubeServer) NewControllerManagerServer() Server {
@@ -28,7 +29,7 @@ func (lk LocalkubeServer) NewControllerManagerServer() Server {
2829
func StartControllerManagerServer(lk LocalkubeServer) func() error {
2930
config := options.NewCMServer()
3031

31-
config.Master = lk.GetAPIServerInsecureURL()
32+
config.Kubeconfig = util.DefaultKubeConfigPath
3233

3334
// defaults from command
3435
config.DeletingPodsQps = 0.1

‎pkg/localkube/kubelet.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ limitations under the License.
1717
package localkube
1818

1919
import (
20+
"k8s.io/apiserver/pkg/util/flag"
2021
kubelet "k8s.io/kubernetes/cmd/kubelet/app"
2122
"k8s.io/kubernetes/cmd/kubelet/app/options"
23+
"k8s.io/minikube/pkg/util"
2224
)
2325

2426
func (lk LocalkubeServer) NewKubeletServer() Server {
@@ -29,7 +31,8 @@ func StartKubeletServer(lk LocalkubeServer) func() error {
2931
config := options.NewKubeletServer()
3032

3133
// Master details
32-
config.APIServerList = []string{lk.GetAPIServerInsecureURL()}
34+
config.KubeConfig = flag.NewStringFlag(util.DefaultKubeConfigPath)
35+
config.RequireKubeConfig = true
3336

3437
// Set containerized based on the flag
3538
config.Containerized = lk.Containerized

‎pkg/localkube/proxy.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package localkube
1818

1919
import (
2020
kubeproxy "k8s.io/kubernetes/cmd/kube-proxy/app"
21+
"k8s.io/minikube/pkg/util"
2122

2223
"time"
2324

@@ -40,16 +41,17 @@ func StartProxyServer(lk LocalkubeServer) func() error {
4041
config := &componentconfig.KubeProxyConfiguration{
4142
OOMScoreAdj: &OOMScoreAdj,
4243
ClientConnection: componentconfig.ClientConnectionConfiguration{
43-
Burst: 10,
44-
QPS: 5,
44+
Burst: 10,
45+
QPS: 5,
46+
KubeConfigFile: util.DefaultKubeConfigPath,
4547
},
4648
ConfigSyncPeriod: v1.Duration{Duration: 15 * time.Minute},
4749
IPTables: componentconfig.KubeProxyIPTablesConfiguration{
4850
MasqueradeBit: &MasqueradeBit,
4951
SyncPeriod: v1.Duration{Duration: 30 * time.Second},
5052
MinSyncPeriod: v1.Duration{Duration: 5 * time.Second},
5153
},
52-
BindAddress: lk.APIServerInsecureAddress.String(),
54+
BindAddress: lk.APIServerAddress.String(),
5355
Mode: componentconfig.ProxyModeIPTables,
5456
FeatureGates: lk.FeatureGates,
5557
// Disable the healthz check
@@ -60,7 +62,7 @@ func StartProxyServer(lk LocalkubeServer) func() error {
6062

6163
return func() error {
6264
// Creating this config requires the API Server to be up, so do it in the start function itself.
63-
server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), lk.GetAPIServerInsecureURL())
65+
server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), "")
6466
if err != nil {
6567
panic(err)
6668
}

‎pkg/localkube/ready.go

+27-2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ limitations under the License.
1717
package localkube
1818

1919
import (
20+
"crypto/tls"
21+
"crypto/x509"
2022
"io/ioutil"
2123
"net/http"
2224

@@ -25,10 +27,33 @@ import (
2527

2628
type HealthCheck func() bool
2729

28-
func healthCheck(addr string) HealthCheck {
30+
func healthCheck(addr string, lk LocalkubeServer) HealthCheck {
2931
return func() bool {
3032
glog.Infof("Performing healthcheck on %s\n", addr)
31-
resp, err := http.Get(addr)
33+
34+
cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
35+
if err != nil {
36+
glog.Error(err)
37+
return false
38+
}
39+
40+
// Load CA cert
41+
caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
42+
if err != nil {
43+
glog.Warning(err)
44+
return false
45+
}
46+
caCertPool := x509.NewCertPool()
47+
caCertPool.AppendCertsFromPEM(caCert)
48+
tlsConfig := &tls.Config{
49+
Certificates: []tls.Certificate{cert},
50+
RootCAs: caCertPool,
51+
}
52+
tlsConfig.BuildNameToCertificate()
53+
transport := &http.Transport{TLSClientConfig: tlsConfig}
54+
client := &http.Client{Transport: transport}
55+
56+
resp, err := client.Get(addr)
3257
if err != nil {
3358
glog.Errorf("Error performing healthcheck: %s", err)
3459
return false

‎pkg/localkube/ready_test.go

+36-4
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,21 @@ limitations under the License.
1717
package localkube
1818

1919
import (
20+
"crypto/tls"
21+
"crypto/x509"
2022
"io"
23+
"io/ioutil"
2124
"net/http"
2225
"net/http/httptest"
26+
"os"
2327
"testing"
28+
29+
"k8s.io/minikube/pkg/minikube/tests"
2430
)
2531

2632
func TestBasicHealthCheck(t *testing.T) {
2733

28-
tests := []struct {
34+
tcs := []struct {
2935
body string
3036
statusCode int
3137
shouldSucceed bool
@@ -34,17 +40,43 @@ func TestBasicHealthCheck(t *testing.T) {
3440
{"notok", 200, false},
3541
}
3642

37-
for _, tc := range tests {
43+
tempDir := tests.MakeTempDir()
44+
defer os.RemoveAll(tempDir)
45+
lk := LocalkubeServer{LocalkubeDirectory: tempDir}
46+
lk.GenerateCerts()
47+
48+
cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
49+
if err != nil {
50+
t.Fatalf("Unable to load server certs.")
51+
}
52+
53+
caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
54+
if err != nil {
55+
t.Fatalf("Unable to load CA certs.")
56+
}
57+
caCertPool := x509.NewCertPool()
58+
caCertPool.AppendCertsFromPEM(caCert)
59+
60+
tls := tls.Config{
61+
Certificates: []tls.Certificate{cert},
62+
ClientCAs: caCertPool,
63+
}
64+
65+
tls.BuildNameToCertificate()
66+
67+
for _, tc := range tcs {
3868
// Do this in a func so we can use defer.
3969
doTest := func() {
4070
handler := func(w http.ResponseWriter, r *http.Request) {
4171
w.WriteHeader(tc.statusCode)
4272
io.WriteString(w, tc.body)
4373
}
44-
server := httptest.NewServer(http.HandlerFunc(handler))
74+
server := httptest.NewUnstartedServer(http.HandlerFunc(handler))
4575
defer server.Close()
76+
server.TLS = &tls
77+
server.StartTLS()
4678

47-
hcFunc := healthCheck(server.URL)
79+
hcFunc := healthCheck(server.URL, lk)
4880
result := hcFunc()
4981
if result != tc.shouldSucceed {
5082
t.Errorf("Expected healthcheck to return %v. Got %v", result, tc.shouldSucceed)

‎pkg/localkube/scheduler.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package localkube
1919
import (
2020
scheduler "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
2121
"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
22+
"k8s.io/minikube/pkg/util"
2223
)
2324

2425
func (lk LocalkubeServer) NewSchedulerServer() Server {
@@ -29,7 +30,7 @@ func StartSchedulerServer(lk LocalkubeServer) func() error {
2930
config := options.NewSchedulerServer()
3031

3132
// master details
32-
config.Master = lk.GetAPIServerInsecureURL()
33+
config.Kubeconfig = util.DefaultKubeConfigPath
3334

3435
// defaults from command
3536
config.EnableProfiling = true

‎pkg/localkube/storage_provisioner.go

+7-8
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@ package localkube
1919
import (
2020
"errors"
2121
"fmt"
22-
"net"
2322
"os"
2423
"path"
25-
"strconv"
2624
"time"
2725

2826
"github.com/golang/glog"
@@ -34,7 +32,8 @@ import (
3432
"k8s.io/apimachinery/pkg/util/wait"
3533
"k8s.io/client-go/kubernetes"
3634
"k8s.io/client-go/pkg/api/v1"
37-
"k8s.io/client-go/rest"
35+
"k8s.io/client-go/tools/clientcmd"
36+
"k8s.io/minikube/pkg/util"
3837
)
3938

4039
const (
@@ -123,12 +122,12 @@ func (lk LocalkubeServer) NewStorageProvisionerServer() Server {
123122

124123
func StartStorageProvisioner(lk LocalkubeServer) func() error {
125124

126-
// Create an InClusterConfig and use it to create a client for the controller
127-
// to use to communicate with Kubernetes
128-
config := rest.Config{Host: net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerInsecurePort))}
129125
return func() error {
130-
131-
clientset, err := kubernetes.NewForConfig(&config)
126+
config, err := clientcmd.BuildConfigFromFlags("", util.DefaultKubeConfigPath)
127+
if err != nil {
128+
return err
129+
}
130+
clientset, err := kubernetes.NewForConfig(config)
132131
if err != nil {
133132
glog.Fatalf("Failed to create client: %v", err)
134133
}

0 commit comments

Comments
 (0)