Skip to content

Commit 312bec6

Browse files
r2d4r2d4
committed
Implement extra-config for kubeadm kubelet
1 parent 93323c6 commit 312bec6

File tree

6 files changed

+458
-60
lines changed

6 files changed

+458
-60
lines changed

‎Makefile

+4
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,10 @@ endif
150150
test-iso:
151151
go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
152152

153+
.PHONY: test-pkg
154+
test-pkg/%:
155+
go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)"
156+
153157
.PHONY: integration
154158
integration: out/minikube
155159
go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS)

‎pkg/minikube/bootstrapper/kubeadm/kubeadm.go

+42-60
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"bytes"
2121
"crypto"
2222
"fmt"
23-
"html/template"
2423
"os"
2524
"path/filepath"
2625
"strings"
@@ -44,48 +43,6 @@ type KubeadmBootstrapper struct {
4443
c bootstrapper.CommandRunner
4544
}
4645

47-
// TODO(r2d4): template this with bootstrapper.KubernetesConfig
48-
const kubeletSystemdConf = `
49-
[Service]
50-
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
51-
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
52-
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
53-
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
54-
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
55-
ExecStart=
56-
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
57-
`
58-
59-
const kubeletService = `
60-
[Unit]
61-
Description=kubelet: The Kubernetes Node Agent
62-
Documentation=http://kubernetes.io/docs/
63-
64-
[Service]
65-
ExecStart=/usr/bin/kubelet
66-
Restart=always
67-
StartLimitInterval=0
68-
RestartSec=10
69-
70-
[Install]
71-
WantedBy=multi-user.target
72-
`
73-
74-
const kubeadmConfigTmpl = `
75-
apiVersion: kubeadm.k8s.io/v1alpha1
76-
kind: MasterConfiguration
77-
api:
78-
advertiseAddress: {{.AdvertiseAddress}}
79-
bindPort: {{.APIServerPort}}
80-
kubernetesVersion: {{.KubernetesVersion}}
81-
certificatesDir: {{.CertDir}}
82-
networking:
83-
serviceSubnet: {{.ServiceCIDR}}
84-
etcd:
85-
dataDir: {{.EtcdDataDir}}
86-
nodeName: {{.NodeName}}
87-
`
88-
8946
func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
9047
h, err := api.Load(config.GetMachineName())
9148
if err != nil {
@@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) {
147104
func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error {
148105
// We use --skip-preflight-checks since we have our own custom addons
149106
// that we also stick in /etc/kubernetes/manifests
150-
kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"
151-
t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl))
152107
b := bytes.Buffer{}
153-
if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
108+
if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
154109
return err
155110
}
156111

@@ -197,22 +152,14 @@ func addAddons(files *[]assets.CopyableFile) error {
197152
}
198153

199154
func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error {
200-
restoreTmpl := `
201-
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
202-
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
203-
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
204-
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
205-
`
206-
t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl))
207-
208155
opts := struct {
209156
KubeadmConfigFile string
210157
}{
211158
KubeadmConfigFile: constants.KubeadmConfigFile,
212159
}
213160

214161
b := bytes.Buffer{}
215-
if err := t.Execute(&b, opts); err != nil {
162+
if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil {
216163
return err
217164
}
218165

@@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
231178
return bootstrapper.SetupCerts(k.c, k8s)
232179
}
233180

181+
func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
182+
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
183+
if err != nil {
184+
return "", errors.Wrap(err, "parsing kubernetes version")
185+
}
186+
187+
extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
188+
if err != nil {
189+
return "", errors.Wrap(err, "generating extra configuration for kubelet")
190+
}
191+
192+
extraFlags := convertToFlags(extraOpts)
193+
b := bytes.Buffer{}
194+
if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
195+
return "", err
196+
}
197+
198+
return b.String(), nil
199+
}
200+
234201
func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error {
235202
if cfg.ShouldLoadCachedImages {
236203
// Make best effort to load any cached images
237204
go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir)
238205
}
239-
kubeadmCfg, err := k.generateConfig(cfg)
206+
kubeadmCfg, err := generateConfig(cfg)
240207
if err != nil {
241208
return errors.Wrap(err, "generating kubeadm cfg")
242209
}
243210

211+
kubeletCfg, err := NewKubeletConfig(cfg)
212+
if err != nil {
213+
return errors.Wrap(err, "generating kubelet config")
214+
}
215+
244216
files := []assets.CopyableFile{
245217
assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
246-
assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"),
218+
assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
247219
assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"),
248220
}
249221

@@ -290,9 +262,17 @@ sudo systemctl start kubelet
290262
return nil
291263
}
292264

293-
func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
294-
t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl))
265+
func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
266+
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
267+
if err != nil {
268+
return "", errors.Wrap(err, "parsing kubernetes version")
269+
}
295270

271+
// generates a map of component to extra args for apiserver, controller-manager, and scheduler
272+
extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
273+
if err != nil {
274+
return "", errors.Wrap(err, "generating extra component config for kubeadm")
275+
}
296276
opts := struct {
297277
CertDir string
298278
ServiceCIDR string
@@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
301281
KubernetesVersion string
302282
EtcdDataDir string
303283
NodeName string
284+
ExtraArgs []ComponentExtraArgs
304285
}{
305286
CertDir: util.DefaultCertPath,
306287
ServiceCIDR: util.DefaultInsecureRegistry,
@@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
309290
KubernetesVersion: k8s.KubernetesVersion,
310291
EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted
311292
NodeName: k8s.NodeName,
293+
ExtraArgs: extraComponentConfig,
312294
}
313295

314296
b := bytes.Buffer{}
315-
if err := t.Execute(&b, opts); err != nil {
297+
if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
316298
return "", err
317299
}
318300

‎pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package kubeadm
2+
3+
import (
4+
"testing"
5+
6+
"k8s.io/minikube/pkg/minikube/bootstrapper"
7+
"k8s.io/minikube/pkg/util"
8+
)
9+
10+
func TestGenerateConfig(t *testing.T) {
11+
tests := []struct {
12+
description string
13+
cfg bootstrapper.KubernetesConfig
14+
expectedCfg string
15+
shouldErr bool
16+
}{
17+
{
18+
description: "no extra args",
19+
cfg: bootstrapper.KubernetesConfig{
20+
NodeIP: "192.168.1.100",
21+
KubernetesVersion: "v1.8.0",
22+
NodeName: "minikube",
23+
},
24+
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
25+
kind: MasterConfiguration
26+
api:
27+
advertiseAddress: 192.168.1.100
28+
bindPort: 8443
29+
kubernetesVersion: v1.8.0
30+
certificatesDir: /var/lib/localkube/certs/
31+
networking:
32+
serviceSubnet: 10.0.0.0/24
33+
etcd:
34+
dataDir: /data
35+
nodeName: minikube
36+
`,
37+
},
38+
{
39+
description: "extra args all components",
40+
cfg: bootstrapper.KubernetesConfig{
41+
NodeIP: "192.168.1.101",
42+
KubernetesVersion: "v1.8.0-alpha.0",
43+
NodeName: "extra-args-minikube",
44+
ExtraOptions: util.ExtraOptionSlice{
45+
util.ExtraOption{
46+
Component: Apiserver,
47+
Key: "fail-no-swap",
48+
Value: "true",
49+
},
50+
util.ExtraOption{
51+
Component: ControllerManager,
52+
Key: "kube-api-burst",
53+
Value: "32",
54+
},
55+
util.ExtraOption{
56+
Component: Scheduler,
57+
Key: "scheduler-name",
58+
Value: "mini-scheduler",
59+
},
60+
},
61+
},
62+
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
63+
kind: MasterConfiguration
64+
api:
65+
advertiseAddress: 192.168.1.101
66+
bindPort: 8443
67+
kubernetesVersion: v1.8.0-alpha.0
68+
certificatesDir: /var/lib/localkube/certs/
69+
networking:
70+
serviceSubnet: 10.0.0.0/24
71+
etcd:
72+
dataDir: /data
73+
nodeName: extra-args-minikube
74+
apiServerExtraArgs:
75+
fail-no-swap: true
76+
controllerManagerExtraArgs:
77+
kube-api-burst: 32
78+
schedulerExtraArgs:
79+
scheduler-name: mini-scheduler
80+
`,
81+
},
82+
{
83+
// Unknown components should fail silently
84+
description: "unknown component",
85+
cfg: bootstrapper.KubernetesConfig{
86+
NodeIP: "192.168.1.101",
87+
KubernetesVersion: "v1.8.0-alpha.0",
88+
NodeName: "extra-args-minikube",
89+
ExtraOptions: util.ExtraOptionSlice{
90+
util.ExtraOption{
91+
Component: "not-a-real-component",
92+
Key: "killswitch",
93+
Value: "true",
94+
},
95+
},
96+
},
97+
shouldErr: true,
98+
},
99+
}
100+
101+
for _, test := range tests {
102+
t.Run(test.description, func(t *testing.T) {
103+
actualCfg, err := generateConfig(test.cfg)
104+
if err != nil && !test.shouldErr {
105+
t.Errorf("got unexpected error generating config: %s", err)
106+
return
107+
}
108+
if err == nil && test.shouldErr {
109+
t.Errorf("expected error but got none, config: %s", actualCfg)
110+
return
111+
}
112+
if actualCfg != test.expectedCfg {
113+
t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg)
114+
return
115+
}
116+
})
117+
}
118+
}

‎pkg/minikube/bootstrapper/kubeadm/templates.go

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package kubeadm
2+
3+
import "html/template"
4+
5+
var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
6+
kind: MasterConfiguration
7+
api:
8+
advertiseAddress: {{.AdvertiseAddress}}
9+
bindPort: {{.APIServerPort}}
10+
kubernetesVersion: {{.KubernetesVersion}}
11+
certificatesDir: {{.CertDir}}
12+
networking:
13+
serviceSubnet: {{.ServiceCIDR}}
14+
etcd:
15+
dataDir: {{.EtcdDataDir}}
16+
nodeName: {{.NodeName}}
17+
{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
18+
{{$key}}: {{$value}}
19+
{{end}}{{end}}`))
20+
21+
var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
22+
[Service]
23+
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
24+
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
25+
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
26+
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
27+
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
28+
ExecStart=
29+
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
30+
`))
31+
32+
const kubeletService = `
33+
[Unit]
34+
Description=kubelet: The Kubernetes Node Agent
35+
Documentation=http://kubernetes.io/docs/
36+
37+
[Service]
38+
ExecStart=/usr/bin/kubelet
39+
Restart=always
40+
StartLimitInterval=0
41+
RestartSec=10
42+
43+
[Install]
44+
WantedBy=multi-user.target
45+
`
46+
47+
var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(`
48+
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
49+
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
50+
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
51+
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
52+
`))
53+
54+
var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))

0 commit comments

Comments
 (0)