add support for kubelet in standalone mode and TLS auth

README.md

@@ -303,8 +303,12 @@ The following settings can be optionally set to customize the node labels and ta
 
 The following settings are optional and allow you to further configure your cluster.
 * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains.  Defaults to `cluster.local`.
+* `settings.kubernetes.standalone-mode`: Whether to run the kubelet in standalone mode, without connecting to an API server.  Defaults to `false`.
+* `settings.kubernetes.authentication-mode`: Which authentication method the kubelet should use to connect to the API server, and for incoming requests.  Defaults to `aws` for AWS variants, and `tls` for other variants.
+* `settings.kubernetes.bootstrap-token`: The token to use for [TLS bootstrapping](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/).  This is only used with the `tls` authentication mode, and is otherwise ignored.
 
 You can also optionally specify static pods for your node with the following settings.
+Static pods can be particularly useful when running in standalone mode.
 * `settings.kubernetes.static-pods.<custom identifier>.manifest`: A base64-encoded pod manifest.
 * `settings.kubernetes.static-pods.<custom identifier>.enabled`: Whether the static pod is enabled.
 

Release.toml

@@ -20,5 +20,10 @@ version = "1.0.5"
     "migrate_v1.0.5_add-proxy-restart.lz4",
     "migrate_v1.0.5_add-proxy-services.lz4"
 ]
-"(1.0.5, 1.0.6)" = ["migrate_v1.0.6_metricdog-init.lz4", "migrate_v1.0.6_add-static-pods.lz4"]
+"(1.0.5, 1.0.6)" = [
+    "migrate_v1.0.6_metricdog-init.lz4",
+    "migrate_v1.0.6_add-static-pods.lz4",
+    "migrate_v1.0.6_kubelet-standalone-tls-settings.lz4",
+    "migrate_v1.0.6_kubelet-standalone-tls-services.lz4",
+]
 

packages/kubernetes-1.15/kubelet-bootstrap-kubeconfig

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+{{~#if settings.kubernetes.api-server}}
+    certificate-authority: "/etc/kubernetes/pki/ca.crt"
+    server: "{{settings.kubernetes.api-server}}"
+{{~/if}}
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: kubelet
+  name: kubelet
+current-context: kubelet
+users:
+- name: kubelet
+{{~#if settings.kubernetes.bootstrap-token}}
+  user:
+    token: "{{settings.kubernetes.bootstrap-token}}"
+{{~/if}}

packages/kubernetes-1.15/kubelet-config

@@ -1,6 +1,16 @@
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1
+{{~#if settings.kubernetes.standalone-mode}}
+address: 127.0.0.1
+authentication:
+  anonymous:
+    enabled: true
+  webhook:
+    enabled: false
+authorization:
+  mode: AlwaysAllow
+{{~else}}
 address: 0.0.0.0
 authentication:
   anonymous:
@@ -15,6 +25,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 5m0s
     cacheUnauthorizedTTL: 30s
+{{~/if}}
 clusterDomain: {{settings.kubernetes.cluster-domain}}
 clusterDNS:
 - {{settings.kubernetes.cluster-dns-ip}}

packages/kubernetes-1.15/kubelet-exec-start-conf

@@ -0,0 +1,24 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/kubelet \
+{{~#unless settings.kubernetes.standalone-mode}}
+    --cloud-provider aws \
+    --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
+{{~#if (eq settings.kubernetes.authentication-mode "tls")}}
+    --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \
+{{~/if}}
+{{~else}}
+    --cloud-provider "" \
+{{~/unless}}
+    --config /etc/kubernetes/kubelet/config \
+    --container-runtime=remote \
+    --container-runtime-endpoint=unix:///run/dockershim.sock \
+    --containerd=/run/dockershim.sock \
+    --network-plugin cni \
+    --root-dir /var/lib/kubelet \
+    --cert-dir /var/lib/kubelet/pki \
+    --volume-plugin-dir /var/lib/kubelet/plugins/volume/exec \
+    --node-ip ${NODE_IP} \
+    --node-labels "${NODE_LABELS}" \
+    --register-with-taints "${NODE_TAINTS}" \
+    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE}

packages/kubernetes-1.15/kubelet-kubeconfig

@@ -3,8 +3,10 @@ apiVersion: v1
 kind: Config
 clusters:
 - cluster:
+{{~#if settings.kubernetes.api-server}}
     certificate-authority: "/etc/kubernetes/pki/ca.crt"
     server: "{{settings.kubernetes.api-server}}"
+{{~/if}}
   name: kubernetes
 contexts:
 - context:
@@ -14,6 +16,8 @@ contexts:
 current-context: kubelet
 users:
 - name: kubelet
+{{~#if (eq settings.kubernetes.authentication-mode "aws")}}
+{{~#if settings.kubernetes.cluster-name}}
   user:
     exec:
       apiVersion: client.authentication.k8s.io/v1alpha1
@@ -22,3 +26,10 @@ users:
       - token
       - "-i"
       - "{{settings.kubernetes.cluster-name}}"
+{{~/if}}
+{{~/if}}
+{{~#if (eq settings.kubernetes.authentication-mode "tls")}}
+  user:
+    client-certificate: "/var/lib/kubelet/pki/kubelet-client-current.pem"
+    client-key: "/var/lib/kubelet/pki/kubelet-client-current.pem"
+{{~/if}}

packages/kubernetes-1.15/kubelet.service

@@ -16,21 +16,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --namespace=k8s.io \
     pull-image \
     --source=${POD_INFRA_CONTAINER_IMAGE}
-ExecStart=/usr/bin/kubelet \
-    --cloud-provider aws \
-    --config /etc/kubernetes/kubelet/config \
-    --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
-    --container-runtime=remote \
-    --container-runtime-endpoint=unix:///run/dockershim.sock \
-    --containerd=/run/dockershim.sock \
-    --network-plugin cni \
-    --root-dir /var/lib/kubelet \
-    --cert-dir /var/lib/kubelet/pki \
-    --volume-plugin-dir /var/lib/kubelet/plugins/volume/exec \
-    --node-ip ${NODE_IP} \
-    --node-labels "${NODE_LABELS}" \
-    --register-with-taints "${NODE_TAINTS}" \
-    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE}
+# Must be overridden by a drop-in file or `kubelet` won't start
+ExecStart=/usr/bin/false
 
 Restart=on-failure
 RestartForceExitStatus=SIGPIPE

packages/kubernetes-1.15/kubernetes-1.15.spec

@@ -20,7 +20,9 @@ Source2: kubelet-env
 Source3: kubelet-config
 Source4: kubelet-kubeconfig
 Source5: kubernetes-ca-crt
-Source6: kubernetes-tmpfiles.conf
+Source6: kubelet-exec-start-conf
+Source7: kubelet-bootstrap-kubeconfig
+Source8: kubernetes-tmpfiles.conf
 Source1000: clarify.toml
 Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch
 
@@ -79,9 +81,11 @@ install -m 0644 %{S:2} %{buildroot}%{_cross_templatedir}/kubelet-env
 install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config
 install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig
 install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt
+install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
+install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
-install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
+install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
 
 %cross_scan_attribution --clarify %{S:1000} go-vendor vendor
 
@@ -95,6 +99,8 @@ install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_templatedir}/kubelet-env
 %{_cross_templatedir}/kubelet-config
 %{_cross_templatedir}/kubelet-kubeconfig
+%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
+%{_cross_templatedir}/kubelet-exec-start-conf
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_tmpfilesdir}/kubernetes.conf
 

packages/kubernetes-1.15/kubernetes-ca-crt

@@ -1 +1,3 @@
+{{~#if settings.kubernetes.cluster-certificate~}}
 {{base64_decode settings.kubernetes.cluster-certificate}}
+{{~/if~}}

packages/kubernetes-1.16/kubelet-bootstrap-kubeconfig

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+{{~#if settings.kubernetes.api-server}}
+    certificate-authority: "/etc/kubernetes/pki/ca.crt"
+    server: "{{settings.kubernetes.api-server}}"
+{{~/if}}
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: kubelet
+  name: kubelet
+current-context: kubelet
+users:
+- name: kubelet
+{{~#if settings.kubernetes.bootstrap-token}}
+  user:
+    token: "{{settings.kubernetes.bootstrap-token}}"
+{{~/if}}

packages/kubernetes-1.16/kubelet-config

@@ -1,6 +1,16 @@
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1
+{{~#if settings.kubernetes.standalone-mode}}
+address: 127.0.0.1
+authentication:
+  anonymous:
+    enabled: true
+  webhook:
+    enabled: false
+authorization:
+  mode: AlwaysAllow
+{{~else}}
 address: 0.0.0.0
 authentication:
   anonymous:
@@ -15,6 +25,7 @@ authorization:
   webhook:
     cacheAuthorizedTTL: 5m0s
     cacheUnauthorizedTTL: 30s
+{{~/if}}
 clusterDomain: {{settings.kubernetes.cluster-domain}}
 clusterDNS:
 - {{settings.kubernetes.cluster-dns-ip}}

packages/kubernetes-1.16/kubelet-exec-start-conf

@@ -0,0 +1,24 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/kubelet \
+{{~#unless settings.kubernetes.standalone-mode}}
+    --cloud-provider aws \
+    --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
+{{~#if (eq settings.kubernetes.authentication-mode "tls")}}
+    --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \
+{{~/if}}
+{{~else}}
+    --cloud-provider "" \
+{{~/unless}}
+    --config /etc/kubernetes/kubelet/config \
+    --container-runtime=remote \
+    --container-runtime-endpoint=unix:///run/dockershim.sock \
+    --containerd=/run/dockershim.sock \
+    --network-plugin cni \
+    --root-dir /var/lib/kubelet \
+    --cert-dir /var/lib/kubelet/pki \
+    --volume-plugin-dir /var/lib/kubelet/plugins/volume/exec \
+    --node-ip ${NODE_IP} \
+    --node-labels "${NODE_LABELS}" \
+    --register-with-taints "${NODE_TAINTS}" \
+    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE}

packages/kubernetes-1.16/kubelet-kubeconfig

@@ -3,8 +3,10 @@ apiVersion: v1
 kind: Config
 clusters:
 - cluster:
+{{~#if settings.kubernetes.api-server}}
     certificate-authority: "/etc/kubernetes/pki/ca.crt"
     server: "{{settings.kubernetes.api-server}}"
+{{~/if}}
   name: kubernetes
 contexts:
 - context:
@@ -14,6 +16,8 @@ contexts:
 current-context: kubelet
 users:
 - name: kubelet
+{{~#if (eq settings.kubernetes.authentication-mode "aws")}}
+{{~#if settings.kubernetes.cluster-name}}
   user:
     exec:
       apiVersion: client.authentication.k8s.io/v1alpha1
@@ -22,3 +26,10 @@ users:
       - token
       - "-i"
       - "{{settings.kubernetes.cluster-name}}"
+{{~/if}}
+{{~/if}}
+{{~#if (eq settings.kubernetes.authentication-mode "tls")}}
+  user:
+    client-certificate: "/var/lib/kubelet/pki/kubelet-client-current.pem"
+    client-key: "/var/lib/kubelet/pki/kubelet-client-current.pem"
+{{~/if}}

packages/kubernetes-1.16/kubelet.service

@@ -16,21 +16,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --namespace=k8s.io \
     pull-image \
     --source=${POD_INFRA_CONTAINER_IMAGE}
-ExecStart=/usr/bin/kubelet \
-    --cloud-provider aws \
-    --config /etc/kubernetes/kubelet/config \
-    --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
-    --container-runtime=remote \
-    --container-runtime-endpoint=unix:///run/dockershim.sock \
-    --containerd=/run/dockershim.sock \
-    --network-plugin cni \
-    --root-dir /var/lib/kubelet \
-    --cert-dir /var/lib/kubelet/pki \
-    --volume-plugin-dir /var/lib/kubelet/plugins/volume/exec \
-    --node-ip ${NODE_IP} \
-    --node-labels "${NODE_LABELS}" \
-    --register-with-taints "${NODE_TAINTS}" \
-    --pod-infra-container-image ${POD_INFRA_CONTAINER_IMAGE}
+# Must be overridden by a drop-in file or `kubelet` won't start
+ExecStart=/usr/bin/false
 
 Restart=on-failure
 RestartForceExitStatus=SIGPIPE

packages/kubernetes-1.16/kubernetes-1.16.spec

@@ -20,7 +20,9 @@ Source2: kubelet-env
 Source3: kubelet-config
 Source4: kubelet-kubeconfig
 Source5: kubernetes-ca-crt
-Source6: kubernetes-tmpfiles.conf
+Source6: kubelet-exec-start-conf
+Source7: kubelet-bootstrap-kubeconfig
+Source8: kubernetes-tmpfiles.conf
 Source1000: clarify.toml
 Patch1: 0001-always-set-relevant-variables-for-cross-compiling.patch
 
@@ -75,9 +77,11 @@ install -m 0644 %{S:2} %{buildroot}%{_cross_templatedir}/kubelet-env
 install -m 0644 %{S:3} %{buildroot}%{_cross_templatedir}/kubelet-config
 install -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/kubelet-kubeconfig
 install -m 0644 %{S:5} %{buildroot}%{_cross_templatedir}/kubernetes-ca-crt
+install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/kubelet-exec-start-conf
+install -m 0644 %{S:7} %{buildroot}%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
 
 install -d %{buildroot}%{_cross_tmpfilesdir}
-install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
+install -p -m 0644 %{S:8} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
 
 %cross_scan_attribution --clarify %{S:1000} go-vendor vendor
 
@@ -91,6 +95,8 @@ install -p -m 0644 %{S:6} %{buildroot}%{_cross_tmpfilesdir}/kubernetes.conf
 %{_cross_templatedir}/kubelet-env
 %{_cross_templatedir}/kubelet-config
 %{_cross_templatedir}/kubelet-kubeconfig
+%{_cross_templatedir}/kubelet-bootstrap-kubeconfig
+%{_cross_templatedir}/kubelet-exec-start-conf
 %{_cross_templatedir}/kubernetes-ca-crt
 %{_cross_tmpfilesdir}/kubernetes.conf
 

packages/kubernetes-1.16/kubernetes-ca-crt

@@ -1 +1,3 @@
+{{~#if settings.kubernetes.cluster-certificate~}}
 {{base64_decode settings.kubernetes.cluster-certificate}}
+{{~/if~}}

packages/kubernetes-1.17/kubelet-bootstrap-kubeconfig

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+{{~#if settings.kubernetes.api-server}}
+    certificate-authority: "/etc/kubernetes/pki/ca.crt"
+    server: "{{settings.kubernetes.api-server}}"
+{{~/if}}
+  name: kubernetes
+contexts:
+- context:
+    cluster: kubernetes
+    user: kubelet
+  name: kubelet
+current-context: kubelet
+users:
+- name: kubelet
+{{~#if settings.kubernetes.bootstrap-token}}
+  user:
+    token: "{{settings.kubernetes.bootstrap-token}}"
+{{~/if}}