Skip to content
This repository has been archived by the owner on Feb 5, 2020. It is now read-only.

Kubelet tls bootstrap #2809

Merged
merged 1 commit into from
Feb 12, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions config.tf
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ variable "tectonic_container_images" {
addon_resizer = "gcr.io/google_containers/addon-resizer:2.1"
awscli = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
gcloudsdk = "google/cloud-sdk:178.0.0-alpine"
bootkube = "quay.io/coreos/bootkube:v0.8.1"
bootkube = "quay.io/coreos/bootkube:v0.10.0"
etcd = "quay.io/coreos/etcd:v3.2.14"
hyperkube = "quay.io/coreos/hyperkube:v1.9.1_coreos.0"
kube_core_renderer = "quay.io/coreos/kube-core-renderer:beryllium-m1"
kube_core_renderer = "quay.io/coreos/kube-core-renderer-dev:6c49ce4da9fc36966812381891b4f558aa53097b"
kube_core_operator = "quay.io/coreos/kube-core-operator:beryllium-m1"
tectonic_channel_operator = "quay.io/coreos/tectonic-channel-operator:0.6.2"
tectonic_prometheus_operator = "quay.io/coreos/tectonic-prometheus-operator:v1.9.1"
Expand Down
49 changes: 25 additions & 24 deletions contrib/user-provided-certs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,29 @@ Once configured, execute `terraform apply`. The folder `generated/tls` will cont

The following table gives an overview which generated certificates have to be configured for the corresponding variables for the `user-provided` flavor of TLS modules:

Certificate | TLS module | Variable
-----------------------| ---------------------------------------| ---------
`aggregator-ca.crt` | `modules/tls/kube/user-provided` | `aggregator_ca_cert_pem_path`
`apiserver.crt` | `modules/tls/kube/user-provided` | `apiserver_cert_pem_path`
`apiserver.key` | `modules/tls/kube/user-provided` | `apiserver_key_pem_path`
`apiserver-proxy.crt` | `modules/tls/kube/user-provided` | `apiserver_proxy_cert_pem_path`
`apiserver-proxy.key` | `modules/tls/kube/user-provided` | `apiserver_proxy_key_pem_path`
`ca.crt` | `modules/tls/kube/user-provided` | `ca_cert_pem_path`
`ca.crt` | `modules/tls/ingress/user-provided` | `ca_cert_pem_path`
`etcd-ca.crt` | `modules/tls/etcd/user-provided` | `etcd_ca_crt_pem_path`
`etcd-client.crt` | `modules/tls/etcd/user-provided` | `etcd_client_crt_pem_path`
`etcd-client.key` | `modules/tls/etcd/user-provided` | `etcd_client_key_pem_path`
`etcd-peer.crt` | `modules/tls/etcd/user-provided` | `etcd_peer_crt_pem_path`
`etcd-peer.key` | `modules/tls/etcd/user-provided` | `etcd_peer_key_pem_path`
`etcd-server.crt` | `modules/tls/etcd/user-provided` | `etcd_server_crt_pem_path`
`etcd-server.key` | `modules/tls/etcd/user-provided` | `etcd_server_key_pem_path`
`identity-client.crt` | `modules/tls/identity/user-provided` | `client_cert_pem_path`
`identity-client.key` | `modules/tls/identity/user-provided` | `client_key_pem_path`
`identity-server.crt` | `modules/tls/identity/user-provided` | `server_cert_pem_path`
`identity-server.key` | `modules/tls/identity/user-provided` | `server_key_pem_path`
`ingress.crt` | `modules/tls/ingress/user-provided` | `cert_pem_path`
`ingress.key` | `modules/tls/ingress/user-provided` | `key_pem_path`
`kubelet.crt` | `modules/tls/kube/user-provided` | `kubelet_cert_pem_path`
`kubelet.key` | `modules/tls/kube/user-provided` | `kubelet_key_pem_path`
Certificate | TLS module | Variable
----------------------|--------------------------------------|---------
`aggregator-ca.crt` | `modules/tls/kube/user-provided` | `aggregator_ca_cert_pem_path`
`apiserver.crt` | `modules/tls/kube/user-provided` | `apiserver_cert_pem_path`
`apiserver.key` | `modules/tls/kube/user-provided` | `apiserver_key_pem_path`
`apiserver-proxy.crt` | `modules/tls/kube/user-provided` | `apiserver_proxy_cert_pem_path`
`apiserver-proxy.key` | `modules/tls/kube/user-provided` | `apiserver_proxy_key_pem_path`
`ca.crt` | `modules/tls/kube/user-provided` | `ca_cert_pem_path`
`ca.key` | `modules/tls/kube/user-provided` | `ca_key_pem_path`
`ca.crt` | `modules/tls/ingress/user-provided` | `ca_cert_pem_path`
`etcd-ca.crt` | `modules/tls/etcd/user-provided` | `etcd_ca_crt_pem_path`
`etcd-client.crt` | `modules/tls/etcd/user-provided` | `etcd_client_crt_pem_path`
`etcd-client.key` | `modules/tls/etcd/user-provided` | `etcd_client_key_pem_path`
`etcd-peer.crt` | `modules/tls/etcd/user-provided` | `etcd_peer_crt_pem_path`
`etcd-peer.key` | `modules/tls/etcd/user-provided` | `etcd_peer_key_pem_path`
`etcd-server.crt` | `modules/tls/etcd/user-provided` | `etcd_server_crt_pem_path`
`etcd-server.key` | `modules/tls/etcd/user-provided` | `etcd_server_key_pem_path`
`identity-client.crt` | `modules/tls/identity/user-provided` | `client_cert_pem_path`
`identity-client.key` | `modules/tls/identity/user-provided` | `client_key_pem_path`
`identity-server.crt` | `modules/tls/identity/user-provided` | `server_cert_pem_path`
`identity-server.key` | `modules/tls/identity/user-provided` | `server_key_pem_path`
`ingress.crt` | `modules/tls/ingress/user-provided` | `cert_pem_path`
`ingress.key` | `modules/tls/ingress/user-provided` | `key_pem_path`
`admin.crt` | `modules/tls/kube/user-provided` | `admin_cert_pem_path`
`admin.key` | `modules/tls/kube/user-provided` | `admin_key_pem_path`

43 changes: 43 additions & 0 deletions contrib/user-provided-certs/kube/self-signed/admin.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# admin
# Admin (generated/tls/{admin.key,admin.crt})
# Used to create kubeconfig (generated/auth/kubeconfig) with admin level privileges.
resource "tls_private_key" "admin" {
algorithm = "RSA"
rsa_bits = "2048"
}

resource "tls_cert_request" "admin" {
key_algorithm = "${tls_private_key.admin.algorithm}"
private_key_pem = "${tls_private_key.admin.private_key_pem}"

subject {
common_name = "admin"
organization = "system:masters"
}
}

resource "tls_locally_signed_cert" "admin" {
cert_request_pem = "${tls_cert_request.admin.cert_request_pem}"

ca_key_algorithm = "${var.ca_cert_pem == "" ? join(" ", tls_self_signed_cert.kube_ca.*.key_algorithm) : var.ca_key_alg}"
ca_private_key_pem = "${var.ca_cert_pem == "" ? join(" ", tls_private_key.kube_ca.*.private_key_pem) : var.ca_key_pem}"
ca_cert_pem = "${var.ca_cert_pem == "" ? join(" ", tls_self_signed_cert.kube_ca.*.cert_pem) : var.ca_cert_pem}"
validity_period_hours = "${var.validity_period}"

allowed_uses = [
"key_encipherment",
"digital_signature",
"server_auth",
"client_auth",
]
}

resource "local_file" "admin_key" {
content = "${tls_private_key.admin.private_key_pem}"
filename = "./generated/tls/admin.key"
}

resource "local_file" "admin_crt" {
content = "${tls_locally_signed_cert.admin.cert_pem}"
filename = "./generated/tls/admin.crt"
}
42 changes: 0 additions & 42 deletions contrib/user-provided-certs/kube/self-signed/kubelet.tf

This file was deleted.

8 changes: 4 additions & 4 deletions contrib/user-provided-certs/kube/self-signed/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ output "ca_key_pem" {
value = "${tls_private_key.kube_ca.private_key_pem}"
}

output "kubelet_cert_pem" {
value = "${tls_locally_signed_cert.kubelet.cert_pem}"
output "admin_cert_pem" {
value = "${tls_locally_signed_cert.admin.cert_pem}"
}

output "kubelet_key_pem" {
value = "${tls_private_key.kubelet.private_key_pem}"
output "admin_key_pem" {
value = "${tls_private_key.admin.private_key_pem}"
}

output "apiserver_cert_pem" {
Expand Down
62 changes: 48 additions & 14 deletions modules/bootkube/assets.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
# Kubelet tls bootstraping id and secret
resource "random_string" "kubelet_bootstrap_token_id" {
length = 6
special = false
upper = false
}

resource "random_string" "kubelet_bootstrap_token_secret" {
length = 16
special = false
upper = false
}

# Self-hosted manifests (resources/generated/manifests/)
resource "template_dir" "bootkube" {
source_dir = "${path.module}/resources/manifests"
Expand All @@ -13,16 +26,19 @@ resource "template_dir" "bootkube" {
cluster_cidr = "${var.cluster_cidr}"
tectonic_networking = "${var.tectonic_networking}"

aggregator_ca_cert = "${base64encode(var.aggregator_ca_cert_pem)}"
kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}"
apiserver_key = "${base64encode(var.apiserver_key_pem)}"
apiserver_cert = "${base64encode(var.apiserver_cert_pem)}"
apiserver_proxy_key = "${base64encode(var.apiserver_proxy_key_pem)}"
apiserver_proxy_cert = "${base64encode(var.apiserver_proxy_cert_pem)}"
oidc_ca_cert = "${base64encode(var.oidc_ca_cert)}"
pull_secret = "${base64encode(file(var.pull_secret_path))}"
serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}"
serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}"
aggregator_ca_cert = "${base64encode(var.aggregator_ca_cert_pem)}"
kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}"
kube_ca_key = "${base64encode(var.kube_ca_key_pem)}"
kubelet_bootstrap_token_id = "${random_string.kubelet_bootstrap_token_id.result}"
kubelet_bootstrap_token_secret = "${random_string.kubelet_bootstrap_token_secret.result}"
apiserver_key = "${base64encode(var.apiserver_key_pem)}"
apiserver_cert = "${base64encode(var.apiserver_cert_pem)}"
apiserver_proxy_key = "${base64encode(var.apiserver_proxy_key_pem)}"
apiserver_proxy_cert = "${base64encode(var.apiserver_proxy_cert_pem)}"
oidc_ca_cert = "${base64encode(var.oidc_ca_cert)}"
pull_secret = "${base64encode(file(var.pull_secret_path))}"
serviceaccount_pub = "${base64encode(tls_private_key.service_account.public_key_pem)}"
serviceaccount_key = "${base64encode(tls_private_key.service_account.private_key_pem)}"

etcd_ca_cert = "${base64encode(var.etcd_ca_cert_pem)}"
etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
Expand All @@ -36,8 +52,8 @@ data "template_file" "kubeconfig" {

vars {
kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}"
kubelet_cert = "${base64encode(var.kubelet_cert_pem)}"
kubelet_key = "${base64encode(var.kubelet_key_pem)}"
admin_cert = "${base64encode(var.admin_cert_pem)}"
admin_key = "${base64encode(var.admin_key_pem)}"
server = "${var.kube_apiserver_url}"
cluster_name = "${var.cluster_name}"
}
Expand All @@ -48,11 +64,31 @@ resource "local_file" "kubeconfig" {
filename = "./generated/auth/kubeconfig"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would consider renaming this one too (to kubeconfig-admin) to make it really clear that you shouldn't use it for anything in the cluster. Others might have different feelings though.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Heh, look at the fool who left that TODO there :-/

Copy link
Member

@enxebre enxebre Jan 24, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

may be just add a comment on the kubeconfig file if that does not break any consumer?

}

# kubeconfig-kubelet (resources/generated/auth/kubeconfig-kubelet)
data "template_file" "kubeconfig-kubelet" {
template = "${file("${path.module}/resources/kubeconfig-kubelet")}"

vars {
kube_ca_cert = "${base64encode(var.kube_ca_cert_pem)}"
kubelet_bootstrap_token_id = "${random_string.kubelet_bootstrap_token_id.result}"
kubelet_bootstrap_token_secret = "${random_string.kubelet_bootstrap_token_secret.result}"
server = "${var.kube_apiserver_url}"
cluster_name = "${var.cluster_name}"
}
}

resource "local_file" "kubeconfig-kubelet" {
content = "${data.template_file.kubeconfig-kubelet.rendered}"
filename = "./generated/auth/kubeconfig-kubelet"
}

# kvo-config.yaml (resources/generated/kco-config.yaml)
data "template_file" "kco-config_yaml" {
template = "${file("${path.module}/resources/kco-config.yaml")}"

vars {
kube_apiserver_url = "${var.kube_apiserver_url}"

cloud_config_path = "${var.cloud_config_path}"
cloud_provider_profile = "${var.cloud_provider != "" ? "${var.cloud_provider}" : "metal"}"

Expand All @@ -67,8 +103,6 @@ data "template_file" "kco-config_yaml" {
oidc_groups_claim = "${var.oidc_groups_claim}"
oidc_issuer_url = "${var.oidc_issuer_url}"
oidc_username_claim = "${var.oidc_username_claim}"

master_count = "${var.master_count}"
}
}

Expand Down
5 changes: 5 additions & 0 deletions modules/bootkube/outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
output "id" {
value = "${sha1("
${local_file.kubeconfig.id}
${local_file.kubeconfig-kubelet.id}
${local_file.bootkube_sh.id}
${local_file.kco-config_yaml.id}
${template_dir.bootkube.id}
Expand All @@ -28,6 +29,10 @@ output "kubeconfig" {
value = "${data.template_file.kubeconfig.rendered}"
}

output "kubeconfig-kubelet" {
value = "${data.template_file.kubeconfig-kubelet.rendered}"
}

output "systemd_service_rendered" {
value = "${data.template_file.bootkube_service.rendered}"
}
Expand Down
4 changes: 2 additions & 2 deletions modules/bootkube/resources/kco-config.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
apiVersion: v1
kind: KubeCoreOperatorConfig
clusterConfig:
apiserver_url: ${kube_apiserver_url}
authConfig:
oidc_client_id: ${oidc_client_id}
oidc_issuer_url: ${oidc_issuer_url}
Expand All @@ -13,5 +15,3 @@ networkConfig:
cluster_cidr: ${cluster_cidr}
etcd_servers: ${etcd_servers}
service_cidr: ${service_cidr}
initialConfig:
initial_master_count: ${master_count}
8 changes: 4 additions & 4 deletions modules/bootkube/resources/kubeconfig
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ clusters:
server: ${server}
certificate-authority-data: ${kube_ca_cert}
users:
- name: kubelet
- name: admin
user:
client-certificate-data: ${kubelet_cert}
client-key-data: ${kubelet_key}
client-certificate-data: ${admin_cert}
client-key-data: ${admin_key}
contexts:
- context:
cluster: ${cluster_name}
user: kubelet
user: admin
15 changes: 15 additions & 0 deletions modules/bootkube/resources/kubeconfig-kubelet
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: v1
kind: Config
clusters:
- name: ${cluster_name}
cluster:
server: ${server}
certificate-authority-data: ${kube_ca_cert}
users:
- name: kubelet
user:
token: ${kubelet_bootstrap_token_id}.${kubelet_bootstrap_token_secret}
contexts:
- context:
cluster: ${cluster_name}
user: kubelet
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ type: Opaque
data:
service-account.key: ${serviceaccount_key}
ca.crt: ${kube_ca_cert}
ca.key: ${kube_ca_key}
10 changes: 10 additions & 0 deletions modules/bootkube/resources/manifests/kubelet-bootstrap-token.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: v1
kind: Secret
metadata:
name: bootstrap-token-${kubelet_bootstrap_token_id}
namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
token-id: ${kubelet_bootstrap_token_id}
token-secret: ${kubelet_bootstrap_token_secret}
usage-bootstrap-authentication: "true"
9 changes: 7 additions & 2 deletions modules/bootkube/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,17 @@ variable "kube_ca_cert_pem" {
description = "The Kubernetes CA in PEM format."
}

variable "kubelet_cert_pem" {
variable "kube_ca_key_pem" {
type = "string"
description = "The Kubernetes CA key in PEM format."
}

variable "admin_cert_pem" {
type = "string"
description = "The kubelet certificate in PEM format."
}

variable "kubelet_key_pem" {
variable "admin_key_pem" {
type = "string"
description = "The kubelet key in PEM format."
}
Expand Down
Loading