Trust the Kubernetes CA on nodes

modules/aws/master-asg/ignition.tf

@@ -4,6 +4,7 @@ data "ignition_config" "main" {
     "${var.ign_s3_puller_id}",
     "${data.ignition_file.init_assets.id}",
     "${data.ignition_file.detect_master.id}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
@@ -14,6 +15,7 @@ data "ignition_config" "main" {
     "${data.ignition_systemd_unit.init_assets.id}",
     "${data.ignition_systemd_unit.bootkube.id}",
     "${data.ignition_systemd_unit.tectonic.id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }
 

modules/aws/worker-asg/ignition.tf

@@ -2,12 +2,14 @@ data "ignition_config" "main" {
   files = [
     "${var.ign_max_user_watches_id}",
     "${var.ign_s3_puller_id}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
     "${var.ign_docker_dropin_id}",
     "${var.ign_locksmithd_service_id}",
     "${var.ign_kubelet_service_id}",
     "${var.ign_s3_kubelet_env_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }

modules/azure/master-as/ignition-master.tf

@@ -5,6 +5,7 @@ data "ignition_config" "master" {
     "${var.ign_azure_udev_rules_id}",
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.cloud_provider_config.id}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
@@ -14,6 +15,7 @@ data "ignition_config" "master" {
     "${data.ignition_systemd_unit.tectonic.id}",
     "${data.ignition_systemd_unit.bootkube.id}",
     "${var.ign_tx_off_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 
   users = [

modules/azure/worker-as/ignition-worker.tf

@@ -5,13 +5,15 @@ data "ignition_config" "worker" {
     "${var.ign_azure_udev_rules_id}",
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.cloud-provider-config.id}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
     "${var.ign_docker_dropin_id}",
     "${var.ign_locksmithd_service_id}",
     "${var.ign_kubelet_service_id}",
     "${var.ign_tx_off_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 
   users = [

modules/ignition/assets.tf

@@ -12,6 +12,18 @@ data "ignition_file" "max_user_watches" {
   }
 }
 
+data "ignition_file" "kube_ca" {
+  filesystem = "root"
+  path       = "/etc/ssl/certs/kube_ca.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  content {
+    content = "${var.kube_ca_crt_pem}"
+  }
+}
+
 data "template_file" "docker_dropin" {
   template = "${file("${path.module}/resources/dropins/10-dockeropts.conf")}"
 }
@@ -28,6 +40,22 @@ data "ignition_systemd_unit" "docker_dropin" {
   ]
 }
 
+data "template_file" "update_ca_certificates_dropin" {
+  template = "${file("${path.module}/resources/dropins/10-dockeropts.conf")}"
+}
+
+data "ignition_systemd_unit" "update_ca_certificates_dropin" {
+  name   = "update-ca-certificates.service"
+  enable = true
+
+  dropin = [
+    {
+      name    = "10-always-update-ca-certificates.conf"
+      content = "${data.template_file.update_ca_certificates_dropin.rendered}"
+    },
+  ]
+}
+
 data "template_file" "kubelet" {
   template = "${file("${path.module}/resources/services/kubelet.service")}"
 

modules/ignition/outputs.import

@@ -15,3 +15,11 @@ variable "ign_kubelet_service_id" {
 variable "ign_locksmithd_service_id" {
   type = "string"
 }
+
+variable "ign_update_ca_certificates_dropin_id" {
+  type = "string"
+}
+
+variable "ign_kube_ca_id" {
+  type = "string"
+}

modules/ignition/outputs.tf

@@ -14,6 +14,14 @@ output "docker_dropin_rendered" {
   value = "${data.template_file.docker_dropin.rendered}"
 }
 
+output "update_ca_certificates_dropin_id" {
+  value = "${data.ignition_systemd_unit.update_ca_certificates_dropin.id}"
+}
+
+output "update_ca_certificates_dropin_rendered" {
+  value = "${data.template_file.update_ca_certificates_dropin.rendered}"
+}
+
 output "kubelet_service_id" {
   value = "${data.ignition_systemd_unit.kubelet.id}"
 }
@@ -50,6 +58,14 @@ output "kubelet_env_rendered" {
   value = "${data.template_file.kubelet_env.rendered}"
 }
 
+output "kube_ca_id" {
+  value = "${data.ignition_file.kube_ca.id}"
+}
+
+output "kube_ca_rendered" {
+  value = "${data.template_file.kube_ca.rendered}"
+}
+
 output "tx_off_service_id" {
   value = "${data.ignition_systemd_unit.tx_off.id}"
 }

modules/ignition/resources/dropins/10-always-update-ca-certificates.conf

@@ -0,0 +1,6 @@
+[Unit]
+ConditionPathIsSymbolicLink=
+
+[Service]
+ExecStart=
+ExecStart=/usr/sbin/update-ca-certificates

modules/ignition/variables.tf

@@ -48,3 +48,7 @@ variable "cloud_provider_config" {
   description = "(optional) The cloud provider config to be used for the kubelet."
   default     = ""
 }
+
+variable "kube_ca_crt_pem" {
+  type = "string"
+}

modules/openstack/nodes/ignition.tf

@@ -11,6 +11,7 @@ data "ignition_config" "node" {
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.resolv_conf.id}",
     "${data.ignition_file.hostname.*.id[count.index]}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
@@ -19,6 +20,7 @@ data "ignition_config" "node" {
     "${var.ign_kubelet_service_id}",
     "${data.ignition_systemd_unit.bootkube.id}",
     "${data.ignition_systemd_unit.tectonic.id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }
 

modules/vmware/node/ignition.tf

@@ -9,6 +9,7 @@ data "ignition_config" "node" {
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.node_hostname.*.id[count.index]}",
     "${var.ign_kubelet_env_id}",
+    "${var.ign_kube_ca_id}",
   ]
 
   systemd = [
@@ -18,6 +19,8 @@ data "ignition_config" "node" {
     "${var.ign_kubelet_env_service_id}",
     "${data.ignition_systemd_unit.bootkube.id}",
     "${data.ignition_systemd_unit.tectonic.id}",
+    "${data.ignition_systemd_unit.vmtoolsd_member.id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 
   networkd = [

platforms/aws/main.tf

@@ -27,18 +27,18 @@ module "vpc" {
   # To enable mode A, configure a set of AZs + CIDRs for masters and workers using the
   # "tectonic_aws_master_custom_subnets" and "tectonic_aws_worker_custom_subnets" variables.
   #
-  # To enable mode B, make sure that "tectonic_aws_master_custom_subnets" and "tectonic_aws_worker_custom_subnets" 
+  # To enable mode B, make sure that "tectonic_aws_master_custom_subnets" and "tectonic_aws_worker_custom_subnets"
   # ARE NOT SET.
 
-  # These counts could be deducted by length(keys(var.tectonic_aws_master_custom_subnets)) 
+  # These counts could be deducted by length(keys(var.tectonic_aws_master_custom_subnets))
   # but there is a restriction on passing computed values as counts. This approach works around that.
   master_az_count = "${length(keys(var.tectonic_aws_master_custom_subnets)) > 0 ? "${length(keys(var.tectonic_aws_master_custom_subnets))}" : "${length(data.aws_availability_zones.azs.names)}"}"
   worker_az_count = "${length(keys(var.tectonic_aws_worker_custom_subnets)) > 0 ? "${length(keys(var.tectonic_aws_worker_custom_subnets))}" : "${length(data.aws_availability_zones.azs.names)}"}"
   # The appending of the "padding" element is required as workaround since the function
   # element() won't work on empty lists. See https://github.com/hashicorp/terraform/issues/11210
   master_subnets = "${concat(values(var.tectonic_aws_master_custom_subnets),list("padding"))}"
   worker_subnets = "${concat(values(var.tectonic_aws_worker_custom_subnets),list("padding"))}"
-  # The split() / join() trick works around the limitation of ternary operator expressions 
+  # The split() / join() trick works around the limitation of ternary operator expressions
   # only being able to return strings.
   master_azs = "${ split("|", "${length(keys(var.tectonic_aws_master_custom_subnets))}" > 0 ?
     join("|", keys(var.tectonic_aws_master_custom_subnets)) :
@@ -92,6 +92,7 @@ module "ignition_masters" {
   kubelet_cni_bin_dir  = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
   kubelet_node_label   = "node-role.kubernetes.io/master"
   kubelet_node_taints  = "node-role.kubernetes.io/master=:NoSchedule"
+  kube_ca_crt_pem      = "${module.bootkube.ca_cert}"
 }
 
 module "masters" {
@@ -126,12 +127,14 @@ module "masters" {
   tectonic_service             = "${module.tectonic.systemd_service}"
   tectonic_service_disabled    = "${var.tectonic_vanilla_k8s}"
 
-  ign_docker_dropin_id          = "${module.ignition_masters.docker_dropin_id}"
-  ign_kubelet_service_id        = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id     = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id       = "${module.ignition_masters.max_user_watches_id}"
-  ign_s3_kubelet_env_service_id = "${module.ignition_masters.kubelet_env_service_id}"
-  ign_s3_puller_id              = "${module.ignition_masters.s3_puller_id}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_kube_ca_id                       = "${module.ignition_masters.kube_ca_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_s3_kubelet_env_service_id        = "${module.ignition_masters.kubelet_env_service_id}"
+  ign_s3_puller_id                     = "${module.ignition_masters.s3_puller_id}"
 }
 
 module "ignition_workers" {
@@ -145,6 +148,7 @@ module "ignition_workers" {
   kubelet_cni_bin_dir  = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
   kubelet_node_label   = "node-role.kubernetes.io/node"
   kubelet_node_taints  = ""
+  kube_ca_crt_pem      = "${module.bootkube.ca_cert}"
 }
 
 module "workers" {
@@ -166,10 +170,12 @@ module "workers" {
   vpc_id                       = "${module.vpc.vpc_id}"
   worker_iam_role              = "${var.tectonic_aws_worker_iam_role_name}"
 
-  ign_docker_dropin_id          = "${module.ignition_workers.docker_dropin_id}"
-  ign_kubelet_service_id        = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id     = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id       = "${module.ignition_workers.max_user_watches_id}"
-  ign_s3_kubelet_env_service_id = "${module.ignition_workers.kubelet_env_service_id}"
-  ign_s3_puller_id              = "${module.ignition_workers.s3_puller_id}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_kube_ca_id                       = "${module.ignition_workers.kube_ca_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_s3_kubelet_env_service_id        = "${module.ignition_workers.kubelet_env_service_id}"
+  ign_s3_puller_id                     = "${module.ignition_workers.s3_puller_id}"
 }

platforms/azure/main.tf

@@ -104,6 +104,7 @@ module "ignition_masters" {
   kubelet_cni_bin_dir   = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
   kubelet_node_label    = "node-role.kubernetes.io/master"
   kubelet_node_taints   = "node-role.kubernetes.io/master=:NoSchedule"
+  kube_ca_crt_pem       = "${module.bootkube.ca_cert}"
 }
 
 module "masters" {
@@ -127,14 +128,16 @@ module "masters" {
   tectonic_service_disabled = "${var.tectonic_vanilla_k8s}"
   vm_size                   = "${var.tectonic_azure_master_vm_size}"
 
-  ign_azure_udev_rules_id   = "${module.ignition_masters.azure_udev_rules_id}"
-  ign_docker_dropin_id      = "${module.ignition_masters.docker_dropin_id}"
-  ign_docker_dropin_id      = "${module.ignition_masters.docker_dropin_id}"
-  ign_kubelet_env_id        = "${module.ignition_masters.kubelet_env_id}"
-  ign_kubelet_service_id    = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id   = "${module.ignition_masters.max_user_watches_id}"
-  ign_tx_off_service_id     = "${module.ignition_masters.tx_off_service_id}"
+  ign_azure_udev_rules_id              = "${module.ignition_masters.azure_udev_rules_id}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_kubelet_env_id                   = "${module.ignition_masters.kubelet_env_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_kube_ca_id                       = "${module.ignition_masters.kube_ca_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_tx_off_service_id                = "${module.ignition_masters.tx_off_service_id}"
 }
 
 module "ignition_workers" {
@@ -148,6 +151,7 @@ module "ignition_workers" {
   kubelet_cni_bin_dir   = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
   kubelet_node_label    = "node-role.kubernetes.io/node"
   kubelet_node_taints   = ""
+  kube_ca_crt_pem       = "${module.bootkube.ca_cert}"
 }
 
 module "workers" {
@@ -169,13 +173,15 @@ module "workers" {
   vm_size                      = "${var.tectonic_azure_worker_vm_size}"
   worker_count                 = "${var.tectonic_worker_count}"
 
-  ign_azure_udev_rules_id   = "${module.ignition_workers.azure_udev_rules_id}"
-  ign_docker_dropin_id      = "${module.ignition_workers.docker_dropin_id}"
-  ign_kubelet_env_id        = "${module.ignition_workers.kubelet_env_id}"
-  ign_kubelet_service_id    = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id   = "${module.ignition_workers.max_user_watches_id}"
-  ign_tx_off_service_id     = "${module.ignition_workers.tx_off_service_id}"
+  ign_azure_udev_rules_id              = "${module.ignition_workers.azure_udev_rules_id}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_kubelet_env_id                   = "${module.ignition_workers.kubelet_env_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_kube_ca_id                       = "${module.ignition_workers.kube_ca_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_tx_off_service_id                = "${module.ignition_workers.tx_off_service_id}"
 }
 
 module "dns" {

platforms/metal/cl/bootkube-controller.yaml.tmpl

@@ -70,6 +70,12 @@ systemd:
         ExecStartPost=/bin/touch /opt/tectonic/init_bootkube.done
         [Install]
         WantedBy=multi-user.target
+    - name: update-ca-certificates.service
+      enable: true
+      dropins:
+        - name: 10-alwaysrun.conf
+          contents: ${{.ign_update_ca_certificates_dropin_json}}
+
 {{ if eq .exclude_tectonic "0" }}
     - name: tectonic.service
       contents: |
@@ -107,6 +113,11 @@ storage:
       mode: 0644
       contents:
         inline: {{.ign_max_user_watches_json}}
+    - path: /etc/ssl/certs/kube_ca.pem
+      filesystem: root
+      mode: 0400
+      contents:
+        inline: {{.ign_kube_ca_json}}
 passwd:
   users:
     - name: core

platforms/metal/cl/bootkube-worker.yaml.tmpl

@@ -35,6 +35,11 @@ systemd:
       contents: {{.ign_kubelet_env_service_json}}
     - name: kubelet.service
       contents: {{.ign_kubelet_service_json}}
+    - name: update-ca-certificates.service
+      enable: true
+      dropins:
+        - name: 10-alwaysrun.conf
+          contents: ${{.ign_update_ca_certificates_dropin_json}}
 
 storage:
   files:
@@ -55,6 +60,11 @@ storage:
       mode: 0644
       contents:
         inline: {{.ign_max_user_watches_json}}
+    - path: /etc/ssl/certs/kube_ca.pem
+      filesystem: root
+      mode: 0400
+      contents:
+        inline: {{.ign_kube_ca_json}}
 passwd:
   users:
     - name: core