From 447af6d1b69a1ea84d1b7f6a423c277dd2924378 Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Fri, 10 Nov 2017 16:39:56 +0100
Subject: [PATCH 1/9] modules/*: trust CA certificates on the nodes

This also bumps the ignition provider to 1.0.0 to support S3 downloads
via ignition.

Fixes #INST-67
---
 config.tf                                     |  19 +-
 modules/aws/etcd/ignition.tf                  |   8 +-
 modules/aws/master-asg/ignition.tf            |  17 ++
 modules/aws/master-asg/variables.tf           |  15 ++
 modules/aws/worker-asg/ignition.tf            |  17 ++
 modules/aws/worker-asg/variables.tf           |  15 ++
 modules/azure/etcd/ignition.tf                |   4 +-
 modules/azure/master-as/ignition-master.tf    |   2 +
 modules/azure/master-as/variables.tf          |   5 +
 modules/azure/worker-as/ignition-worker.tf    |   2 +
 modules/azure/worker-as/variables.tf          |   5 +
 modules/bootkube/assets.tf                    |   4 +-
 modules/gcp/etcd/ignition.tf                  |   4 +-
 modules/gcp/master-igm/ignition.tf            |   2 +
 modules/gcp/master-igm/variables.tf           |   5 +
 modules/gcp/worker-igm/ignition.tf            |   2 +
 modules/gcp/worker-igm/variables.tf           |   5 +
 modules/ignition/assets.tf                    |  20 +--
 modules/ignition/certs.tf                     |  65 +++++++
 modules/ignition/etcd.tf                      |   6 +-
 modules/ignition/outputs.import               |   4 +
 modules/ignition/outputs.tf                   |  26 +++
 .../10-always-update-ca-certificates.conf     |   6 +
 modules/ignition/variables.tf                 |  20 +++
 modules/openstack/etcd/ignition.tf            |   4 +-
 modules/openstack/nodes/ignition.tf           |   2 +
 modules/openstack/nodes/variables.tf          |   5 +
 modules/tectonic/assets.tf                    |   4 +-
 modules/vmware/etcd/ignition.tf               |   4 +-
 modules/vmware/node/ignition.tf               |   2 +
 modules/vmware/node/variables.tf              |   5 +
 platforms/aws/main.tf                         | 162 ++++++++++--------
 platforms/aws/s3.tf                           |  24 +++
 platforms/azure/main.tf                       | 144 +++++++++-------
 .../metal/cl/bootkube-controller.yaml.tmpl    |  10 ++
 platforms/metal/cl/bootkube-worker.yaml.tmpl  |  10 ++
 platforms/metal/matchers.tf                   |  64 ++++---
 platforms/openstack/neutron/main.tf           |  86 ++++++----
 platforms/vmware/main.tf                      | 150 ++++++++--------
 39 files changed, 653 insertions(+), 301 deletions(-)
 create mode 100644 modules/ignition/certs.tf
 create mode 100644 modules/ignition/resources/dropins/10-always-update-ca-certificates.conf

diff --git a/config.tf b/config.tf
index 300c75f5bb..b8b0ab56b0 100644
--- a/config.tf
+++ b/config.tf
@@ -11,7 +11,7 @@ provider "external" {
 }
 
 provider "ignition" {
-  version = "0.1.0"
+  version = "1.0.0"
 }
 
 provider "local" {
@@ -34,6 +34,14 @@ provider "tls" {
   version = "1.0.0"
 }
 
+locals {
+  // The total amount of public CA certificates present in Tectonic.
+  // That is all custom CAs + kube CA + etcd CA + ingress CA
+  // This is a local constant, which needs to be dependency inject because TF cannot handle length() on computed values,
+  // see https://github.com/hashicorp/terraform/issues/10857#issuecomment-268289775.
+  tectonic_ca_count = "${length(var.tectonic_custom_ca_pem_list) + 3}"
+}
+
 variable "tectonic_config_version" {
   description = <<EOF
 (internal) This declares the version of the global configuration variables.
@@ -512,3 +520,12 @@ variable "tectonic_kubelet_debug_config" {
 
   description = "(internal) debug flags for the kubelet (used in CI only)"
 }
+
+variable "tectonic_custom_ca_pem_list" {
+  type    = "list"
+  default = []
+
+  description = <<EOF
+(optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+EOF
+}
diff --git a/modules/aws/etcd/ignition.tf b/modules/aws/etcd/ignition.tf
index 5e262e6e04..8710f66f84 100644
--- a/modules/aws/etcd/ignition.tf
+++ b/modules/aws/etcd/ignition.tf
@@ -38,8 +38,8 @@ data "ignition_file" "etcd_tls_zip" {
 }
 
 data "ignition_systemd_unit" "etcd_unzip_tls" {
-  name   = "etcd-unzip-tls.service"
-  enable = true
+  name    = "etcd-unzip-tls.service"
+  enabled = true
 
   content = <<EOF
 [Unit]
@@ -60,8 +60,8 @@ EOF
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
-  name   = "locksmithd.service"
-  enable = true
+  name    = "locksmithd.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf
index 32fb06d753..9f15f37c7c 100644
--- a/modules/aws/master-asg/ignition.tf
+++ b/modules/aws/master-asg/ignition.tf
@@ -7,6 +7,7 @@ data "ignition_config" "main" {
     "${var.ign_installer_runtime_mappings_id}",
     "${var.ign_max_user_watches_id}",
     "${var.ign_s3_puller_id}",
+    "${data.ignition_file.ca_cert_pem.*.id}",
   ]
 
   systemd = ["${compact(list(
@@ -21,6 +22,7 @@ data "ignition_config" "main" {
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,
     var.ign_rm_assets_path_unit_id,
+    var.ign_update_ca_certificates_dropin_id,
    ))}"]
 }
 
@@ -81,3 +83,18 @@ data "ignition_file" "rm_assets" {
     content = "${data.template_file.rm_assets.rendered}"
   }
 }
+
+data "ignition_file" "ca_cert_pem" {
+  count = "${var.ign_ca_cert_list_count}"
+
+  filesystem = "root"
+  path       = "/etc/ssl/certs/ca_${count.index}.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  source {
+    source       = "${var.ign_ca_cert_s3_list[count.index]}"
+    verification = "sha512-${sha512(var.ign_ca_cert_pem_list[count.index])}"
+  }
+}
diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf
index cceaeb4e7c..65c7f91934 100644
--- a/modules/aws/master-asg/variables.tf
+++ b/modules/aws/master-asg/variables.tf
@@ -138,3 +138,18 @@ variable "ign_rm_assets_service_id" {
 variable "ign_rm_assets_path_unit_id" {
   type = "string"
 }
+
+variable "ign_ca_cert_s3_list" {
+  type        = "list"
+  description = "The list of CA certificates present on S3"
+}
+
+variable "ign_ca_cert_list_count" {
+  type        = "string"
+  description = "The length of ign_ca_cert_s3_list"
+}
+
+variable "ign_ca_cert_pem_list" {
+  type        = "list"
+  description = "The actual content of CA certificates present on S3. This is only used for hash calculation, not for provisioning."
+}
diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf
index eac9216079..07a2a9cd69 100644
--- a/modules/aws/worker-asg/ignition.tf
+++ b/modules/aws/worker-asg/ignition.tf
@@ -4,6 +4,7 @@ data "ignition_config" "main" {
     "${var.ign_installer_runtime_mappings_id}",
     "${var.ign_max_user_watches_id}",
     "${var.ign_s3_puller_id}",
+    "${data.ignition_file.ca_cert_pem.*.id}",
   ]
 
   systemd = [
@@ -11,5 +12,21 @@ data "ignition_config" "main" {
     "${var.ign_k8s_node_bootstrap_service_id}",
     "${var.ign_kubelet_service_id}",
     "${var.ign_locksmithd_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }
+
+data "ignition_file" "ca_cert_pem" {
+  count = "${var.ign_ca_cert_list_count}"
+
+  filesystem = "root"
+  path       = "/etc/ssl/certs/ca_${count.index}.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  source {
+    source       = "${var.ign_ca_cert_s3_list[count.index]}"
+    verification = "sha512-${sha512(var.ign_ca_cert_pem_list[count.index])}"
+  }
+}
diff --git a/modules/aws/worker-asg/variables.tf b/modules/aws/worker-asg/variables.tf
index 7730a0d2d4..7e3d8bfa92 100644
--- a/modules/aws/worker-asg/variables.tf
+++ b/modules/aws/worker-asg/variables.tf
@@ -82,3 +82,18 @@ variable "worker_iam_role" {
 variable "ign_s3_puller_id" {
   type = "string"
 }
+
+variable "ign_ca_cert_s3_list" {
+  type        = "list"
+  description = "The list of CA certificates present on S3"
+}
+
+variable "ign_ca_cert_list_count" {
+  type        = "string"
+  description = "The length of ign_ca_cert_s3_list"
+}
+
+variable "ign_ca_cert_pem_list" {
+  type        = "list"
+  description = "The actual content of CA certificates present on S3. This is only used for hash calculation, not for provisioning."
+}
diff --git a/modules/azure/etcd/ignition.tf b/modules/azure/etcd/ignition.tf
index b1c8f1ce58..5a07f2ab7f 100644
--- a/modules/azure/etcd/ignition.tf
+++ b/modules/azure/etcd/ignition.tf
@@ -128,8 +128,8 @@ data "ignition_user" "core" {
 data "ignition_systemd_unit" "locksmithd" {
   count = "${var.etcd_count}"
 
-  name   = "locksmithd.service"
-  enable = true
+  name    = "locksmithd.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf
index 4816be9a70..6be4d51793 100644
--- a/modules/azure/master-as/ignition-master.tf
+++ b/modules/azure/master-as/ignition-master.tf
@@ -6,6 +6,7 @@ data "ignition_config" "master" {
     "${var.ign_azure_udev_rules_id}",
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.cloud_provider_config.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = ["${compact(list(
@@ -18,6 +19,7 @@ data "ignition_config" "master" {
     var.ign_tectonic_service_id,
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,
+    var.ign_update_ca_certificates_dropin_id,
    ))}"]
 
   users = [
diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf
index f221f985a5..b22040ea80 100644
--- a/modules/azure/master-as/variables.tf
+++ b/modules/azure/master-as/variables.tf
@@ -93,3 +93,8 @@ variable "ign_tectonic_path_unit_id" {
 variable "fault_domains" {
   type = "string"
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf
index 2813121402..18635f2eae 100644
--- a/modules/azure/worker-as/ignition-worker.tf
+++ b/modules/azure/worker-as/ignition-worker.tf
@@ -6,6 +6,7 @@ data "ignition_config" "worker" {
     "${var.ign_azure_udev_rules_id}",
     "${var.ign_max_user_watches_id}",
     "${data.ignition_file.cloud-provider-config.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = [
@@ -14,6 +15,7 @@ data "ignition_config" "worker" {
     "${var.ign_k8s_node_bootstrap_service_id}",
     "${var.ign_kubelet_service_id}",
     "${var.ign_tx_off_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 
   users = [
diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf
index 9fa821ca24..eb84009c9e 100644
--- a/modules/azure/worker-as/variables.tf
+++ b/modules/azure/worker-as/variables.tf
@@ -86,3 +86,8 @@ variable "worker_count" {
 variable "fault_domains" {
   type = "string"
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/bootkube/assets.tf b/modules/bootkube/assets.tf
index d4f2abc9cd..00ec5c61bb 100644
--- a/modules/bootkube/assets.tf
+++ b/modules/bootkube/assets.tf
@@ -146,7 +146,7 @@ data "template_file" "bootkube_service" {
 
 data "ignition_systemd_unit" "bootkube_service" {
   name    = "bootkube.service"
-  enable  = false
+  enabled = false
   content = "${data.template_file.bootkube_service.rendered}"
 }
 
@@ -157,6 +157,6 @@ data "template_file" "bootkube_path_unit" {
 
 data "ignition_systemd_unit" "bootkube_path_unit" {
   name    = "bootkube.path"
-  enable  = true
+  enabled = true
   content = "${data.template_file.bootkube_path_unit.rendered}"
 }
diff --git a/modules/gcp/etcd/ignition.tf b/modules/gcp/etcd/ignition.tf
index 40be7da9d9..95d267647f 100644
--- a/modules/gcp/etcd/ignition.tf
+++ b/modules/gcp/etcd/ignition.tf
@@ -126,8 +126,8 @@ data "ignition_file" "etcd_peer_crt" {
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
-  name   = "locksmithd.service"
-  enable = true
+  name    = "locksmithd.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/gcp/master-igm/ignition.tf b/modules/gcp/master-igm/ignition.tf
index 3ab283f574..732c967a93 100644
--- a/modules/gcp/master-igm/ignition.tf
+++ b/modules/gcp/master-igm/ignition.tf
@@ -7,6 +7,7 @@ data "ignition_config" "main" {
     "${var.ign_gcs_puller_id}",
     "${data.ignition_file.init_assets.id}",
     "${data.ignition_file.detect_master.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = ["${compact(list(
@@ -19,6 +20,7 @@ data "ignition_config" "main" {
     var.ign_tectonic_service_id,
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,
+    var.ign_update_ca_certificates_dropin_id,
    ))}"]
 }
 
diff --git a/modules/gcp/master-igm/variables.tf b/modules/gcp/master-igm/variables.tf
index 232f01f647..66704141e8 100644
--- a/modules/gcp/master-igm/variables.tf
+++ b/modules/gcp/master-igm/variables.tf
@@ -100,3 +100,8 @@ variable "assets_gcs_location" {
   type        = "string"
   description = "Location on gcs of the Bootkube/Tectonic assets to use (bucket/key)"
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/gcp/worker-igm/ignition.tf b/modules/gcp/worker-igm/ignition.tf
index 929a810e17..68ab9baae0 100644
--- a/modules/gcp/worker-igm/ignition.tf
+++ b/modules/gcp/worker-igm/ignition.tf
@@ -4,6 +4,7 @@ data "ignition_config" "main" {
     "${var.ign_max_user_watches_id}",
     "${var.ign_installer_kubelet_env_id}",
     "${var.ign_installer_runtime_mappings_id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = [
@@ -11,6 +12,7 @@ data "ignition_config" "main" {
     "${var.ign_k8s_node_bootstrap_service_id}",
     "${var.ign_locksmithd_service_id}",
     "${var.ign_kubelet_service_id}",
+    "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }
 
diff --git a/modules/gcp/worker-igm/variables.tf b/modules/gcp/worker-igm/variables.tf
index e3de1ff321..193bb4c1bf 100644
--- a/modules/gcp/worker-igm/variables.tf
+++ b/modules/gcp/worker-igm/variables.tf
@@ -59,3 +59,8 @@ variable "public_ssh_key" {
 variable "kubeconfig_content" {
   type = "string"
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf
index 1e9b579b84..bf1676b05c 100644
--- a/modules/ignition/assets.tf
+++ b/modules/ignition/assets.tf
@@ -17,8 +17,8 @@ data "template_file" "docker_dropin" {
 }
 
 data "ignition_systemd_unit" "docker_dropin" {
-  name   = "docker.service"
-  enable = true
+  name    = "docker.service"
+  enabled = true
 
   dropin = [
     {
@@ -59,7 +59,7 @@ data "template_file" "kubelet" {
 
 data "ignition_systemd_unit" "kubelet" {
   name    = "kubelet.service"
-  enable  = true
+  enabled = true
   content = "${data.template_file.kubelet.rendered}"
 }
 
@@ -78,25 +78,25 @@ data "template_file" "k8s_node_bootstrap" {
 
 data "ignition_systemd_unit" "k8s_node_bootstrap" {
   name    = "k8s-node-bootstrap.service"
-  enable  = true
+  enabled = true
   content = "${data.template_file.k8s_node_bootstrap.rendered}"
 }
 
 data "ignition_systemd_unit" "init_assets" {
   name    = "init-assets.service"
-  enable  = "${var.assets_location != "" ? true : false}"
+  enabled = "${var.assets_location != "" ? true : false}"
   content = "${file("${path.module}/resources/services/init-assets.service")}"
 }
 
 data "ignition_systemd_unit" "rm_assets_path_unit" {
   name    = "rm-assets.path"
-  enable  = true
+  enabled = true
   content = "${file("${path.module}/resources/paths/rm-assets.path")}"
 }
 
 data "ignition_systemd_unit" "rm_assets" {
   name    = "rm-assets.service"
-  enable  = false
+  enabled = false
   content = "${file("${path.module}/resources/services/rm-assets.service")}"
 }
 
@@ -148,7 +148,7 @@ data "template_file" "tx_off" {
 
 data "ignition_systemd_unit" "tx_off" {
   name    = "tx-off.service"
-  enable  = true
+  enabled = true
   content = "${data.template_file.tx_off.rendered}"
 }
 
@@ -175,8 +175,8 @@ data "template_file" "coreos_metadata" {
 }
 
 data "ignition_systemd_unit" "coreos_metadata" {
-  name   = "coreos-metadata.service"
-  enable = true
+  name    = "coreos-metadata.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/ignition/certs.tf b/modules/ignition/certs.tf
new file mode 100644
index 0000000000..edf7d44f4e
--- /dev/null
+++ b/modules/ignition/certs.tf
@@ -0,0 +1,65 @@
+data "template_file" "update_ca_certificates_dropin" {
+  template = "${file("${path.module}/resources/dropins/10-always-update-ca-certificates.conf")}"
+}
+
+data "ignition_systemd_unit" "update_ca_certificates_dropin" {
+  name    = "update-ca-certificates.service"
+  enabled = true
+
+  dropin = [
+    {
+      name    = "10-always-update-ca-certificates.conf"
+      content = "${data.template_file.update_ca_certificates_dropin.rendered}"
+    },
+  ]
+}
+
+data "ignition_file" "kube_ca_cert_pem" {
+  filesystem = "root"
+  path       = "/etc/ssl/certs/kube_ca.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  content {
+    content = "${var.kube_ca_cert_pem}"
+  }
+}
+
+data "ignition_file" "etcd_ca_cert_pem" {
+  filesystem = "root"
+  path       = "/etc/ssl/certs/etcd_ca.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  content {
+    content = "${var.etcd_ca_cert_pem}"
+  }
+}
+
+data "ignition_file" "ingress_ca_cert_pem" {
+  filesystem = "root"
+  path       = "/etc/ssl/certs/ingress_ca.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  content {
+    content = "${var.ingress_ca_cert_pem}"
+  }
+}
+
+data "ignition_file" "custom_ca_cert_pem" {
+  count = "${length(var.custom_ca_cert_pem_list)}"
+
+  filesystem = "root"
+  path       = "/etc/ssl/certs/custom_ca_${count.index}.pem"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+
+  content {
+    content = "${var.custom_ca_cert_pem_list[count.index]}"
+  }
+}
diff --git a/modules/ignition/etcd.tf b/modules/ignition/etcd.tf
index 6ea1f4ac94..605a95541b 100644
--- a/modules/ignition/etcd.tf
+++ b/modules/ignition/etcd.tf
@@ -58,9 +58,9 @@ data "template_file" "etcd" {
 }
 
 data "ignition_systemd_unit" "etcd" {
-  count  = "${var.etcd_count}"
-  name   = "etcd-member.service"
-  enable = true
+  count   = "${var.etcd_count}"
+  name    = "etcd-member.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import
index 98a938565f..4db4bb9401 100644
--- a/modules/ignition/outputs.import
+++ b/modules/ignition/outputs.import
@@ -27,3 +27,7 @@ variable "ign_installer_kubelet_env_id" {
 variable "ign_k8s_node_bootstrap_service_id" {
   type = "string"
 }
+
+variable "ign_update_ca_certificates_dropin_id" {
+  type = "string"
+}
diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf
index cf20de3699..73e1ee853d 100644
--- a/modules/ignition/outputs.tf
+++ b/modules/ignition/outputs.tf
@@ -105,3 +105,29 @@ output "coreos_metadata_dropin_rendered" {
 output "gcs_puller_id" {
   value = "${data.ignition_file.gcs_puller.id}"
 }
+
+output "update_ca_certificates_dropin_id" {
+  value = "${data.ignition_systemd_unit.update_ca_certificates_dropin.id}"
+}
+
+output "update_ca_certificates_dropin_rendered" {
+  value = "${data.template_file.update_ca_certificates_dropin.rendered}"
+}
+
+output "ca_cert_id_list" {
+  value = [
+    "${data.ignition_file.kube_ca_cert_pem.id}",
+    "${data.ignition_file.ingress_ca_cert_pem.id}",
+    "${data.ignition_file.etcd_ca_cert_pem.id}",
+    "${data.ignition_file.custom_ca_cert_pem.*.id}",
+  ]
+}
+
+output "ca_cert_pem_list" {
+  value = [
+    "${var.kube_ca_cert_pem}",
+    "${var.ingress_ca_cert_pem}",
+    "${var.etcd_ca_cert_pem}",
+    "${var.custom_ca_cert_pem_list}",
+  ]
+}
diff --git a/modules/ignition/resources/dropins/10-always-update-ca-certificates.conf b/modules/ignition/resources/dropins/10-always-update-ca-certificates.conf
new file mode 100644
index 0000000000..7940085db2
--- /dev/null
+++ b/modules/ignition/resources/dropins/10-always-update-ca-certificates.conf
@@ -0,0 +1,6 @@
+[Unit]
+ConditionPathIsSymbolicLink=
+
+[Service]
+ExecStart=
+ExecStart=/usr/sbin/update-ca-certificates
diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf
index f731e12802..e0e70e2da2 100644
--- a/modules/ignition/variables.tf
+++ b/modules/ignition/variables.tf
@@ -117,3 +117,23 @@ variable "metadata_provider" {
 variable "use_metadata" {
   default = true
 }
+
+variable "kube_ca_cert_pem" {
+  type        = "string"
+  description = "The public kube CA certificate in PEM format."
+}
+
+variable "ingress_ca_cert_pem" {
+  type        = "string"
+  description = "The ingress kube CA certificate in PEM format."
+}
+
+variable "etcd_ca_cert_pem" {
+  type        = "string"
+  description = "The etcd kube CA certificate in PEM format."
+}
+
+variable "custom_ca_cert_pem_list" {
+  type        = "list"
+  description = "(optional) A list of custom CAs in PEM format."
+}
diff --git a/modules/openstack/etcd/ignition.tf b/modules/openstack/etcd/ignition.tf
index cb5590fd92..b6d3af9f2c 100644
--- a/modules/openstack/etcd/ignition.tf
+++ b/modules/openstack/etcd/ignition.tf
@@ -119,8 +119,8 @@ data "ignition_file" "resolv_conf" {
 data "ignition_systemd_unit" "locksmithd" {
   count = "${var.instance_count}"
 
-  name   = "locksmithd.service"
-  enable = true
+  name    = "locksmithd.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/openstack/nodes/ignition.tf b/modules/openstack/nodes/ignition.tf
index 071b0aab78..e55b92a741 100644
--- a/modules/openstack/nodes/ignition.tf
+++ b/modules/openstack/nodes/ignition.tf
@@ -13,6 +13,7 @@ data "ignition_config" "node" {
     "${data.ignition_file.resolv_conf.id}",
     "${data.ignition_file.hostname.*.id[count.index]}",
     "${data.ignition_file.sshd.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = ["${compact(list(
@@ -24,6 +25,7 @@ data "ignition_config" "node" {
     var.ign_tectonic_service_id,
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,
+    var.ign_update_ca_certificates_dropin_id,
    ))}"]
 }
 
diff --git a/modules/openstack/nodes/variables.tf b/modules/openstack/nodes/variables.tf
index 55151b8e00..82ff3f3cc1 100644
--- a/modules/openstack/nodes/variables.tf
+++ b/modules/openstack/nodes/variables.tf
@@ -47,3 +47,8 @@ variable "ign_tectonic_path_unit_id" {
   type    = "string"
   default = ""
 }
+
+variable "ign_ca_cert_id_list" 
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/tectonic/assets.tf b/modules/tectonic/assets.tf
index 97cefa6db4..f43fab7061 100644
--- a/modules/tectonic/assets.tf
+++ b/modules/tectonic/assets.tf
@@ -137,7 +137,7 @@ data "template_file" "tectonic_service" {
 
 data "ignition_systemd_unit" "tectonic_service" {
   name    = "tectonic.service"
-  enable  = false
+  enabled = false
   content = "${data.template_file.tectonic_service.rendered}"
 }
 
@@ -148,6 +148,6 @@ data "template_file" "tectonic_path" {
 
 data "ignition_systemd_unit" "tectonic_path" {
   name    = "tectonic.path"
-  enable  = true
+  enabled = true
   content = "${data.template_file.tectonic_path.rendered}"
 }
diff --git a/modules/vmware/etcd/ignition.tf b/modules/vmware/etcd/ignition.tf
index fe0dd835e5..d55a700250 100644
--- a/modules/vmware/etcd/ignition.tf
+++ b/modules/vmware/etcd/ignition.tf
@@ -118,8 +118,8 @@ data "ignition_file" "etcd_peer_crt" {
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
-  name   = "locksmithd.service"
-  enable = true
+  name    = "locksmithd.service"
+  enabled = true
 
   dropin = [
     {
diff --git a/modules/vmware/node/ignition.tf b/modules/vmware/node/ignition.tf
index 1ae5806f66..d54667e819 100644
--- a/modules/vmware/node/ignition.tf
+++ b/modules/vmware/node/ignition.tf
@@ -10,6 +10,7 @@ data "ignition_config" "node" {
     "${data.ignition_file.node_hostname.*.id[count.index]}",
     "${var.ign_installer_kubelet_env_id}",
     "${var.ign_installer_runtime_mappings_id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = ["${compact(list(
@@ -21,6 +22,7 @@ data "ignition_config" "node" {
     var.ign_tectonic_service_id,
     var.ign_bootkube_path_unit_id,
     var.ign_tectonic_path_unit_id,
+    var.ign_update_ca_certificates_dropin_id,
    ))}"]
 
   networkd = [
diff --git a/modules/vmware/node/variables.tf b/modules/vmware/node/variables.tf
index 72060b4152..1baa016719 100644
--- a/modules/vmware/node/variables.tf
+++ b/modules/vmware/node/variables.tf
@@ -124,3 +124,8 @@ variable "ign_tectonic_path_unit_id" {
   type    = "string"
   default = ""
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf
index 6bea5784ba..e8f107e43b 100644
--- a/platforms/aws/main.tf
+++ b/platforms/aws/main.tf
@@ -95,11 +95,15 @@ module "ignition_masters" {
   cloud_provider            = "aws"
   cluster_name              = "${var.tectonic_cluster_name}"
   container_images          = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
   etcd_count                = "${length(data.template_file.etcd_hostname_list.*.id)}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem          = "${module.kube_certs.ca_cert_pem}"
   kube_dns_service_ip       = "${module.bootkube.kube_dns_service_ip}"
   kubeconfig_fetch_cmd      = "/opt/s3-puller.sh ${aws_s3_bucket_object.kubeconfig.bucket}/${aws_s3_bucket_object.kubeconfig.key} /etc/kubernetes/kubeconfig"
   kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
@@ -112,89 +116,101 @@ module "ignition_masters" {
 module "masters" {
   source = "../../modules/aws/master-asg"
 
-  assets_s3_location                = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}"
-  autoscaling_group_extra_tags      = "${var.tectonic_autoscaling_group_extra_tags}"
-  aws_lbs                           = "${module.vpc.aws_lbs}"
-  base_domain                       = "${var.tectonic_base_domain}"
-  cluster_id                        = "${module.tectonic.cluster_id}"
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  container_images                  = "${var.tectonic_container_images}"
-  container_linux_channel           = "${var.tectonic_container_linux_channel}"
-  container_linux_version           = "${module.container_linux.version}"
-  ec2_type                          = "${var.tectonic_aws_master_ec2_type}"
-  extra_tags                        = "${var.tectonic_aws_extra_tags}"
-  ign_bootkube_path_unit_id         = "${module.bootkube.systemd_path_unit_id}"
-  ign_bootkube_service_id           = "${module.bootkube.systemd_service_id}"
-  ign_docker_dropin_id              = "${module.ignition_masters.docker_dropin_id}"
-  ign_init_assets_service_id        = "${module.ignition_masters.init_assets_service_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_masters.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_masters.max_user_watches_id}"
-  ign_rm_assets_path_unit_id        = "${module.ignition_masters.rm_assets_path_unit_id}"
-  ign_rm_assets_service_id          = "${module.ignition_masters.rm_assets_service_id}"
-  ign_s3_puller_id                  = "${module.ignition_masters.s3_puller_id}"
-  ign_tectonic_path_unit_id         = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
-  ign_tectonic_service_id           = "${module.tectonic.systemd_service_id}"
-  image_re                          = "${var.tectonic_image_re}"
-  instance_count                    = "${var.tectonic_master_count}"
-  master_iam_role                   = "${var.tectonic_aws_master_iam_role_name}"
-  master_sg_ids                     = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
-  private_endpoints                 = "${var.tectonic_aws_private_endpoints}"
-  public_endpoints                  = "${var.tectonic_aws_public_endpoints}"
-  root_volume_iops                  = "${var.tectonic_aws_master_root_volume_iops}"
-  root_volume_size                  = "${var.tectonic_aws_master_root_volume_size}"
-  root_volume_type                  = "${var.tectonic_aws_master_root_volume_type}"
-  ssh_key                           = "${var.tectonic_aws_ssh_key}"
-  subnet_ids                        = "${module.vpc.master_subnet_ids}"
+  assets_s3_location                   = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}"
+  autoscaling_group_extra_tags         = "${var.tectonic_autoscaling_group_extra_tags}"
+  aws_lbs                              = "${module.vpc.aws_lbs}"
+  base_domain                          = "${var.tectonic_base_domain}"
+  cluster_id                           = "${module.tectonic.cluster_id}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  container_images                     = "${var.tectonic_container_images}"
+  container_linux_channel              = "${var.tectonic_container_linux_channel}"
+  container_linux_version              = "${module.container_linux.version}"
+  ec2_type                             = "${var.tectonic_aws_master_ec2_type}"
+  extra_tags                           = "${var.tectonic_aws_extra_tags}"
+  ign_bootkube_path_unit_id            = "${module.bootkube.systemd_path_unit_id}"
+  ign_bootkube_service_id              = "${module.bootkube.systemd_service_id}"
+  ign_ca_cert_list_count               = "${local.tectonic_ca_count}"
+  ign_ca_cert_pem_list                 = "${module.ignition_masters.ca_cert_pem_list}"
+  ign_ca_cert_s3_list                  = "${local.ign_ca_cert_s3_list}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_init_assets_service_id           = "${module.ignition_masters.init_assets_service_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_masters.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_masters.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_rm_assets_path_unit_id           = "${module.ignition_masters.rm_assets_path_unit_id}"
+  ign_rm_assets_service_id             = "${module.ignition_masters.rm_assets_service_id}"
+  ign_s3_puller_id                     = "${module.ignition_masters.s3_puller_id}"
+  ign_tectonic_path_unit_id            = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+  ign_tectonic_service_id              = "${module.tectonic.systemd_service_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  image_re                             = "${var.tectonic_image_re}"
+  instance_count                       = "${var.tectonic_master_count}"
+  master_iam_role                      = "${var.tectonic_aws_master_iam_role_name}"
+  master_sg_ids                        = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
+  private_endpoints                    = "${var.tectonic_aws_private_endpoints}"
+  public_endpoints                     = "${var.tectonic_aws_public_endpoints}"
+  root_volume_iops                     = "${var.tectonic_aws_master_root_volume_iops}"
+  root_volume_size                     = "${var.tectonic_aws_master_root_volume_size}"
+  root_volume_type                     = "${var.tectonic_aws_master_root_volume_type}"
+  ssh_key                              = "${var.tectonic_aws_ssh_key}"
+  subnet_ids                           = "${module.vpc.master_subnet_ids}"
 }
 
 module "ignition_workers" {
   source = "../../modules/ignition"
 
-  bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
-  cloud_provider       = "aws"
-  container_images     = "${var.tectonic_container_images}"
-  image_re             = "${var.tectonic_image_re}"
-  kube_dns_service_ip  = "${module.bootkube.kube_dns_service_ip}"
-  kubeconfig_fetch_cmd = "/opt/s3-puller.sh ${aws_s3_bucket_object.kubeconfig.bucket}/${aws_s3_bucket_object.kubeconfig.key} /etc/kubernetes/kubeconfig"
-  kubelet_cni_bin_dir  = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label   = "node-role.kubernetes.io/node"
-  kubelet_node_taints  = ""
-  tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
+  bootstrap_upgrade_cl    = "${var.tectonic_bootstrap_upgrade_cl}"
+  cloud_provider          = "aws"
+  container_images        = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}"
+  etcd_ca_cert_pem        = "${module.etcd_certs.etcd_ca_crt_pem}"
+  image_re                = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem     = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem        = "${module.kube_certs.ca_cert_pem}"
+  kube_dns_service_ip     = "${module.bootkube.kube_dns_service_ip}"
+  kubeconfig_fetch_cmd    = "/opt/s3-puller.sh ${aws_s3_bucket_object.kubeconfig.bucket}/${aws_s3_bucket_object.kubeconfig.key} /etc/kubernetes/kubeconfig"
+  kubelet_cni_bin_dir     = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config    = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label      = "node-role.kubernetes.io/node"
+  kubelet_node_taints     = ""
+  tectonic_vanilla_k8s    = "${var.tectonic_vanilla_k8s}"
 }
 
 module "workers" {
   source = "../../modules/aws/worker-asg"
 
-  autoscaling_group_extra_tags      = "${var.tectonic_autoscaling_group_extra_tags}"
-  cluster_id                        = "${module.tectonic.cluster_id}"
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  container_linux_channel           = "${var.tectonic_container_linux_channel}"
-  container_linux_version           = "${module.container_linux.version}"
-  ec2_type                          = "${var.tectonic_aws_worker_ec2_type}"
-  extra_tags                        = "${var.tectonic_aws_extra_tags}"
-  ign_docker_dropin_id              = "${module.ignition_workers.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_workers.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_workers.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_workers.max_user_watches_id}"
-  ign_s3_puller_id                  = "${module.ignition_workers.s3_puller_id}"
-  instance_count                    = "${var.tectonic_worker_count}"
-  load_balancers                    = "${var.tectonic_aws_worker_load_balancers}"
-  root_volume_iops                  = "${var.tectonic_aws_worker_root_volume_iops}"
-  root_volume_size                  = "${var.tectonic_aws_worker_root_volume_size}"
-  root_volume_type                  = "${var.tectonic_aws_worker_root_volume_type}"
-  sg_ids                            = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}"
-  ssh_key                           = "${var.tectonic_aws_ssh_key}"
-  subnet_ids                        = "${module.vpc.worker_subnet_ids}"
-  vpc_id                            = "${module.vpc.vpc_id}"
-  worker_iam_role                   = "${var.tectonic_aws_worker_iam_role_name}"
+  autoscaling_group_extra_tags         = "${var.tectonic_autoscaling_group_extra_tags}"
+  cluster_id                           = "${module.tectonic.cluster_id}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  container_linux_channel              = "${var.tectonic_container_linux_channel}"
+  container_linux_version              = "${module.container_linux.version}"
+  ec2_type                             = "${var.tectonic_aws_worker_ec2_type}"
+  extra_tags                           = "${var.tectonic_aws_extra_tags}"
+  ign_ca_cert_list_count               = "${local.tectonic_ca_count}"
+  ign_ca_cert_pem_list                 = "${module.ignition_workers.ca_cert_pem_list}"
+  ign_ca_cert_s3_list                  = "${local.ign_ca_cert_s3_list}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_workers.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_workers.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_workers.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_s3_puller_id                     = "${module.ignition_workers.s3_puller_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  instance_count                       = "${var.tectonic_worker_count}"
+  load_balancers                       = "${var.tectonic_aws_worker_load_balancers}"
+  root_volume_iops                     = "${var.tectonic_aws_worker_root_volume_iops}"
+  root_volume_size                     = "${var.tectonic_aws_worker_root_volume_size}"
+  root_volume_type                     = "${var.tectonic_aws_worker_root_volume_type}"
+  sg_ids                               = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}"
+  ssh_key                              = "${var.tectonic_aws_ssh_key}"
+  subnet_ids                           = "${module.vpc.worker_subnet_ids}"
+  vpc_id                               = "${module.vpc.vpc_id}"
+  worker_iam_role                      = "${var.tectonic_aws_worker_iam_role_name}"
 }
 
 module "dns" {
diff --git a/platforms/aws/s3.tf b/platforms/aws/s3.tf
index 9fa6d7c154..789660fd35 100644
--- a/platforms/aws/s3.tf
+++ b/platforms/aws/s3.tf
@@ -63,3 +63,27 @@ resource "aws_s3_bucket_object" "kubeconfig" {
       "tectonicClusterID", "${module.tectonic.cluster_id}"
     ), var.tectonic_aws_extra_tags)}"
 }
+
+locals {
+  ign_ca_cert_s3_list = [
+    "${formatlist("s3://%s/%s", aws_s3_bucket_object.tectonic_assets.bucket, aws_s3_bucket_object.custom_ca_cert_pem.*.key)}",
+  ]
+}
+
+# custom CA certs
+resource "aws_s3_bucket_object" "custom_ca_cert_pem" {
+  count = "${local.tectonic_ca_count}"
+
+  bucket  = "${aws_s3_bucket.tectonic.bucket}"
+  key     = "custom_ca_${count.index}.pem"
+  content = "${module.ignition_masters.ca_cert_pem_list[count.index]}"
+  acl     = "private"
+
+  server_side_encryption = "AES256"
+
+  tags = "${merge(map(
+      "Name", "${var.tectonic_cluster_name}-custom-ca-${count.index}",
+      "KubernetesCluster", "${var.tectonic_cluster_name}",
+      "tectonicClusterID", "${module.tectonic.cluster_id}"
+    ), var.tectonic_aws_extra_tags)}"
+}
diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf
index 5085b4c35c..dc5d1e0ca0 100644
--- a/platforms/azure/main.tf
+++ b/platforms/azure/main.tf
@@ -116,11 +116,15 @@ module "ignition_masters" {
   cloud_provider_config     = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
   cluster_name              = "${var.tectonic_cluster_name}"
   container_images          = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_advertise_name_list.*.rendered}"
+  etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
   etcd_count                = "${local.etcd_count}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem          = "${module.kube_certs.ca_cert_pem}"
   kube_dns_service_ip       = "${module.bootkube.kube_dns_service_ip}"
   kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
   kubelet_debug_config      = "${var.tectonic_kubelet_debug_config}"
@@ -132,82 +136,90 @@ module "ignition_masters" {
 module "masters" {
   source = "../../modules/azure/master-as"
 
-  cloud_provider_config             = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
-  cluster_id                        = "${module.tectonic.cluster_id}"
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  container_linux_channel           = "${var.tectonic_container_linux_channel}"
-  container_linux_version           = "${module.container_linux.version}"
-  extra_tags                        = "${var.tectonic_azure_extra_tags}"
-  fault_domains                     = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
-  ign_azure_udev_rules_id           = "${module.ignition_masters.azure_udev_rules_id}"
-  ign_bootkube_path_unit_id         = "${module.bootkube.systemd_path_unit_id}"
-  ign_bootkube_service_id           = "${module.bootkube.systemd_service_id}"
-  ign_docker_dropin_id              = "${module.ignition_masters.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_masters.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_masters.max_user_watches_id}"
-  ign_tectonic_path_unit_id         = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
-  ign_tectonic_service_id           = "${module.tectonic.systemd_service_id}"
-  ign_tx_off_service_id             = "${module.ignition_masters.tx_off_service_id}"
-  kubeconfig_content                = "${module.bootkube.kubeconfig}"
-  location                          = "${var.tectonic_azure_location}"
-  master_count                      = "${var.tectonic_master_count}"
-  network_interface_ids             = "${module.vnet.master_network_interface_ids}"
-  public_ssh_key                    = "${var.tectonic_azure_ssh_key}"
-  resource_group_name               = "${module.resource_group.name}"
-  storage_id                        = "${module.resource_group.storage_id}"
-  storage_type                      = "${var.tectonic_azure_master_storage_type}"
-  vm_size                           = "${var.tectonic_azure_master_vm_size}"
+  cloud_provider_config                = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
+  cluster_id                           = "${module.tectonic.cluster_id}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  container_linux_channel              = "${var.tectonic_container_linux_channel}"
+  container_linux_version              = "${module.container_linux.version}"
+  extra_tags                           = "${var.tectonic_azure_extra_tags}"
+  fault_domains                        = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
+  ign_azure_udev_rules_id              = "${module.ignition_masters.azure_udev_rules_id}"
+  ign_bootkube_path_unit_id            = "${module.bootkube.systemd_path_unit_id}"
+  ign_bootkube_service_id              = "${module.bootkube.systemd_service_id}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_masters.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_masters.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_tectonic_path_unit_id            = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+  ign_tectonic_service_id              = "${module.tectonic.systemd_service_id}"
+  ign_tx_off_service_id                = "${module.ignition_masters.tx_off_service_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  kubeconfig_content                   = "${module.bootkube.kubeconfig}"
+  location                             = "${var.tectonic_azure_location}"
+  master_count                         = "${var.tectonic_master_count}"
+  network_interface_ids                = "${module.vnet.master_network_interface_ids}"
+  public_ssh_key                       = "${var.tectonic_azure_ssh_key}"
+  resource_group_name                  = "${module.resource_group.name}"
+  storage_id                           = "${module.resource_group.storage_id}"
+  storage_type                         = "${var.tectonic_azure_master_storage_type}"
+  vm_size                              = "${var.tectonic_azure_master_vm_size}"
 }
 
 module "ignition_workers" {
   source = "../../modules/ignition"
 
-  bootstrap_upgrade_cl  = "${var.tectonic_bootstrap_upgrade_cl}"
-  cloud_provider        = "azure"
-  cloud_provider_config = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
-  container_images      = "${var.tectonic_container_images}"
-  image_re              = "${var.tectonic_image_re}"
-  kube_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
-  kubelet_cni_bin_dir   = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config  = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label    = "node-role.kubernetes.io/node"
-  kubelet_node_taints   = ""
-  tectonic_vanilla_k8s  = "${var.tectonic_vanilla_k8s}"
+  bootstrap_upgrade_cl    = "${var.tectonic_bootstrap_upgrade_cl}"
+  cloud_provider          = "azure"
+  cloud_provider_config   = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
+  container_images        = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}"
+  etcd_ca_cert_pem        = "${module.etcd_certs.etcd_ca_crt_pem}"
+  ingress_ca_cert_pem     = "${module.ingress_certs.ca_cert_pem}"
+  image_re                = "${var.tectonic_image_re}"
+  kube_dns_service_ip     = "${module.bootkube.kube_dns_service_ip}"
+  kube_ca_cert_pem        = "${module.kube_certs.ca_cert_pem}"
+  kubelet_cni_bin_dir     = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config    = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label      = "node-role.kubernetes.io/node"
+  kubelet_node_taints     = ""
+  tectonic_vanilla_k8s    = "${var.tectonic_vanilla_k8s}"
 }
 
 module "workers" {
   source = "../../modules/azure/worker-as"
 
-  cloud_provider_config             = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
-  cluster_id                        = "${module.tectonic.cluster_id}"
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  container_linux_channel           = "${var.tectonic_container_linux_channel}"
-  container_linux_version           = "${module.container_linux.version}"
-  extra_tags                        = "${var.tectonic_azure_extra_tags}"
-  fault_domains                     = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
-  ign_azure_udev_rules_id           = "${module.ignition_workers.azure_udev_rules_id}"
-  ign_docker_dropin_id              = "${module.ignition_workers.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_workers.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_workers.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_workers.max_user_watches_id}"
-  ign_tx_off_service_id             = "${module.ignition_workers.tx_off_service_id}"
-  kubeconfig_content                = "${module.bootkube.kubeconfig}"
-  location                          = "${var.tectonic_azure_location}"
-  network_interface_ids             = "${module.vnet.worker_network_interface_ids}"
-  public_ssh_key                    = "${var.tectonic_azure_ssh_key}"
-  resource_group_name               = "${module.resource_group.name}"
-  storage_id                        = "${module.resource_group.storage_id}"
-  storage_type                      = "${var.tectonic_azure_worker_storage_type}"
-  tectonic_kube_dns_service_ip      = "${module.bootkube.kube_dns_service_ip}"
-  vm_size                           = "${var.tectonic_azure_worker_vm_size}"
-  worker_count                      = "${var.tectonic_worker_count}"
+  cloud_provider_config                = "${jsonencode(data.null_data_source.cloud_provider.inputs)}"
+  cluster_id                           = "${module.tectonic.cluster_id}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  container_linux_channel              = "${var.tectonic_container_linux_channel}"
+  container_linux_version              = "${module.container_linux.version}"
+  extra_tags                           = "${var.tectonic_azure_extra_tags}"
+  fault_domains                        = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
+  ign_azure_udev_rules_id              = "${module.ignition_workers.azure_udev_rules_id}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_workers.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_workers.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_workers.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_tx_off_service_id                = "${module.ignition_workers.tx_off_service_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  kubeconfig_content                   = "${module.bootkube.kubeconfig}"
+  location                             = "${var.tectonic_azure_location}"
+  network_interface_ids                = "${module.vnet.worker_network_interface_ids}"
+  public_ssh_key                       = "${var.tectonic_azure_ssh_key}"
+  resource_group_name                  = "${module.resource_group.name}"
+  storage_id                           = "${module.resource_group.storage_id}"
+  storage_type                         = "${var.tectonic_azure_worker_storage_type}"
+  tectonic_kube_dns_service_ip         = "${module.bootkube.kube_dns_service_ip}"
+  vm_size                              = "${var.tectonic_azure_worker_vm_size}"
+  worker_count                         = "${var.tectonic_worker_count}"
 }
 
 module "dns" {
diff --git a/platforms/metal/cl/bootkube-controller.yaml.tmpl b/platforms/metal/cl/bootkube-controller.yaml.tmpl
index 68d85b6924..f70a38960c 100644
--- a/platforms/metal/cl/bootkube-controller.yaml.tmpl
+++ b/platforms/metal/cl/bootkube-controller.yaml.tmpl
@@ -38,6 +38,11 @@ systemd:
     - name: bootkube.service
       enable: false
       contents: {{.ign_bootkube_service_json}}
+    - name: update-ca-certificates.service
+      enable: true
+      dropins:
+        - name: 10-alwaysrun.conf
+          contents: {{.ign_update_ca_certificates_dropin_json}}
 {{ if eq .exclude_tectonic "0" }}
     - name: tectonic.path
       enable: true
@@ -69,6 +74,11 @@ storage:
       mode: 0644
       contents:
         inline: {{.ign_installer_runtime_mappings_json}}
+    - path: /etc/ssl/certs/custom_ca_certs.pem
+      filesystem: root
+      mode: 0400
+      contents:
+        inline: {{.ign_custom_ca_certs_json}}
 passwd:
   users:
     - name: core
diff --git a/platforms/metal/cl/bootkube-worker.yaml.tmpl b/platforms/metal/cl/bootkube-worker.yaml.tmpl
index 72dd3a4e6e..846a3117e7 100644
--- a/platforms/metal/cl/bootkube-worker.yaml.tmpl
+++ b/platforms/metal/cl/bootkube-worker.yaml.tmpl
@@ -27,6 +27,11 @@ systemd:
     - name: kubelet.service
       enable: true
       contents: {{.ign_kubelet_service_json}}
+    - name: update-ca-certificates.service
+      enable: true
+      dropins:
+        - name: 10-alwaysrun.conf
+          contents: {{.ign_update_ca_certificates_dropin_json}}
 
 storage:
   files:
@@ -51,6 +56,11 @@ storage:
       mode: 0644
       contents:
         inline: {{.ign_max_user_watches_json}}
+    - path: /etc/ssl/certs/custom_ca_certs.pem
+      filesystem: root
+      mode: 0400
+      contents:
+        inline: {{.ign_custom_ca_certs_json}}
 passwd:
   users:
     - name: core
diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf
index d0365bfd69..76f86f2ca0 100644
--- a/platforms/metal/matchers.tf
+++ b/platforms/metal/matchers.tf
@@ -32,10 +32,14 @@ module "ignition_masters" {
   bootstrap_upgrade_cl      = "${var.tectonic_bootstrap_upgrade_cl}"
   cluster_name              = "${var.tectonic_cluster_name}"
   container_images          = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${var.tectonic_metal_controller_domains}"
+  etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
   etcd_count                = "${length(var.tectonic_metal_controller_names)}"
   etcd_initial_cluster_list = "${var.tectonic_metal_controller_domains}"
   image_re                  = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem          = "${module.kube_certs.ca_cert_pem}"
   kube_dns_service_ip       = "${module.bootkube.kube_dns_service_ip}"
   kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
   kubelet_debug_config      = "${var.tectonic_kubelet_debug_config}"
@@ -61,32 +65,38 @@ resource "matchbox_group" "controller" {
     exclude_tectonic   = "${var.tectonic_vanilla_k8s}"
     ssh_authorized_key = "${var.tectonic_ssh_authorized_key}"
 
-    ign_bootkube_path_unit_json         = "${jsonencode(module.bootkube.systemd_path_unit_rendered)}"
-    ign_bootkube_service_json           = "${jsonencode(module.bootkube.systemd_service_rendered)}"
-    ign_docker_dropin_json              = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}"
-    ign_etcd_dropin_json                = "${jsonencode(module.ignition_masters.etcd_dropin_rendered_list[count.index])}"
-    ign_installer_kubelet_env_json      = "${jsonencode(module.ignition_masters.installer_kubelet_env_rendered)}"
-    ign_installer_runtime_mappings_json = "${jsonencode(module.ignition_masters.installer_runtime_mappings_rendered)}"
-    ign_k8s_node_bootstrap_service_json = "${jsonencode(module.ignition_masters.k8s_node_bootstrap_service_rendered)}"
-    ign_kubelet_service_json            = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}"
-    ign_max_user_watches_json           = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}"
-    ign_tectonic_path_unit_json         = "${jsonencode(module.tectonic.systemd_path_unit_rendered)}"
-    ign_tectonic_service_json           = "${jsonencode(module.tectonic.systemd_service_rendered)}"
+    ign_bootkube_path_unit_json            = "${jsonencode(module.bootkube.systemd_path_unit_rendered)}"
+    ign_bootkube_service_json              = "${jsonencode(module.bootkube.systemd_service_rendered)}"
+    ign_custom_ca_certs_json               = "${jsonencode(join("\n", module.ignition_masters.ca_cert_pem_list))}"
+    ign_docker_dropin_json                 = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}"
+    ign_etcd_dropin_json                   = "${jsonencode(module.ignition_masters.etcd_dropin_rendered_list[count.index])}"
+    ign_installer_kubelet_env_json         = "${jsonencode(module.ignition_masters.installer_kubelet_env_rendered)}"
+    ign_installer_runtime_mappings_json    = "${jsonencode(module.ignition_masters.installer_runtime_mappings_rendered)}"
+    ign_k8s_node_bootstrap_service_json    = "${jsonencode(module.ignition_masters.k8s_node_bootstrap_service_rendered)}"
+    ign_kubelet_service_json               = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}"
+    ign_max_user_watches_json              = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}"
+    ign_tectonic_path_unit_json            = "${jsonencode(module.tectonic.systemd_path_unit_rendered)}"
+    ign_tectonic_service_json              = "${jsonencode(module.tectonic.systemd_service_rendered)}"
+    ign_update_ca_certificates_dropin_json = "${jsonencode(module.ignition_masters.update_ca_certificates_dropin_rendered)}"
   }
 }
 
 module "ignition_workers" {
   source = "../../modules/ignition"
 
-  bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
-  container_images     = "${var.tectonic_container_images}"
-  image_re             = "${var.tectonic_image_re}"
-  kube_dns_service_ip  = "${module.bootkube.kube_dns_service_ip}"
-  kubelet_cni_bin_dir  = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label   = "node-role.kubernetes.io/node"
-  kubelet_node_taints  = ""
-  tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
+  bootstrap_upgrade_cl    = "${var.tectonic_bootstrap_upgrade_cl}"
+  container_images        = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}"
+  etcd_ca_cert_pem        = "${module.etcd_certs.etcd_ca_crt_pem}"
+  image_re                = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem     = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem        = "${module.kube_certs.ca_cert_pem}"
+  kube_dns_service_ip     = "${module.bootkube.kube_dns_service_ip}"
+  kubelet_cni_bin_dir     = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config    = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label      = "node-role.kubernetes.io/node"
+  kubelet_node_taints     = ""
+  tectonic_vanilla_k8s    = "${var.tectonic_vanilla_k8s}"
 }
 
 resource "matchbox_group" "worker" {
@@ -108,11 +118,13 @@ resource "matchbox_group" "worker" {
     kubelet_image_tag  = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}"
     kube_version_image = "${var.tectonic_container_images["kube_version"]}"
 
-    ign_docker_dropin_json              = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}"
-    ign_installer_kubelet_env_json      = "${jsonencode(module.ignition_workers.installer_kubelet_env_rendered)}"
-    ign_installer_runtime_mappings_json = "${jsonencode(module.ignition_workers.installer_runtime_mappings_rendered)}"
-    ign_k8s_node_bootstrap_service_json = "${jsonencode(module.ignition_workers.k8s_node_bootstrap_service_rendered)}"
-    ign_kubelet_service_json            = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}"
-    ign_max_user_watches_json           = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}"
+    ign_custom_ca_certs_json               = "${jsonencode(join("\n", module.ignition_workers.ca_cert_pem_list))}"
+    ign_docker_dropin_json                 = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}"
+    ign_installer_kubelet_env_json         = "${jsonencode(module.ignition_workers.installer_kubelet_env_rendered)}"
+    ign_installer_runtime_mappings_json    = "${jsonencode(module.ignition_workers.installer_runtime_mappings_rendered)}"
+    ign_k8s_node_bootstrap_service_json    = "${jsonencode(module.ignition_workers.k8s_node_bootstrap_service_rendered)}"
+    ign_kubelet_service_json               = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}"
+    ign_max_user_watches_json              = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}"
+    ign_update_ca_certificates_dropin_json = "${jsonencode(module.ignition_workers.update_ca_certificates_dropin_rendered)}"
   }
 }
diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf
index ecd5078ce0..f544995aa9 100644
--- a/platforms/openstack/neutron/main.tf
+++ b/platforms/openstack/neutron/main.tf
@@ -177,11 +177,15 @@ module "ignition_masters" {
   bootstrap_upgrade_cl      = "${var.tectonic_bootstrap_upgrade_cl}"
   cluster_name              = "${var.tectonic_cluster_name}"
   container_images          = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
   etcd_count                = "${var.tectonic_etcd_count}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem          = "${module.kube_certs.ca_cert_pem}"
   kube_dns_service_ip       = "${module.bootkube.kube_dns_service_ip}"
   kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
   kubelet_debug_config      = "${var.tectonic_kubelet_debug_config}"
@@ -199,36 +203,42 @@ search ${var.tectonic_base_domain}
 ${join("\n", formatlist("nameserver %s", var.tectonic_openstack_dns_nameservers))}
 EOF
 
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  core_public_keys                  = ["${module.secrets.core_public_key_openssh}"]
-  hostname_infix                    = "master"
-  ign_bootkube_path_unit_id         = "${module.bootkube.systemd_path_unit_id}"
-  ign_bootkube_service_id           = "${module.bootkube.systemd_service_id}"
-  ign_docker_dropin_id              = "${module.ignition_masters.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_masters.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_masters.max_user_watches_id}"
-  ign_tectonic_path_unit_id         = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
-  ign_tectonic_service_id           = "${module.tectonic.systemd_service_id}"
-  instance_count                    = "${var.tectonic_master_count}"
-  kubeconfig_content                = "${module.bootkube.kubeconfig}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  core_public_keys                     = ["${module.secrets.core_public_key_openssh}"]
+  hostname_infix                       = "master"
+  ign_bootkube_path_unit_id            = "${module.bootkube.systemd_path_unit_id}"
+  ign_bootkube_service_id              = "${module.bootkube.systemd_service_id}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_masters.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_masters.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_tectonic_path_unit_id            = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+  ign_tectonic_service_id              = "${module.tectonic.systemd_service_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  instance_count                       = "${var.tectonic_master_count}"
+  kubeconfig_content                   = "${module.bootkube.kubeconfig}"
 }
 
 module "ignition_workers" {
   source = "../../../modules/ignition"
 
-  bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
-  container_images     = "${var.tectonic_container_images}"
-  image_re             = "${var.tectonic_image_re}"
-  kube_dns_service_ip  = "${module.bootkube.kube_dns_service_ip}"
-  kubelet_cni_bin_dir  = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label   = "node-role.kubernetes.io/node"
-  kubelet_node_taints  = ""
-  tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
+  bootstrap_upgrade_cl    = "${var.tectonic_bootstrap_upgrade_cl}"
+  container_images        = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}"
+  etcd_ca_cert_pem        = "${module.etcd_certs.etcd_ca_crt_pem}"
+  image_re                = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem     = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem        = "${module.kube_certs.ca_cert_pem}"
+  kube_dns_service_ip     = "${module.bootkube.kube_dns_service_ip}"
+  kubelet_cni_bin_dir     = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config    = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label      = "node-role.kubernetes.io/node"
+  kubelet_node_taints     = ""
+  tectonic_vanilla_k8s    = "${var.tectonic_vanilla_k8s}"
 }
 
 module "worker_nodes" {
@@ -239,18 +249,20 @@ search ${var.tectonic_base_domain}
 ${join("\n", formatlist("nameserver %s", var.tectonic_openstack_dns_nameservers))}
 EOF
 
-  cluster_name                      = "${var.tectonic_cluster_name}"
-  core_public_keys                  = ["${module.secrets.core_public_key_openssh}"]
-  hostname_infix                    = "worker"
-  ign_docker_dropin_id              = "${module.ignition_workers.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_workers.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_workers.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_workers.max_user_watches_id}"
-  instance_count                    = "${var.tectonic_worker_count}"
-  kubeconfig_content                = "${module.bootkube.kubeconfig}"
+  cluster_name                         = "${var.tectonic_cluster_name}"
+  core_public_keys                     = ["${module.secrets.core_public_key_openssh}"]
+  hostname_infix                       = "worker"
+  ign_ca_cert_id_list                  = "${module.ignition_workers.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_workers.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_workers.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_workers.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  instance_count                       = "${var.tectonic_worker_count}"
+  kubeconfig_content                   = "${module.bootkube.kubeconfig}"
 }
 
 module "secrets" {
diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf
index 2c273e6aee..c60917d01c 100644
--- a/platforms/vmware/main.tf
+++ b/platforms/vmware/main.tf
@@ -43,9 +43,13 @@ module "ignition_masters" {
   bootstrap_upgrade_cl     = "${var.tectonic_bootstrap_upgrade_cl}"
   cluster_name             = "${var.tectonic_cluster_name}"
   container_images         = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list  = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_ca_cert_pem         = "${module.etcd_certs.etcd_ca_crt_pem}"
   etcd_count               = "${length(data.template_file.etcd_hostname_list.*.rendered)}"
   image_re                 = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem      = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem         = "${module.kube_certs.ca_cert_pem}"
   kube_dns_service_ip      = "${module.bootkube.kube_dns_service_ip}"
   kubelet_cni_bin_dir      = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
   kubelet_debug_config     = "${var.tectonic_kubelet_debug_config}"
@@ -57,83 +61,91 @@ module "ignition_masters" {
 module "masters" {
   source = "../../modules/vmware/node"
 
-  base_domain                       = "${var.tectonic_base_domain}"
-  container_images                  = "${var.tectonic_container_images}"
-  core_public_keys                  = ["${var.tectonic_vmware_ssh_authorized_key}"]
-  dns_server                        = "${var.tectonic_vmware_node_dns}"
-  gateways                          = "${var.tectonic_vmware_master_gateways}"
-  hostname                          = "${var.tectonic_vmware_master_hostnames}"
-  ign_bootkube_path_unit_id         = "${module.bootkube.systemd_path_unit_id}"
-  ign_bootkube_service_id           = "${module.bootkube.systemd_service_id}"
-  ign_docker_dropin_id              = "${module.ignition_masters.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_masters.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_masters.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_masters.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_masters.max_user_watches_id}"
-  ign_tectonic_path_unit_id         = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
-  ign_tectonic_service_id           = "${module.tectonic.systemd_service_id}"
-  image_re                          = "${var.tectonic_image_re}"
-  instance_count                    = "${var.tectonic_master_count}"
-  ip_address                        = "${var.tectonic_vmware_master_ip}"
-  kubeconfig                        = "${module.bootkube.kubeconfig}"
-  private_key                       = "${var.tectonic_vmware_ssh_private_key_path}"
-  vm_disk_datastore                 = "${var.tectonic_vmware_master_datastore}"
-  vm_disk_template                  = "${var.tectonic_vmware_vm_template}"
-  vm_disk_template_folder           = "${var.tectonic_vmware_vm_template_folder}"
-  vm_memory                         = "${var.tectonic_vmware_master_memory}"
-  vm_network_labels                 = "${var.tectonic_vmware_master_networks}"
-  vm_vcpu                           = "${var.tectonic_vmware_master_vcpu}"
-  vmware_clusters                   = "${var.tectonic_vmware_master_clusters}"
-  vmware_datacenters                = "${var.tectonic_vmware_master_datacenters}"
-  vmware_folder                     = "${vsphere_folder.tectonic_vsphere_folder.path}"
-  vmware_resource_pool              = "${var.tectonic_vmware_master_resource_pool}"
+  base_domain                          = "${var.tectonic_base_domain}"
+  container_images                     = "${var.tectonic_container_images}"
+  core_public_keys                     = ["${var.tectonic_vmware_ssh_authorized_key}"]
+  dns_server                           = "${var.tectonic_vmware_node_dns}"
+  gateways                             = "${var.tectonic_vmware_master_gateways}"
+  hostname                             = "${var.tectonic_vmware_master_hostnames}"
+  ign_bootkube_path_unit_id            = "${module.bootkube.systemd_path_unit_id}"
+  ign_bootkube_service_id              = "${module.bootkube.systemd_service_id}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_masters.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_masters.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_masters.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_masters.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_masters.max_user_watches_id}"
+  ign_tectonic_path_unit_id            = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+  ign_tectonic_service_id              = "${module.tectonic.systemd_service_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}"
+  image_re                             = "${var.tectonic_image_re}"
+  instance_count                       = "${var.tectonic_master_count}"
+  ip_address                           = "${var.tectonic_vmware_master_ip}"
+  kubeconfig                           = "${module.bootkube.kubeconfig}"
+  private_key                          = "${var.tectonic_vmware_ssh_private_key_path}"
+  vm_disk_datastore                    = "${var.tectonic_vmware_master_datastore}"
+  vm_disk_template                     = "${var.tectonic_vmware_vm_template}"
+  vm_disk_template_folder              = "${var.tectonic_vmware_vm_template_folder}"
+  vm_memory                            = "${var.tectonic_vmware_master_memory}"
+  vm_network_labels                    = "${var.tectonic_vmware_master_networks}"
+  vm_vcpu                              = "${var.tectonic_vmware_master_vcpu}"
+  vmware_clusters                      = "${var.tectonic_vmware_master_clusters}"
+  vmware_datacenters                   = "${var.tectonic_vmware_master_datacenters}"
+  vmware_folder                        = "${vsphere_folder.tectonic_vsphere_folder.path}"
+  vmware_resource_pool                 = "${var.tectonic_vmware_master_resource_pool}"
 }
 
 module "ignition_workers" {
   source = "../../modules/ignition"
 
-  bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
-  container_images     = "${var.tectonic_container_images}"
-  image_re             = "${var.tectonic_image_re}"
-  kube_dns_service_ip  = "${module.bootkube.kube_dns_service_ip}"
-  kubelet_cni_bin_dir  = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label   = "node-role.kubernetes.io/node"
-  kubelet_node_taints  = ""
-  tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
+  bootstrap_upgrade_cl    = "${var.tectonic_bootstrap_upgrade_cl}"
+  container_images        = "${var.tectonic_container_images}"
+  custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}"
+  etcd_ca_cert_pem        = "${module.etcd_certs.etcd_ca_crt_pem}"
+  image_re                = "${var.tectonic_image_re}"
+  ingress_ca_cert_pem     = "${module.ingress_certs.ca_cert_pem}"
+  kube_ca_cert_pem        = "${module.kube_certs.ca_cert_pem}"
+  kube_dns_service_ip     = "${module.bootkube.kube_dns_service_ip}"
+  kubelet_cni_bin_dir     = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config    = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label      = "node-role.kubernetes.io/node"
+  kubelet_node_taints     = ""
+  tectonic_vanilla_k8s    = "${var.tectonic_vanilla_k8s}"
 }
 
 module "workers" {
   source = "../../modules/vmware/node"
 
-  base_domain                       = "${var.tectonic_base_domain}"
-  container_images                  = "${var.tectonic_container_images}"
-  core_public_keys                  = ["${var.tectonic_vmware_ssh_authorized_key}"]
-  dns_server                        = "${var.tectonic_vmware_node_dns}"
-  gateways                          = "${var.tectonic_vmware_worker_gateways}"
-  hostname                          = "${var.tectonic_vmware_worker_hostnames}"
-  ign_docker_dropin_id              = "${module.ignition_workers.docker_dropin_id}"
-  ign_installer_kubelet_env_id      = "${module.ignition_workers.installer_kubelet_env_id}"
-  ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}"
-  ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
-  ign_kubelet_service_id            = "${module.ignition_workers.kubelet_service_id}"
-  ign_locksmithd_service_id         = "${module.ignition_workers.locksmithd_service_id}"
-  ign_max_user_watches_id           = "${module.ignition_workers.max_user_watches_id}"
-  image_re                          = "${var.tectonic_image_re}"
-  instance_count                    = "${var.tectonic_worker_count}"
-  ip_address                        = "${var.tectonic_vmware_worker_ip}"
-  kubeconfig                        = "${module.bootkube.kubeconfig}"
-  private_key                       = "${var.tectonic_vmware_ssh_private_key_path}"
-  vm_disk_datastore                 = "${var.tectonic_vmware_worker_datastore}"
-  vm_disk_template                  = "${var.tectonic_vmware_vm_template}"
-  vm_disk_template_folder           = "${var.tectonic_vmware_vm_template_folder}"
-  vm_memory                         = "${var.tectonic_vmware_worker_memory}"
-  vm_network_labels                 = "${var.tectonic_vmware_worker_networks}"
-  vm_vcpu                           = "${var.tectonic_vmware_worker_vcpu}"
-  vmware_clusters                   = "${var.tectonic_vmware_worker_clusters}"
-  vmware_datacenters                = "${var.tectonic_vmware_worker_datacenters}"
-  vmware_folder                     = "${vsphere_folder.tectonic_vsphere_folder.path}"
-  vmware_resource_pool              = "${var.tectonic_vmware_worker_resource_pool}"
+  base_domain                          = "${var.tectonic_base_domain}"
+  container_images                     = "${var.tectonic_container_images}"
+  core_public_keys                     = ["${var.tectonic_vmware_ssh_authorized_key}"]
+  dns_server                           = "${var.tectonic_vmware_node_dns}"
+  gateways                             = "${var.tectonic_vmware_worker_gateways}"
+  hostname                             = "${var.tectonic_vmware_worker_hostnames}"
+  ign_ca_cert_id_list                  = "${module.ignition_workers.ca_cert_id_list}"
+  ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
+  ign_installer_kubelet_env_id         = "${module.ignition_workers.installer_kubelet_env_id}"
+  ign_installer_runtime_mappings_id    = "${module.ignition_workers.installer_runtime_mappings_id}"
+  ign_k8s_node_bootstrap_service_id    = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+  ign_kubelet_service_id               = "${module.ignition_workers.kubelet_service_id}"
+  ign_locksmithd_service_id            = "${module.ignition_workers.locksmithd_service_id}"
+  ign_max_user_watches_id              = "${module.ignition_workers.max_user_watches_id}"
+  ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}"
+  image_re                             = "${var.tectonic_image_re}"
+  instance_count                       = "${var.tectonic_worker_count}"
+  ip_address                           = "${var.tectonic_vmware_worker_ip}"
+  kubeconfig                           = "${module.bootkube.kubeconfig}"
+  private_key                          = "${var.tectonic_vmware_ssh_private_key_path}"
+  vm_disk_datastore                    = "${var.tectonic_vmware_worker_datastore}"
+  vm_disk_template                     = "${var.tectonic_vmware_vm_template}"
+  vm_disk_template_folder              = "${var.tectonic_vmware_vm_template_folder}"
+  vm_memory                            = "${var.tectonic_vmware_worker_memory}"
+  vm_network_labels                    = "${var.tectonic_vmware_worker_networks}"
+  vm_vcpu                              = "${var.tectonic_vmware_worker_vcpu}"
+  vmware_clusters                      = "${var.tectonic_vmware_worker_clusters}"
+  vmware_datacenters                   = "${var.tectonic_vmware_worker_datacenters}"
+  vmware_folder                        = "${vsphere_folder.tectonic_vsphere_folder.path}"
+  vmware_resource_pool                 = "${var.tectonic_vmware_worker_resource_pool}"
 }

From c8e828a4786274f2859d4a54a0c38eb3b994dd27 Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:11:51 +0100
Subject: [PATCH 2/9] modules/ignition: unify etcd certificate generation

Currently etcd certificate ignition files are created in their
respective platform module.

This unifies it by declaring the ignition file units centrally in the
ignition module.
---
 modules/ignition/{certs.tf => ca_certs.tf} |  0
 modules/ignition/etcd.tf                   | 94 ++++++++++++++++++++++
 modules/ignition/outputs.import            |  5 ++
 modules/ignition/outputs.tf                | 12 +++
 modules/ignition/variables.tf              | 24 ++++++
 5 files changed, 135 insertions(+)
 rename modules/ignition/{certs.tf => ca_certs.tf} (100%)

diff --git a/modules/ignition/certs.tf b/modules/ignition/ca_certs.tf
similarity index 100%
rename from modules/ignition/certs.tf
rename to modules/ignition/ca_certs.tf
diff --git a/modules/ignition/etcd.tf b/modules/ignition/etcd.tf
index 605a95541b..f5596e440c 100644
--- a/modules/ignition/etcd.tf
+++ b/modules/ignition/etcd.tf
@@ -69,3 +69,97 @@ data "ignition_systemd_unit" "etcd" {
     },
   ]
 }
+
+data "ignition_file" "etcd_ca" {
+  count = "${var.etcd_count > 0 ? 1 : 0}"
+
+  path       = "/etc/ssl/etcd/ca.crt"
+  mode       = 0644
+  uid        = 232
+  gid        = 232
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_ca_cert_pem}"
+  }
+}
+
+data "ignition_file" "etcd_client_key" {
+  path       = "/etc/ssl/etcd/client.key"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_client_key_pem}"
+  }
+}
+
+data "ignition_file" "etcd_client_crt" {
+  path       = "/etc/ssl/etcd/client.crt"
+  mode       = 0400
+  uid        = 0
+  gid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_client_crt_pem}"
+  }
+}
+
+data "ignition_file" "etcd_server_key" {
+  count = "${var.etcd_count > 0 ? 1 : 0}"
+
+  path       = "/etc/ssl/etcd/server.key"
+  mode       = 0400
+  uid        = 232
+  gid        = 232
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_server_key_pem}"
+  }
+}
+
+data "ignition_file" "etcd_server_crt" {
+  count = "${var.etcd_count > 0 ? 1 : 0}"
+
+  path       = "/etc/ssl/etcd/server.crt"
+  mode       = 0400
+  uid        = 232
+  gid        = 232
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_server_crt_pem}"
+  }
+}
+
+data "ignition_file" "etcd_peer_key" {
+  count = "${var.etcd_count > 0 ? 1 : 0}"
+
+  path       = "/etc/ssl/etcd/peer.key"
+  mode       = 0400
+  uid        = 232
+  gid        = 232
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_peer_key_pem}"
+  }
+}
+
+data "ignition_file" "etcd_peer_crt" {
+  count = "${var.etcd_count > 0 ? 1 : 0}"
+
+  path       = "/etc/ssl/etcd/peer.crt"
+  mode       = 0400
+  uid        = 232
+  gid        = 232
+  filesystem = "root"
+
+  content {
+    content = "${var.etcd_peer_crt_pem}"
+  }
+}
diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import
index 4db4bb9401..80ca72dd8f 100644
--- a/modules/ignition/outputs.import
+++ b/modules/ignition/outputs.import
@@ -31,3 +31,8 @@ variable "ign_k8s_node_bootstrap_service_id" {
 variable "ign_update_ca_certificates_dropin_id" {
   type = "string"
 }
+
+variable "ign_ca_cert_id_list" {
+  type        = "list"
+  description = "The list of public CA certificate ignition file IDs."
+}
diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf
index 73e1ee853d..c7fc62e174 100644
--- a/modules/ignition/outputs.tf
+++ b/modules/ignition/outputs.tf
@@ -131,3 +131,15 @@ output "ca_cert_pem_list" {
     "${var.custom_ca_cert_pem_list}",
   ]
 }
+
+output "etcd_crt_id_list" {
+  value = [
+    "${data.ignition_file.etcd_ca.*.id}",
+    "${data.ignition_file.etcd_client_key.*.id}",
+    "${data.ignition_file.etcd_client_crt.*.id}",
+    "${data.ignition_file.etcd_server_key.*.id}",
+    "${data.ignition_file.etcd_server_crt.*.id}",
+    "${data.ignition_file.etcd_peer_key.*.id}",
+    "${data.ignition_file.etcd_peer_crt.*.id}",
+  ]
+}
diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf
index e0e70e2da2..98d3c01852 100644
--- a/modules/ignition/variables.tf
+++ b/modules/ignition/variables.tf
@@ -133,6 +133,30 @@ variable "etcd_ca_cert_pem" {
   description = "The etcd kube CA certificate in PEM format."
 }
 
+variable "etcd_client_key_pem" {
+  default = ""
+}
+
+variable "etcd_client_crt_pem" {
+  default = ""
+}
+
+variable "etcd_server_key_pem" {
+  default = ""
+}
+
+variable "etcd_server_crt_pem" {
+  default = ""
+}
+
+variable "etcd_peer_key_pem" {
+  default = ""
+}
+
+variable "etcd_peer_crt_pem" {
+  default = ""
+}
+
 variable "custom_ca_cert_pem_list" {
   type        = "list"
   description = "(optional) A list of custom CAs in PEM format."

From 9c133abe70553500c7ad44fa75d7910b752f7e2d Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:13:08 +0100
Subject: [PATCH 3/9] modules/tls/etcd: remove zip generation

Currently the etcd TLS module also generates a zip file which is only
used on AWS to reduce the userdata size to be <20k (hard limit on AWS).

Since the etcd TLS assets will be provisioned via S3 this optimization/hack
is not needed any more.
---
 modules/tls/etcd/signed/outputs.tf        |  7 ----
 modules/tls/etcd/signed/zip.tf            | 40 -----------------------
 modules/tls/etcd/user-provided/outputs.tf |  7 ----
 modules/tls/etcd/user-provided/zip.tf     | 40 -----------------------
 4 files changed, 94 deletions(-)
 delete mode 100644 modules/tls/etcd/signed/zip.tf
 delete mode 100644 modules/tls/etcd/user-provided/zip.tf

diff --git a/modules/tls/etcd/signed/outputs.tf b/modules/tls/etcd/signed/outputs.tf
index f883c7371b..436d803830 100644
--- a/modules/tls/etcd/signed/outputs.tf
+++ b/modules/tls/etcd/signed/outputs.tf
@@ -26,15 +26,8 @@ output "etcd_server_key_pem" {
   value = "${join("", tls_private_key.etcd_server.*.private_key_pem)}"
 }
 
-// The data.archive_file.etcd_tls_zip.id != "" assertion forces the etcd_tls_zip datasource to be run,
-// hence ./.terraform/etcd_tls.zip must be generated and present.
-output "etcd_tls_zip" {
-  value = "${data.archive_file.etcd_tls_zip.id != "" ? file("./.terraform/etcd_tls.zip") : ""}"
-}
-
 output "id" {
   value = "${sha1("
-  ${data.archive_file.etcd_tls_zip.id},
   ${join(" ",
     local_file.etcd_ca_crt.*.id,
     local_file.etcd_server_crt.*.id,
diff --git a/modules/tls/etcd/signed/zip.tf b/modules/tls/etcd/signed/zip.tf
deleted file mode 100644
index 247c2c9a99..0000000000
--- a/modules/tls/etcd/signed/zip.tf
+++ /dev/null
@@ -1,40 +0,0 @@
-data "archive_file" "etcd_tls_zip" {
-  type = "zip"
-
-  output_path = "./.terraform/etcd_tls.zip"
-
-  source {
-    filename = "ca.crt"
-    content  = "${data.template_file.etcd_ca_cert_pem.rendered}"
-  }
-
-  source {
-    filename = "server.crt"
-    content  = "${join("", tls_locally_signed_cert.etcd_server.*.cert_pem)}"
-  }
-
-  source {
-    filename = "server.key"
-    content  = "${join("", tls_private_key.etcd_server.*.private_key_pem)}"
-  }
-
-  source {
-    filename = "peer.crt"
-    content  = "${join("", tls_locally_signed_cert.etcd_peer.*.cert_pem)}"
-  }
-
-  source {
-    filename = "peer.key"
-    content  = "${join("", tls_private_key.etcd_peer.*.private_key_pem)}"
-  }
-
-  source {
-    filename = "client.crt"
-    content  = "${data.template_file.etcd_client_crt.rendered}"
-  }
-
-  source {
-    filename = "client.key"
-    content  = "${data.template_file.etcd_client_key.rendered}"
-  }
-}
diff --git a/modules/tls/etcd/user-provided/outputs.tf b/modules/tls/etcd/user-provided/outputs.tf
index ed8c9fbeb5..52aae973ad 100644
--- a/modules/tls/etcd/user-provided/outputs.tf
+++ b/modules/tls/etcd/user-provided/outputs.tf
@@ -26,15 +26,8 @@ output "etcd_server_key_pem" {
   value = "${file(var.etcd_server_key_pem_path)}"
 }
 
-// The data.archive_file.etcd_tls_zip.id != "" assertion forces the etcd_tls_zip datasource to be run,
-// hence ./.terraform/etcd_tls.zip must be generated and present.
-output "etcd_tls_zip" {
-  value = "${data.archive_file.etcd_tls_zip.id != "" ? file("./.terraform/etcd_tls.zip") : ""}"
-}
-
 output "id" {
   value = "${sha1("
-  ${data.archive_file.etcd_tls_zip.id},
   ${join(" ",
     local_file.etcd_ca_crt.*.id,
     local_file.etcd_server_crt.*.id,
diff --git a/modules/tls/etcd/user-provided/zip.tf b/modules/tls/etcd/user-provided/zip.tf
deleted file mode 100644
index 357126114b..0000000000
--- a/modules/tls/etcd/user-provided/zip.tf
+++ /dev/null
@@ -1,40 +0,0 @@
-data "archive_file" "etcd_tls_zip" {
-  type = "zip"
-
-  output_path = "./.terraform/etcd_tls.zip"
-
-  source {
-    filename = "ca.crt"
-    content  = "${file(var.etcd_ca_crt_pem_path)}"
-  }
-
-  source {
-    filename = "server.crt"
-    content  = "${file(var.etcd_server_crt_pem_path)}"
-  }
-
-  source {
-    filename = "server.key"
-    content  = "${file(var.etcd_server_key_pem_path)}"
-  }
-
-  source {
-    filename = "peer.crt"
-    content  = "${file(var.etcd_peer_crt_pem_path)}"
-  }
-
-  source {
-    filename = "peer.key"
-    content  = "${file(var.etcd_peer_key_pem_path)}"
-  }
-
-  source {
-    filename = "client.crt"
-    content  = "${file(var.etcd_client_crt_pem_path)}"
-  }
-
-  source {
-    filename = "client.key"
-    content  = "${file(var.etcd_client_key_pem_path)}"
-  }
-}

From fa0f3bd1c33706732b228267128aeace97c1d417 Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:14:51 +0100
Subject: [PATCH 4/9] */aws: use S3 for ignition provisioning

We hit the limits of the AWS userdata limit (20k) constantly.

This fixes it by provisioning a minimal ignition configuration only
which points to a replacement ignition configuration hosted on S3.

This also removes workarounds/hacks to keep the userdata size small,
especially for provisioning TLS assets on etcd nodes.
---
 modules/aws/etcd/ignition.tf          | 48 +----------------
 modules/aws/etcd/ignition_s3.tf       | 25 +++++++++
 modules/aws/etcd/nodes.tf             | 78 ++++++++++++++++++++++++++-
 modules/aws/etcd/variables.tf         |  8 ++-
 modules/aws/master-asg/ignition.tf    | 17 +-----
 modules/aws/master-asg/ignition_s3.tf | 21 ++++++++
 modules/aws/master-asg/master.tf      |  2 +-
 modules/aws/master-asg/variables.tf   | 15 +-----
 modules/aws/worker-asg/ignition.tf    | 17 +-----
 modules/aws/worker-asg/ignition_s3.tf | 21 ++++++++
 modules/aws/worker-asg/variables.tf   | 15 +-----
 modules/aws/worker-asg/worker.tf      |  2 +-
 platforms/aws/main.tf                 | 21 +++++---
 platforms/aws/s3.tf                   | 24 ---------
 14 files changed, 171 insertions(+), 143 deletions(-)
 create mode 100644 modules/aws/etcd/ignition_s3.tf
 create mode 100644 modules/aws/master-asg/ignition_s3.tf
 create mode 100644 modules/aws/worker-asg/ignition_s3.tf

diff --git a/modules/aws/etcd/ignition.tf b/modules/aws/etcd/ignition.tf
index 8710f66f84..bb29af7749 100644
--- a/modules/aws/etcd/ignition.tf
+++ b/modules/aws/etcd/ignition.tf
@@ -4,59 +4,13 @@ data "ignition_config" "etcd" {
   systemd = [
     "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}",
     "${var.ign_etcd_dropin_id_list[count.index]}",
-    "${data.ignition_systemd_unit.etcd_unzip_tls.id}",
   ]
 
   files = [
-    "${data.ignition_file.node_hostname.*.id[count.index]}",
-    "${data.ignition_file.etcd_tls_zip.id}",
+    "${var.ign_etcd_crt_id_list}",
   ]
 }
 
-data "ignition_file" "node_hostname" {
-  count      = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
-  path       = "/etc/hostname"
-  mode       = 0644
-  filesystem = "root"
-
-  content {
-    content = "${var.cluster_name}-etcd-${count.index}.${var.base_domain}"
-  }
-}
-
-data "ignition_file" "etcd_tls_zip" {
-  path       = "/etc/ssl/etcd/tls.zip"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    mime    = "application/octet-stream"
-    content = "${var.tls_zip}"
-  }
-}
-
-data "ignition_systemd_unit" "etcd_unzip_tls" {
-  name    = "etcd-unzip-tls.service"
-  enabled = true
-
-  content = <<EOF
-[Unit]
-ConditionPathExists=!/etc/ssl/etcd/ca.crt
-[Service]
-Type=oneshot
-WorkingDirectory=/etc/ssl/etcd
-ExecStart=/usr/bin/bash -c 'unzip /etc/ssl/etcd/tls.zip && \
-chown etcd:etcd /etc/ssl/etcd/peer.* && \
-chown etcd:etcd /etc/ssl/etcd/server.* && \
-chmod 0400 /etc/ssl/etcd/peer.* /etc/ssl/etcd/server.* /etc/ssl/etcd/client.*'
-[Install]
-WantedBy=multi-user.target
-RequiredBy=etcd-member.service locksmithd.service
-EOF
-}
-
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
diff --git a/modules/aws/etcd/ignition_s3.tf b/modules/aws/etcd/ignition_s3.tf
new file mode 100644
index 0000000000..b0b98f1373
--- /dev/null
+++ b/modules/aws/etcd/ignition_s3.tf
@@ -0,0 +1,25 @@
+resource "aws_s3_bucket_object" "ignition_etcd" {
+  count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
+
+  bucket  = "${var.s3_bucket}"
+  key     = "ignition_etcd_${count.index}.json"
+  content = "${data.ignition_config.etcd.*.rendered[count.index]}"
+  acl     = "private"
+
+  server_side_encryption = "AES256"
+
+  tags = "${merge(map(
+      "Name", "${var.cluster_name}-ignition-etcd-${count.index}",
+      "KubernetesCluster", "${var.cluster_name}",
+      "tectonicClusterID", "${var.cluster_id}"
+    ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+  count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
+
+  replace {
+    source       = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_etcd.*.key[count.index])}"
+    verification = "sha512-${sha512(data.ignition_config.etcd.*.rendered[count.index])}"
+  }
+}
diff --git a/modules/aws/etcd/nodes.tf b/modules/aws/etcd/nodes.tf
index 80bc959658..446da092c8 100644
--- a/modules/aws/etcd/nodes.tf
+++ b/modules/aws/etcd/nodes.tf
@@ -20,14 +20,88 @@ data "aws_ami" "coreos_ami" {
   }
 }
 
+resource "aws_iam_role" "etcd" {
+  count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+  name  = "${var.cluster_name}-etcd-role"
+  path  = "/"
+
+  assume_role_policy = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": "sts:AssumeRole",
+            "Principal": {
+                "Service": "ec2.amazonaws.com"
+            },
+            "Effect": "Allow",
+            "Sid": ""
+        }
+    ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "etcd" {
+  count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+  name  = "${var.cluster_name}_etcd_policy"
+  role  = "${aws_iam_role.etcd.id}"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:AttachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:DetachVolume",
+      "Resource": "*"
+    },
+    {
+      "Action": [
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": "*",
+      "Effect": "Allow"
+    },
+    {
+      "Action" : [
+        "s3:GetObject"
+      ],
+      "Resource": "arn:aws:s3:::*",
+      "Effect": "Allow"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_instance_profile" "etcd" {
+  count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+  name  = "${var.cluster_name}-etcd-profile"
+  role  = "${aws_iam_role.etcd.name}"
+}
+
 resource "aws_instance" "etcd_node" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
   ami   = "${data.aws_ami.coreos_ami.image_id}"
 
+  iam_instance_profile   = "${aws_iam_instance_profile.etcd.name}"
   instance_type          = "${var.ec2_type}"
-  subnet_id              = "${element(var.subnets, count.index)}"
   key_name               = "${var.ssh_key}"
-  user_data              = "${data.ignition_config.etcd.*.rendered[count.index]}"
+  subnet_id              = "${element(var.subnets, count.index)}"
+  user_data              = "${data.ignition_config.s3.*.rendered[count.index]}"
   vpc_security_group_ids = ["${var.sg_ids}"]
 
   lifecycle {
diff --git a/modules/aws/etcd/variables.tf b/modules/aws/etcd/variables.tf
index 8f74170440..cdf031dd3d 100644
--- a/modules/aws/etcd/variables.tf
+++ b/modules/aws/etcd/variables.tf
@@ -73,10 +73,14 @@ variable "tls_enabled" {
   default = false
 }
 
-variable "tls_zip" {
+variable "ign_etcd_dropin_id_list" {
+  type = "list"
+}
+
+variable "s3_bucket" {
   type = "string"
 }
 
-variable "ign_etcd_dropin_id_list" {
+variable "ign_etcd_crt_id_list" {
   type = "list"
 }
diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf
index 9f15f37c7c..632d300ba4 100644
--- a/modules/aws/master-asg/ignition.tf
+++ b/modules/aws/master-asg/ignition.tf
@@ -7,7 +7,7 @@ data "ignition_config" "main" {
     "${var.ign_installer_runtime_mappings_id}",
     "${var.ign_max_user_watches_id}",
     "${var.ign_s3_puller_id}",
-    "${data.ignition_file.ca_cert_pem.*.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = ["${compact(list(
@@ -83,18 +83,3 @@ data "ignition_file" "rm_assets" {
     content = "${data.template_file.rm_assets.rendered}"
   }
 }
-
-data "ignition_file" "ca_cert_pem" {
-  count = "${var.ign_ca_cert_list_count}"
-
-  filesystem = "root"
-  path       = "/etc/ssl/certs/ca_${count.index}.pem"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-
-  source {
-    source       = "${var.ign_ca_cert_s3_list[count.index]}"
-    verification = "sha512-${sha512(var.ign_ca_cert_pem_list[count.index])}"
-  }
-}
diff --git a/modules/aws/master-asg/ignition_s3.tf b/modules/aws/master-asg/ignition_s3.tf
new file mode 100644
index 0000000000..a6e5a6ca00
--- /dev/null
+++ b/modules/aws/master-asg/ignition_s3.tf
@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_object" "ignition_master" {
+  bucket  = "${var.s3_bucket}"
+  key     = "ignition_master.json"
+  content = "${data.ignition_config.main.rendered}"
+  acl     = "private"
+
+  server_side_encryption = "AES256"
+
+  tags = "${merge(map(
+      "Name", "${var.cluster_name}-ignition-master",
+      "KubernetesCluster", "${var.cluster_name}",
+      "tectonicClusterID", "${var.cluster_id}"
+    ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+  replace {
+    source       = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_master.key)}"
+    verification = "sha512-${sha512(data.ignition_config.main.rendered)}"
+  }
+}
diff --git a/modules/aws/master-asg/master.tf b/modules/aws/master-asg/master.tf
index afc83084b2..fa3fbab130 100644
--- a/modules/aws/master-asg/master.tf
+++ b/modules/aws/master-asg/master.tf
@@ -62,7 +62,7 @@ resource "aws_launch_configuration" "master_conf" {
   security_groups             = ["${var.master_sg_ids}"]
   iam_instance_profile        = "${aws_iam_instance_profile.master_profile.arn}"
   associate_public_ip_address = "${var.public_endpoints}"
-  user_data                   = "${data.ignition_config.main.rendered}"
+  user_data                   = "${data.ignition_config.s3.rendered}"
 
   lifecycle {
     create_before_destroy = true
diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf
index 65c7f91934..f269fd425b 100644
--- a/modules/aws/master-asg/variables.tf
+++ b/modules/aws/master-asg/variables.tf
@@ -139,17 +139,6 @@ variable "ign_rm_assets_path_unit_id" {
   type = "string"
 }
 
-variable "ign_ca_cert_s3_list" {
-  type        = "list"
-  description = "The list of CA certificates present on S3"
-}
-
-variable "ign_ca_cert_list_count" {
-  type        = "string"
-  description = "The length of ign_ca_cert_s3_list"
-}
-
-variable "ign_ca_cert_pem_list" {
-  type        = "list"
-  description = "The actual content of CA certificates present on S3. This is only used for hash calculation, not for provisioning."
+variable "s3_bucket" {
+  type = "string"
 }
diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf
index 07a2a9cd69..7c0dcca517 100644
--- a/modules/aws/worker-asg/ignition.tf
+++ b/modules/aws/worker-asg/ignition.tf
@@ -4,7 +4,7 @@ data "ignition_config" "main" {
     "${var.ign_installer_runtime_mappings_id}",
     "${var.ign_max_user_watches_id}",
     "${var.ign_s3_puller_id}",
-    "${data.ignition_file.ca_cert_pem.*.id}",
+    "${var.ign_ca_cert_id_list}",
   ]
 
   systemd = [
@@ -15,18 +15,3 @@ data "ignition_config" "main" {
     "${var.ign_update_ca_certificates_dropin_id}",
   ]
 }
-
-data "ignition_file" "ca_cert_pem" {
-  count = "${var.ign_ca_cert_list_count}"
-
-  filesystem = "root"
-  path       = "/etc/ssl/certs/ca_${count.index}.pem"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-
-  source {
-    source       = "${var.ign_ca_cert_s3_list[count.index]}"
-    verification = "sha512-${sha512(var.ign_ca_cert_pem_list[count.index])}"
-  }
-}
diff --git a/modules/aws/worker-asg/ignition_s3.tf b/modules/aws/worker-asg/ignition_s3.tf
new file mode 100644
index 0000000000..9766c6ba5b
--- /dev/null
+++ b/modules/aws/worker-asg/ignition_s3.tf
@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_object" "ignition_worker" {
+  bucket  = "${var.s3_bucket}"
+  key     = "ignition_worker.json"
+  content = "${data.ignition_config.main.rendered}"
+  acl     = "private"
+
+  server_side_encryption = "AES256"
+
+  tags = "${merge(map(
+      "Name", "${var.cluster_name}-ignition-worker",
+      "KubernetesCluster", "${var.cluster_name}",
+      "tectonicClusterID", "${var.cluster_id}"
+    ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+  replace {
+    source       = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_worker.key)}"
+    verification = "sha512-${sha512(data.ignition_config.main.rendered)}"
+  }
+}
diff --git a/modules/aws/worker-asg/variables.tf b/modules/aws/worker-asg/variables.tf
index 7e3d8bfa92..bdbddfc0ff 100644
--- a/modules/aws/worker-asg/variables.tf
+++ b/modules/aws/worker-asg/variables.tf
@@ -83,17 +83,6 @@ variable "ign_s3_puller_id" {
   type = "string"
 }
 
-variable "ign_ca_cert_s3_list" {
-  type        = "list"
-  description = "The list of CA certificates present on S3"
-}
-
-variable "ign_ca_cert_list_count" {
-  type        = "string"
-  description = "The length of ign_ca_cert_s3_list"
-}
-
-variable "ign_ca_cert_pem_list" {
-  type        = "list"
-  description = "The actual content of CA certificates present on S3. This is only used for hash calculation, not for provisioning."
+variable "s3_bucket" {
+  type = "string"
 }
diff --git a/modules/aws/worker-asg/worker.tf b/modules/aws/worker-asg/worker.tf
index 58a7a9d986..72e46cd3c7 100644
--- a/modules/aws/worker-asg/worker.tf
+++ b/modules/aws/worker-asg/worker.tf
@@ -27,7 +27,7 @@ resource "aws_launch_configuration" "worker_conf" {
   key_name             = "${var.ssh_key}"
   security_groups      = ["${var.sg_ids}"]
   iam_instance_profile = "${aws_iam_instance_profile.worker_profile.arn}"
-  user_data            = "${data.ignition_config.main.rendered}"
+  user_data            = "${data.ignition_config.s3.rendered}"
 
   lifecycle {
     create_before_destroy = true
diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf
index e8f107e43b..27f9050f01 100644
--- a/platforms/aws/main.tf
+++ b/platforms/aws/main.tf
@@ -74,16 +74,17 @@ module "etcd" {
   ec2_type                = "${var.tectonic_aws_etcd_ec2_type}"
   external_endpoints      = "${compact(var.tectonic_etcd_servers)}"
   extra_tags              = "${var.tectonic_aws_extra_tags}"
+  ign_etcd_crt_id_list    = "${module.ignition_masters.etcd_crt_id_list}"
+  ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
   instance_count          = "${length(data.template_file.etcd_hostname_list.*.id)}"
   root_volume_iops        = "${var.tectonic_aws_etcd_root_volume_iops}"
   root_volume_size        = "${var.tectonic_aws_etcd_root_volume_size}"
   root_volume_type        = "${var.tectonic_aws_etcd_root_volume_type}"
+  s3_bucket               = "${aws_s3_bucket.tectonic.bucket}"
   sg_ids                  = "${concat(var.tectonic_aws_etcd_extra_sg_ids, list(module.vpc.etcd_sg_id))}"
   ssh_key                 = "${var.tectonic_aws_ssh_key}"
   subnets                 = "${module.vpc.worker_subnet_ids}"
   tls_enabled             = "${var.tectonic_etcd_tls_enabled}"
-  tls_zip                 = "${module.etcd_certs.etcd_tls_zip}"
-  ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
 }
 
 module "ignition_masters" {
@@ -98,8 +99,14 @@ module "ignition_masters" {
   custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
+  etcd_client_crt_pem       = "${module.etcd_certs.etcd_client_crt_pem}"
+  etcd_client_key_pem       = "${module.etcd_certs.etcd_client_key_pem}"
   etcd_count                = "${length(data.template_file.etcd_hostname_list.*.id)}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_peer_crt_pem         = "${module.etcd_certs.etcd_peer_crt_pem}"
+  etcd_peer_key_pem         = "${module.etcd_certs.etcd_peer_key_pem}"
+  etcd_server_crt_pem       = "${module.etcd_certs.etcd_server_crt_pem}"
+  etcd_server_key_pem       = "${module.etcd_certs.etcd_server_key_pem}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
   ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"
@@ -129,9 +136,7 @@ module "masters" {
   extra_tags                           = "${var.tectonic_aws_extra_tags}"
   ign_bootkube_path_unit_id            = "${module.bootkube.systemd_path_unit_id}"
   ign_bootkube_service_id              = "${module.bootkube.systemd_service_id}"
-  ign_ca_cert_list_count               = "${local.tectonic_ca_count}"
-  ign_ca_cert_pem_list                 = "${module.ignition_masters.ca_cert_pem_list}"
-  ign_ca_cert_s3_list                  = "${local.ign_ca_cert_s3_list}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
   ign_docker_dropin_id                 = "${module.ignition_masters.docker_dropin_id}"
   ign_init_assets_service_id           = "${module.ignition_masters.init_assets_service_id}"
   ign_installer_kubelet_env_id         = "${module.ignition_masters.installer_kubelet_env_id}"
@@ -155,6 +160,7 @@ module "masters" {
   root_volume_iops                     = "${var.tectonic_aws_master_root_volume_iops}"
   root_volume_size                     = "${var.tectonic_aws_master_root_volume_size}"
   root_volume_type                     = "${var.tectonic_aws_master_root_volume_type}"
+  s3_bucket                            = "${aws_s3_bucket.tectonic.bucket}"
   ssh_key                              = "${var.tectonic_aws_ssh_key}"
   subnet_ids                           = "${module.vpc.master_subnet_ids}"
 }
@@ -189,9 +195,7 @@ module "workers" {
   container_linux_version              = "${module.container_linux.version}"
   ec2_type                             = "${var.tectonic_aws_worker_ec2_type}"
   extra_tags                           = "${var.tectonic_aws_extra_tags}"
-  ign_ca_cert_list_count               = "${local.tectonic_ca_count}"
-  ign_ca_cert_pem_list                 = "${module.ignition_workers.ca_cert_pem_list}"
-  ign_ca_cert_s3_list                  = "${local.ign_ca_cert_s3_list}"
+  ign_ca_cert_id_list                  = "${module.ignition_masters.ca_cert_id_list}"
   ign_docker_dropin_id                 = "${module.ignition_workers.docker_dropin_id}"
   ign_installer_kubelet_env_id         = "${module.ignition_workers.installer_kubelet_env_id}"
   ign_installer_runtime_mappings_id    = "${module.ignition_workers.installer_runtime_mappings_id}"
@@ -206,6 +210,7 @@ module "workers" {
   root_volume_iops                     = "${var.tectonic_aws_worker_root_volume_iops}"
   root_volume_size                     = "${var.tectonic_aws_worker_root_volume_size}"
   root_volume_type                     = "${var.tectonic_aws_worker_root_volume_type}"
+  s3_bucket                            = "${aws_s3_bucket.tectonic.bucket}"
   sg_ids                               = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}"
   ssh_key                              = "${var.tectonic_aws_ssh_key}"
   subnet_ids                           = "${module.vpc.worker_subnet_ids}"
diff --git a/platforms/aws/s3.tf b/platforms/aws/s3.tf
index 789660fd35..9fa6d7c154 100644
--- a/platforms/aws/s3.tf
+++ b/platforms/aws/s3.tf
@@ -63,27 +63,3 @@ resource "aws_s3_bucket_object" "kubeconfig" {
       "tectonicClusterID", "${module.tectonic.cluster_id}"
     ), var.tectonic_aws_extra_tags)}"
 }
-
-locals {
-  ign_ca_cert_s3_list = [
-    "${formatlist("s3://%s/%s", aws_s3_bucket_object.tectonic_assets.bucket, aws_s3_bucket_object.custom_ca_cert_pem.*.key)}",
-  ]
-}
-
-# custom CA certs
-resource "aws_s3_bucket_object" "custom_ca_cert_pem" {
-  count = "${local.tectonic_ca_count}"
-
-  bucket  = "${aws_s3_bucket.tectonic.bucket}"
-  key     = "custom_ca_${count.index}.pem"
-  content = "${module.ignition_masters.ca_cert_pem_list[count.index]}"
-  acl     = "private"
-
-  server_side_encryption = "AES256"
-
-  tags = "${merge(map(
-      "Name", "${var.tectonic_cluster_name}-custom-ca-${count.index}",
-      "KubernetesCluster", "${var.tectonic_cluster_name}",
-      "tectonicClusterID", "${module.tectonic.cluster_id}"
-    ), var.tectonic_aws_extra_tags)}"
-}

From 046c277e7b9f9da36ff1b32900adfd256596673a Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:18:00 +0100
Subject: [PATCH 5/9] */azure: use unified etcd TLS ignition files

---
 modules/azure/etcd/ignition.tf       | 102 +--------------------------
 modules/azure/etcd/variables.tf      |  32 ++-------
 modules/azure/master-as/variables.tf |   5 --
 modules/azure/worker-as/variables.tf |   5 --
 platforms/azure/main.tf              |  19 +++--
 5 files changed, 14 insertions(+), 149 deletions(-)

diff --git a/modules/azure/etcd/ignition.tf b/modules/azure/etcd/ignition.tf
index 5a07f2ab7f..2a6fb1e848 100644
--- a/modules/azure/etcd/ignition.tf
+++ b/modules/azure/etcd/ignition.tf
@@ -11,110 +11,10 @@ data "ignition_config" "etcd" {
   ]
 
   files = [
-    "${data.ignition_file.etcd_ca.id}",
-    "${data.ignition_file.etcd_server_crt.id}",
-    "${data.ignition_file.etcd_server_key.id}",
-    "${data.ignition_file.etcd_client_crt.id}",
-    "${data.ignition_file.etcd_client_key.id}",
-    "${data.ignition_file.etcd_peer_crt.id}",
-    "${data.ignition_file.etcd_peer_key.id}",
+    "${var.ign_etcd_crt_id_list}",
   ]
 }
 
-data "ignition_file" "etcd_ca" {
-  count = "${var.etcd_count > 0 ? 1 : 0}"
-
-  path       = "/etc/ssl/etcd/ca.crt"
-  mode       = 0644
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_ca_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_key" {
-  path       = "/etc/ssl/etcd/client.key"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_crt" {
-  path       = "/etc/ssl/etcd/client.crt"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_key" {
-  count = "${var.etcd_count > 0 ? 1 : 0}"
-
-  path       = "/etc/ssl/etcd/server.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_crt" {
-  count = "${var.etcd_count > 0 ? 1 : 0}"
-
-  path       = "/etc/ssl/etcd/server.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_key" {
-  count = "${var.etcd_count > 0 ? 1 : 0}"
-
-  path       = "/etc/ssl/etcd/peer.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_crt" {
-  count = "${var.etcd_count > 0 ? 1 : 0}"
-
-  path       = "/etc/ssl/etcd/peer.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_crt_pem}"
-  }
-}
-
 data "ignition_user" "core" {
   count = "${var.etcd_count > 0 ? 1 : 0}"
 
diff --git a/modules/azure/etcd/variables.tf b/modules/azure/etcd/variables.tf
index 9a683f6493..d9298361b3 100644
--- a/modules/azure/etcd/variables.tf
+++ b/modules/azure/etcd/variables.tf
@@ -72,34 +72,6 @@ variable "tls_enabled" {
   default = false
 }
 
-variable "tls_ca_crt_pem" {
-  default = ""
-}
-
-variable "tls_client_key_pem" {
-  default = ""
-}
-
-variable "tls_client_crt_pem" {
-  default = ""
-}
-
-variable "tls_server_key_pem" {
-  default = ""
-}
-
-variable "tls_server_crt_pem" {
-  default = ""
-}
-
-variable "tls_peer_key_pem" {
-  default = ""
-}
-
-variable "tls_peer_crt_pem" {
-  default = ""
-}
-
 variable "container_image" {
   type = "string"
 }
@@ -115,3 +87,7 @@ variable "ign_etcd_dropin_id_list" {
 variable "fault_domains" {
   type = "string"
 }
+
+variable "ign_etcd_crt_id_list" {
+  type = "list"
+}
diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf
index b22040ea80..f221f985a5 100644
--- a/modules/azure/master-as/variables.tf
+++ b/modules/azure/master-as/variables.tf
@@ -93,8 +93,3 @@ variable "ign_tectonic_path_unit_id" {
 variable "fault_domains" {
   type = "string"
 }
-
-variable "ign_ca_cert_id_list" {
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf
index eb84009c9e..9fa821ca24 100644
--- a/modules/azure/worker-as/variables.tf
+++ b/modules/azure/worker-as/variables.tf
@@ -86,8 +86,3 @@ variable "worker_count" {
 variable "fault_domains" {
   type = "string"
 }
-
-variable "ign_ca_cert_id_list" {
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/platforms/azure/main.tf b/platforms/azure/main.tf
index dc5d1e0ca0..8d72997e8c 100644
--- a/platforms/azure/main.tf
+++ b/platforms/azure/main.tf
@@ -64,25 +64,18 @@ module "etcd" {
   container_linux_version = "${module.container_linux.version}"
   etcd_count              = "${local.etcd_count}"
   extra_tags              = "${var.tectonic_azure_extra_tags}"
+  fault_domains           = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
+  ign_etcd_crt_id_list    = "${module.ignition_masters.etcd_crt_id_list}"
+  ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
   location                = "${var.tectonic_azure_location}"
   network_interface_ids   = "${module.vnet.etcd_network_interface_ids}"
   public_ssh_key          = "${var.tectonic_azure_ssh_key}"
   resource_group_name     = "${module.resource_group.name}"
   storage_id              = "${module.resource_group.storage_id}"
   storage_type            = "${var.tectonic_azure_etcd_storage_type}"
-  tls_ca_crt_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
-  tls_client_crt_pem      = "${module.etcd_certs.etcd_client_crt_pem}"
-  tls_client_key_pem      = "${module.etcd_certs.etcd_client_key_pem}"
   tls_enabled             = "${var.tectonic_etcd_tls_enabled}"
-  tls_peer_crt_pem        = "${module.etcd_certs.etcd_peer_crt_pem}"
-  tls_peer_key_pem        = "${module.etcd_certs.etcd_peer_key_pem}"
-  tls_server_crt_pem      = "${module.etcd_certs.etcd_server_crt_pem}"
-  tls_server_key_pem      = "${module.etcd_certs.etcd_server_key_pem}"
   versions                = "${var.tectonic_versions}"
   vm_size                 = "${var.tectonic_azure_etcd_vm_size}"
-  fault_domains           = "${var.tectonic_azure_location_fault_domains["${var.tectonic_azure_location}"]}"
-
-  ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
 }
 
 # Workaround for https://github.com/hashicorp/terraform/issues/4084
@@ -119,8 +112,14 @@ module "ignition_masters" {
   custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_advertise_name_list.*.rendered}"
   etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
+  etcd_client_crt_pem       = "${module.etcd_certs.etcd_client_crt_pem}"
+  etcd_client_key_pem       = "${module.etcd_certs.etcd_client_key_pem}"
   etcd_count                = "${local.etcd_count}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_peer_crt_pem         = "${module.etcd_certs.etcd_peer_crt_pem}"
+  etcd_peer_key_pem         = "${module.etcd_certs.etcd_peer_key_pem}"
+  etcd_server_crt_pem       = "${module.etcd_certs.etcd_server_crt_pem}"
+  etcd_server_key_pem       = "${module.etcd_certs.etcd_server_key_pem}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
   ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"

From 532ff5b4ad524963623dfb9932ec1122ee3b6147 Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:19:14 +0100
Subject: [PATCH 6/9] */gcp: use unified etcd TLS ignition files

---
 modules/gcp/etcd/ignition.tf        | 102 +---------------------------
 modules/gcp/etcd/variables.tf       |  32 ++-------
 modules/gcp/master-igm/variables.tf |   5 --
 modules/gcp/worker-igm/variables.tf |   5 --
 platforms/gcp/main.tf               |  31 ++++-----
 5 files changed, 20 insertions(+), 155 deletions(-)

diff --git a/modules/gcp/etcd/ignition.tf b/modules/gcp/etcd/ignition.tf
index 95d267647f..9149f86a91 100644
--- a/modules/gcp/etcd/ignition.tf
+++ b/modules/gcp/etcd/ignition.tf
@@ -8,13 +8,7 @@ data "ignition_config" "etcd" {
 
   files = [
     "${data.ignition_file.node_hostname.*.id[count.index]}",
-    "${data.ignition_file.etcd_ca.id}",
-    "${data.ignition_file.etcd_server_crt.id}",
-    "${data.ignition_file.etcd_server_key.id}",
-    "${data.ignition_file.etcd_client_crt.id}",
-    "${data.ignition_file.etcd_client_key.id}",
-    "${data.ignition_file.etcd_peer_crt.id}",
-    "${data.ignition_file.etcd_peer_key.id}",
+    "${var.ign_etcd_crt_id_list}",
   ]
 }
 
@@ -29,100 +23,6 @@ data "ignition_file" "node_hostname" {
   }
 }
 
-data "ignition_file" "etcd_ca" {
-  count = "${length(var.external_endpoints) == 0 ? var.instance_count > 0 ? 1 : 0 : 0}"
-
-  path       = "/etc/ssl/etcd/ca.crt"
-  mode       = 0644
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_ca_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_key" {
-  path       = "/etc/ssl/etcd/client.key"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_crt" {
-  path       = "/etc/ssl/etcd/client.crt"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_key" {
-  count = "${length(var.external_endpoints) == 0 ? var.instance_count > 0 ? 1 : 0 : 0}"
-
-  path       = "/etc/ssl/etcd/server.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_crt" {
-  count = "${length(var.external_endpoints) == 0 ? var.instance_count > 0 ? 1 : 0 : 0}"
-
-  path       = "/etc/ssl/etcd/server.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_key" {
-  count = "${length(var.external_endpoints) == 0 ? var.instance_count > 0 ? 1 : 0 : 0}"
-
-  path       = "/etc/ssl/etcd/peer.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_crt" {
-  count = "${length(var.external_endpoints) == 0 ? var.instance_count > 0 ? 1 : 0 : 0}"
-
-  path       = "/etc/ssl/etcd/peer.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_crt_pem}"
-  }
-}
-
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
diff --git a/modules/gcp/etcd/variables.tf b/modules/gcp/etcd/variables.tf
index 78cdc1a51e..fb142c6e0b 100644
--- a/modules/gcp/etcd/variables.tf
+++ b/modules/gcp/etcd/variables.tf
@@ -68,34 +68,6 @@ variable "tls_enabled" {
   default = false
 }
 
-variable "tls_ca_crt_pem" {
-  default = ""
-}
-
-variable "tls_client_key_pem" {
-  default = ""
-}
-
-variable "tls_client_crt_pem" {
-  default = ""
-}
-
-variable "tls_server_key_pem" {
-  default = ""
-}
-
-variable "tls_server_crt_pem" {
-  default = ""
-}
-
-variable "tls_peer_key_pem" {
-  default = ""
-}
-
-variable "tls_peer_crt_pem" {
-  default = ""
-}
-
 variable "ign_etcd_dropin_id_list" {
   type = "list"
 }
@@ -103,3 +75,7 @@ variable "ign_etcd_dropin_id_list" {
 variable "public_ssh_key" {
   default = ""
 }
+
+variable "ign_etcd_crt_id_list" {
+  type = "list"
+}
diff --git a/modules/gcp/master-igm/variables.tf b/modules/gcp/master-igm/variables.tf
index 66704141e8..232f01f647 100644
--- a/modules/gcp/master-igm/variables.tf
+++ b/modules/gcp/master-igm/variables.tf
@@ -100,8 +100,3 @@ variable "assets_gcs_location" {
   type        = "string"
   description = "Location on gcs of the Bootkube/Tectonic assets to use (bucket/key)"
 }
-
-variable "ign_ca_cert_id_list" {
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/modules/gcp/worker-igm/variables.tf b/modules/gcp/worker-igm/variables.tf
index 193bb4c1bf..e3de1ff321 100644
--- a/modules/gcp/worker-igm/variables.tf
+++ b/modules/gcp/worker-igm/variables.tf
@@ -59,8 +59,3 @@ variable "public_ssh_key" {
 variable "kubeconfig_content" {
   type = "string"
 }
-
-variable "ign_ca_cert_id_list" {
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/platforms/gcp/main.tf b/platforms/gcp/main.tf
index f6dc61e57e..2964c006d8 100644
--- a/platforms/gcp/main.tf
+++ b/platforms/gcp/main.tf
@@ -50,20 +50,14 @@ module "etcd" {
   disk_size               = "${var.tectonic_gcp_etcd_disk_size}"
   disk_type               = "${var.tectonic_gcp_etcd_disktype}"
   external_endpoints      = ["${compact(var.tectonic_etcd_servers)}"]
+  ign_etcd_crt_id_list    = "${module.ignition_masters.etcd_crt_id_list}"
   ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
   instance_count          = "${var.tectonic_self_hosted_etcd != "" ? 0 : var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : length(data.google_compute_zones.available.names) == 5 ? 5 : 3}"
   machine_type            = "${var.tectonic_gcp_etcd_gce_type}"
   managed_zone_name       = "${var.tectonic_gcp_ext_google_managedzone_name}"
   master_subnetwork_name  = "${module.network.master_subnetwork_name}"
   public_ssh_key          = "${var.tectonic_gcp_ssh_key}"
-  tls_ca_crt_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
-  tls_client_crt_pem      = "${module.etcd_certs.etcd_client_crt_pem}"
-  tls_client_key_pem      = "${module.etcd_certs.etcd_client_key_pem}"
   tls_enabled             = "${var.tectonic_etcd_tls_enabled}"
-  tls_peer_crt_pem        = "${module.etcd_certs.etcd_peer_crt_pem}"
-  tls_peer_key_pem        = "${module.etcd_certs.etcd_peer_key_pem}"
-  tls_server_crt_pem      = "${module.etcd_certs.etcd_server_crt_pem}"
-  tls_server_key_pem      = "${module.etcd_certs.etcd_server_key_pem}"
   zone_list               = "${data.google_compute_zones.available.names}"
 }
 
@@ -138,22 +132,27 @@ module "workers" {
 module "ignition_masters" {
   source = "../../modules/ignition"
 
+  assets_location           = "${google_storage_bucket.assets_storage_bucket.name}/${google_storage_bucket_object.tectonic-assets.name}"
   bootstrap_upgrade_cl      = "${var.tectonic_bootstrap_upgrade_cl}"
   cluster_name              = "${var.tectonic_cluster_name}"
   container_images          = "${var.tectonic_container_images}"
   etcd_advertise_name_list  = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_client_crt_pem       = "${module.etcd_certs.etcd_client_crt_pem}"
+  etcd_client_key_pem       = "${module.etcd_certs.etcd_client_key_pem}"
   etcd_count                = "${length(data.template_file.etcd_hostname_list.*.id)}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_peer_crt_pem         = "${module.etcd_certs.etcd_peer_crt_pem}"
+  etcd_peer_key_pem         = "${module.etcd_certs.etcd_peer_key_pem}"
+  etcd_server_crt_pem       = "${module.etcd_certs.etcd_server_crt_pem}"
+  etcd_server_key_pem       = "${module.etcd_certs.etcd_server_key_pem}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
-
-  image_re             = "${var.tectonic_image_re}"
-  kube_dns_service_ip  = "${module.bootkube.kube_dns_service_ip}"
-  kubelet_cni_bin_dir  = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
-  kubelet_debug_config = "${var.tectonic_kubelet_debug_config}"
-  kubelet_node_label   = "node-role.kubernetes.io/master"
-  kubelet_node_taints  = "node-role.kubernetes.io/master=:NoSchedule"
-  tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
-  assets_location      = "${google_storage_bucket.assets_storage_bucket.name}/${google_storage_bucket_object.tectonic-assets.name}"
+  image_re                  = "${var.tectonic_image_re}"
+  kube_dns_service_ip       = "${module.bootkube.kube_dns_service_ip}"
+  kubelet_cni_bin_dir       = "${var.tectonic_networking == "calico" || var.tectonic_networking == "canal" ? "/var/lib/cni/bin" : "" }"
+  kubelet_debug_config      = "${var.tectonic_kubelet_debug_config}"
+  kubelet_node_label        = "node-role.kubernetes.io/master"
+  kubelet_node_taints       = "node-role.kubernetes.io/master=:NoSchedule"
+  tectonic_vanilla_k8s      = "${var.tectonic_vanilla_k8s}"
 }
 
 module "ignition_workers" {

From f328eb95903ba882d22306f02f52c349592094a8 Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:19:49 +0100
Subject: [PATCH 7/9] */openstack: use unified etcd TLS ignition files

---
 modules/openstack/etcd/ignition.tf   | 90 +---------------------------
 modules/openstack/etcd/variables.tf  | 32 ++--------
 modules/openstack/nodes/variables.tf |  5 --
 platforms/openstack/neutron/main.tf  | 14 ++---
 4 files changed, 12 insertions(+), 129 deletions(-)

diff --git a/modules/openstack/etcd/ignition.tf b/modules/openstack/etcd/ignition.tf
index b6d3af9f2c..b7532aa6fa 100644
--- a/modules/openstack/etcd/ignition.tf
+++ b/modules/openstack/etcd/ignition.tf
@@ -7,13 +7,7 @@ data "ignition_config" "etcd" {
 
   files = [
     "${data.ignition_file.resolv_conf.id}",
-    "${data.ignition_file.etcd_ca.id}",
-    "${data.ignition_file.etcd_server_crt.id}",
-    "${data.ignition_file.etcd_server_key.id}",
-    "${data.ignition_file.etcd_client_crt.id}",
-    "${data.ignition_file.etcd_client_key.id}",
-    "${data.ignition_file.etcd_peer_crt.id}",
-    "${data.ignition_file.etcd_peer_key.id}",
+    "${var.ign_etcd_crt_id_list}",
   ]
 
   systemd = [
@@ -23,88 +17,6 @@ data "ignition_config" "etcd" {
   ]
 }
 
-data "ignition_file" "etcd_ca" {
-  path       = "/etc/ssl/etcd/ca.crt"
-  mode       = 0644
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_ca_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_key" {
-  path       = "/etc/ssl/etcd/client.key"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_crt" {
-  path       = "/etc/ssl/etcd/client.crt"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_key" {
-  path       = "/etc/ssl/etcd/server.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_crt" {
-  path       = "/etc/ssl/etcd/server.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_key" {
-  path       = "/etc/ssl/etcd/peer.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_crt" {
-  path       = "/etc/ssl/etcd/peer.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_crt_pem}"
-  }
-}
-
 data "ignition_file" "resolv_conf" {
   path       = "/etc/resolv.conf"
   mode       = 0644
diff --git a/modules/openstack/etcd/variables.tf b/modules/openstack/etcd/variables.tf
index 95638e1519..bf80210d2a 100644
--- a/modules/openstack/etcd/variables.tf
+++ b/modules/openstack/etcd/variables.tf
@@ -32,34 +32,6 @@ variable "tls_enabled" {
   default = false
 }
 
-variable "tls_ca_crt_pem" {
-  default = ""
-}
-
-variable "tls_server_key_pem" {
-  default = ""
-}
-
-variable "tls_server_crt_pem" {
-  default = ""
-}
-
-variable "tls_client_key_pem" {
-  default = ""
-}
-
-variable "tls_client_crt_pem" {
-  default = ""
-}
-
-variable "tls_peer_key_pem" {
-  default = ""
-}
-
-variable "tls_peer_crt_pem" {
-  default = ""
-}
-
 variable "ign_etcd_dropin_id_list" {
   type = "list"
 }
@@ -67,3 +39,7 @@ variable "ign_etcd_dropin_id_list" {
 variable "ign_coreos_metadata_dropin_id" {
   type = "string"
 }
+
+variable "ign_etcd_crt_id_list" {
+  type = "list"
+}
diff --git a/modules/openstack/nodes/variables.tf b/modules/openstack/nodes/variables.tf
index 82ff3f3cc1..55151b8e00 100644
--- a/modules/openstack/nodes/variables.tf
+++ b/modules/openstack/nodes/variables.tf
@@ -47,8 +47,3 @@ variable "ign_tectonic_path_unit_id" {
   type    = "string"
   default = ""
 }
-
-variable "ign_ca_cert_id_list" 
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf
index f544995aa9..48bce1ef59 100644
--- a/platforms/openstack/neutron/main.tf
+++ b/platforms/openstack/neutron/main.tf
@@ -157,17 +157,11 @@ EOF
   container_image               = "${var.tectonic_container_images["etcd"]}"
   core_public_keys              = ["${module.secrets.core_public_key_openssh}"]
   ign_coreos_metadata_dropin_id = "${module.ignition_masters.coreos_metadata_dropin_id}"
+  ign_etcd_crt_id_list          = "${module.ignition_masters.etcd_crt_id_list}"
   ign_etcd_dropin_id_list       = "${module.ignition_masters.etcd_dropin_id_list}"
   instance_count                = "${var.tectonic_etcd_count}"
   self_hosted_etcd              = "${var.tectonic_self_hosted_etcd}"
-  tls_ca_crt_pem                = "${module.etcd_certs.etcd_ca_crt_pem}"
-  tls_client_crt_pem            = "${module.etcd_certs.etcd_client_crt_pem}"
-  tls_client_key_pem            = "${module.etcd_certs.etcd_client_key_pem}"
   tls_enabled                   = "${var.tectonic_etcd_tls_enabled}"
-  tls_peer_crt_pem              = "${module.etcd_certs.etcd_peer_crt_pem}"
-  tls_peer_key_pem              = "${module.etcd_certs.etcd_peer_key_pem}"
-  tls_server_crt_pem            = "${module.etcd_certs.etcd_server_crt_pem}"
-  tls_server_key_pem            = "${module.etcd_certs.etcd_server_key_pem}"
 }
 
 module "ignition_masters" {
@@ -180,8 +174,14 @@ module "ignition_masters" {
   custom_ca_cert_pem_list   = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list  = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_ca_cert_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
+  etcd_client_crt_pem       = "${module.etcd_certs.etcd_client_crt_pem}"
+  etcd_client_key_pem       = "${module.etcd_certs.etcd_client_key_pem}"
   etcd_count                = "${var.tectonic_etcd_count}"
   etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}"
+  etcd_peer_crt_pem         = "${module.etcd_certs.etcd_peer_crt_pem}"
+  etcd_peer_key_pem         = "${module.etcd_certs.etcd_peer_key_pem}"
+  etcd_server_crt_pem       = "${module.etcd_certs.etcd_server_crt_pem}"
+  etcd_server_key_pem       = "${module.etcd_certs.etcd_server_key_pem}"
   etcd_tls_enabled          = "${var.tectonic_etcd_tls_enabled}"
   image_re                  = "${var.tectonic_image_re}"
   ingress_ca_cert_pem       = "${module.ingress_certs.ca_cert_pem}"

From dd2e0fdced967d7d80f781db68ca48b95514680a Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Tue, 21 Nov 2017 12:20:03 +0100
Subject: [PATCH 8/9] */vmware: use unified etcd TLS ignition files

---
 modules/vmware/etcd/ignition.tf  | 92 +-------------------------------
 modules/vmware/etcd/variables.tf | 30 ++---------
 modules/vmware/node/variables.tf |  5 --
 platforms/vmware/main.tf         | 14 ++---
 4 files changed, 11 insertions(+), 130 deletions(-)

diff --git a/modules/vmware/etcd/ignition.tf b/modules/vmware/etcd/ignition.tf
index d55a700250..105d0845a5 100644
--- a/modules/vmware/etcd/ignition.tf
+++ b/modules/vmware/etcd/ignition.tf
@@ -7,13 +7,7 @@ data "ignition_config" "etcd" {
 
   files = [
     "${data.ignition_file.node_hostname.*.id[count.index]}",
-    "${data.ignition_file.etcd_ca.id}",
-    "${data.ignition_file.etcd_server_crt.id}",
-    "${data.ignition_file.etcd_server_key.id}",
-    "${data.ignition_file.etcd_client_crt.id}",
-    "${data.ignition_file.etcd_client_key.id}",
-    "${data.ignition_file.etcd_peer_crt.id}",
-    "${data.ignition_file.etcd_peer_key.id}",
+    "${var.ign_etcd_crt_id_list}",
   ]
 
   systemd = [
@@ -31,90 +25,6 @@ data "ignition_user" "core" {
   ssh_authorized_keys = ["${var.core_public_keys}"]
 }
 
-data "ignition_file" "etcd_ca" {
-  path       = "/etc/ssl/etcd/ca.crt"
-  mode       = 0644
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_ca_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_key" {
-  path       = "/etc/ssl/etcd/client.key"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_client_crt" {
-  path       = "/etc/ssl/etcd/client.crt"
-  mode       = 0400
-  uid        = 0
-  gid        = 0
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_client_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_key" {
-  path       = "/etc/ssl/etcd/server.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_server_crt" {
-  path       = "/etc/ssl/etcd/server.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_server_crt_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_key" {
-  path       = "/etc/ssl/etcd/peer.key"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_key_pem}"
-  }
-}
-
-data "ignition_file" "etcd_peer_crt" {
-  path       = "/etc/ssl/etcd/peer.crt"
-  mode       = 0400
-  uid        = 232
-  gid        = 232
-  filesystem = "root"
-
-  content {
-    content = "${var.tls_peer_crt_pem}"
-  }
-}
-
 data "ignition_systemd_unit" "locksmithd" {
   count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
diff --git a/modules/vmware/etcd/variables.tf b/modules/vmware/etcd/variables.tf
index 1beb85573f..049a0e00f7 100644
--- a/modules/vmware/etcd/variables.tf
+++ b/modules/vmware/etcd/variables.tf
@@ -95,34 +95,10 @@ variable hostname {
   description = "Hostname of the node"
 }
 
-variable "tls_ca_crt_pem" {
-  default = ""
-}
-
-variable "tls_client_key_pem" {
-  default = ""
-}
-
-variable "tls_client_crt_pem" {
-  default = ""
-}
-
-variable "tls_server_key_pem" {
-  default = ""
-}
-
-variable "tls_server_crt_pem" {
-  default = ""
-}
-
-variable "tls_peer_key_pem" {
-  default = ""
-}
-
-variable "tls_peer_crt_pem" {
-  default = ""
+variable "ign_etcd_dropin_id_list" {
+  type = "list"
 }
 
-variable "ign_etcd_dropin_id_list" {
+variable "ign_etcd_crt_id_list" {
   type = "list"
 }
diff --git a/modules/vmware/node/variables.tf b/modules/vmware/node/variables.tf
index 1baa016719..72060b4152 100644
--- a/modules/vmware/node/variables.tf
+++ b/modules/vmware/node/variables.tf
@@ -124,8 +124,3 @@ variable "ign_tectonic_path_unit_id" {
   type    = "string"
   default = ""
 }
-
-variable "ign_ca_cert_id_list" {
-  type        = "list"
-  description = "The list of public CA certificate ignition file IDs."
-}
diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf
index c60917d01c..d8788a2bb8 100644
--- a/platforms/vmware/main.tf
+++ b/platforms/vmware/main.tf
@@ -9,16 +9,10 @@ module "etcd" {
   external_endpoints      = ["${compact(var.tectonic_etcd_servers)}"]
   gateways                = "${var.tectonic_vmware_etcd_gateways}"
   hostname                = "${var.tectonic_vmware_etcd_hostnames}"
+  ign_etcd_crt_id_list    = "${module.ignition_masters.etcd_crt_id_list}"
   ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}"
   instance_count          = "${var.tectonic_self_hosted_etcd != "" ? 0 : var.tectonic_etcd_count }"
   ip_address              = "${var.tectonic_vmware_etcd_ip}"
-  tls_ca_crt_pem          = "${module.etcd_certs.etcd_ca_crt_pem}"
-  tls_client_crt_pem      = "${module.etcd_certs.etcd_client_crt_pem}"
-  tls_client_key_pem      = "${module.etcd_certs.etcd_client_key_pem}"
-  tls_peer_crt_pem        = "${module.etcd_certs.etcd_peer_crt_pem}"
-  tls_peer_key_pem        = "${module.etcd_certs.etcd_peer_key_pem}"
-  tls_server_crt_pem      = "${module.etcd_certs.etcd_server_crt_pem}"
-  tls_server_key_pem      = "${module.etcd_certs.etcd_server_key_pem}"
   vm_disk_datastore       = "${var.tectonic_vmware_etcd_datastore}"
   vm_disk_template        = "${var.tectonic_vmware_vm_template}"
   vm_disk_template_folder = "${var.tectonic_vmware_vm_template_folder}"
@@ -46,7 +40,13 @@ module "ignition_masters" {
   custom_ca_cert_pem_list  = "${var.tectonic_custom_ca_pem_list}"
   etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}"
   etcd_ca_cert_pem         = "${module.etcd_certs.etcd_ca_crt_pem}"
+  etcd_client_crt_pem      = "${module.etcd_certs.etcd_client_crt_pem}"
+  etcd_client_key_pem      = "${module.etcd_certs.etcd_client_key_pem}"
   etcd_count               = "${length(data.template_file.etcd_hostname_list.*.rendered)}"
+  etcd_peer_crt_pem        = "${module.etcd_certs.etcd_peer_crt_pem}"
+  etcd_peer_key_pem        = "${module.etcd_certs.etcd_peer_key_pem}"
+  etcd_server_crt_pem      = "${module.etcd_certs.etcd_server_crt_pem}"
+  etcd_server_key_pem      = "${module.etcd_certs.etcd_server_key_pem}"
   image_re                 = "${var.tectonic_image_re}"
   ingress_ca_cert_pem      = "${module.ingress_certs.ca_cert_pem}"
   kube_ca_cert_pem         = "${module.kube_certs.ca_cert_pem}"

From 6d66b86112082d209d6e6cbeac8f1e07137301de Mon Sep 17 00:00:00 2001
From: Sergiusz Urbaniak <sur@coreos.com>
Date: Fri, 10 Nov 2017 17:28:35 +0100
Subject: [PATCH 9/9] Documentation/examples: regenerate

---
 Documentation/variables/config.md           | 1 +
 examples/terraform.tfvars.aws               | 3 +++
 examples/terraform.tfvars.azure             | 3 +++
 examples/terraform.tfvars.gcp               | 3 +++
 examples/terraform.tfvars.metal             | 3 +++
 examples/terraform.tfvars.openstack-neutron | 3 +++
 examples/terraform.tfvars.vmware            | 3 +++
 7 files changed, 19 insertions(+)

diff --git a/Documentation/variables/config.md b/Documentation/variables/config.md
index 41caa7e032..4e8cbbc463 100644
--- a/Documentation/variables/config.md
+++ b/Documentation/variables/config.md
@@ -20,6 +20,7 @@ This document gives an overview of variables used in all platforms of the Tecton
 | tectonic_container_images | (internal) Container images to use | map | `<map>` |
 | tectonic_container_linux_channel | (optional) The Container Linux update channel.<br><br>Examples: `stable`, `beta`, `alpha` | string | `stable` |
 | tectonic_container_linux_version | The Container Linux version to use. Set to `latest` to select the latest available version for the selected update channel.<br><br>Examples: `latest`, `1465.6.0` | string | `latest` |
+| tectonic_custom_ca_pem_list | (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes. | list | `<list>` |
 | tectonic_ddns_key_algorithm | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key algorithm. | string | `` |
 | tectonic_ddns_key_name | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key name. | string | `` |
 | tectonic_ddns_key_secret | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key secret. | string | `` |
diff --git a/examples/terraform.tfvars.aws b/examples/terraform.tfvars.aws
index 1a84cd32be..390f455b4a 100644
--- a/examples/terraform.tfvars.aws
+++ b/examples/terraform.tfvars.aws
@@ -194,6 +194,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.
diff --git a/examples/terraform.tfvars.azure b/examples/terraform.tfvars.azure
index 7cfaeaa653..488f04994e 100644
--- a/examples/terraform.tfvars.azure
+++ b/examples/terraform.tfvars.azure
@@ -155,6 +155,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.
diff --git a/examples/terraform.tfvars.gcp b/examples/terraform.tfvars.gcp
index 74b44eaf58..9db52c0c6a 100644
--- a/examples/terraform.tfvars.gcp
+++ b/examples/terraform.tfvars.gcp
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.
diff --git a/examples/terraform.tfvars.metal b/examples/terraform.tfvars.metal
index 9ab604ccfe..044fa33dbc 100644
--- a/examples/terraform.tfvars.metal
+++ b/examples/terraform.tfvars.metal
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.
diff --git a/examples/terraform.tfvars.openstack-neutron b/examples/terraform.tfvars.openstack-neutron
index d67e73a936..ab50e8efe5 100644
--- a/examples/terraform.tfvars.openstack-neutron
+++ b/examples/terraform.tfvars.openstack-neutron
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.
diff --git a/examples/terraform.tfvars.vmware b/examples/terraform.tfvars.vmware
index e5c2a4f183..78b18b0052 100644
--- a/examples/terraform.tfvars.vmware
+++ b/examples/terraform.tfvars.vmware
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.