poseidon · dghubble · Mar 19, 2018 · Feb 20, 2018 · Mar 19, 2018 · Mar 19, 2018
diff --git a/CHANGES.md b/CHANGES.md
@@ -4,6 +4,23 @@ Notable changes between versions.
 
 ## Latest
 
+* Introduce [Container Linux Config snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) on cloud platforms ([#145](https://github.com/poseidon/typhoon/pull/145))
+  * Validate and additively merge custom Container Linux Configs during `terraform plan`
+  * Define files, systemd units, dropins, networkd configs, mounts, users, and more
+  * Require updating `terraform-provider-ct` plugin from v0.2.0 to v0.2.1
+
+#### AWS
+
+* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!)
+
+#### Digital Ocean
+
+* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!)
+
+#### Google Cloud
+
+* [Require](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action required!)
+
 #### Addons
 
 * Update Prometheus from 2.2.0 to 2.2.1

diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf
@@ -78,4 +78,5 @@ data "ct_config" "controller_ign" {
   count        = "${var.controller_count}"
   content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
+  snippets     = ["${var.controller_clc_snippets}"]
 }
diff --git a/aws/container-linux/kubernetes/variables.tf b/aws/container-linux/kubernetes/variables.tf
@@ -60,6 +60,18 @@ variable "worker_type" {
   description = "Worker EC2 instance type"
 }
 
+variable "controller_clc_snippets" {
+  type        = "list"
+  description = "Controller Container Linux Config snippets"
+  default     = []
+}
+
+variable "worker_clc_snippets" {
+  type        = "list"
+  description = "Worker Container Linux Config snippets"
+  default     = []
+}
+
 # bootkube assets
 
 variable "asset_dir" {

diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf
@@ -16,4 +16,5 @@ module "workers" {
   ssh_authorized_key    = "${var.ssh_authorized_key}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+  clc_snippets          = "${var.worker_clc_snippets}"
 }
diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf
@@ -71,3 +71,9 @@ variable "cluster_domain_suffix" {
   type        = "string"
   default     = "cluster.local"
 }
+
+variable "clc_snippets" {
+  type        = "list"
+  description = "Container Linux Config snippets"
+  default     = []
+}
diff --git a/aws/container-linux/kubernetes/workers/workers.tf b/aws/container-linux/kubernetes/workers/workers.tf
@@ -71,4 +71,5 @@ data "template_file" "worker_config" {
 data "ct_config" "worker_ign" {
   content      = "${data.template_file.worker_config.rendered}"
   pretty_print = false
+  snippets     = ["${var.clc_snippets}"]
 }
diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf
@@ -90,4 +90,6 @@ data "ct_config" "controller_ign" {
   count        = "${var.controller_count}"
   content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
+
+  snippets = ["${var.controller_clc_snippets}"]
 }
diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf
@@ -48,6 +48,18 @@ variable "ssh_fingerprints" {
   description = "SSH public key fingerprints. (e.g. see `ssh-add -l -E md5`)"
 }
 
+variable "controller_clc_snippets" {
+  type        = "list"
+  description = "Controller Container Linux Config snippets"
+  default     = []
+}
+
+variable "worker_clc_snippets" {
+  type        = "list"
+  description = "Worker Container Linux Config snippets"
+  default     = []
+}
+
 # bootkube assets
 
 variable "asset_dir" {

diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf
@@ -51,4 +51,5 @@ data "template_file" "worker_config" {
 data "ct_config" "worker_ign" {
   content      = "${data.template_file.worker_config.rendered}"
   pretty_print = false
+  snippets     = ["${var.worker_clc_snippets}"]
 }
diff --git a/docs/advanced/customization.md b/docs/advanced/customization.md
@@ -1,6 +1,130 @@
 # Customization
 
-To customize clusters in ways that aren't supported by input variables, fork the repo and make changes to the Terraform module. Stay tuned for improvements to this strategy since it is beneficial to stay close to this upstream.
+Typhoon provides minimal Kubernetes clusters with defaults we recommend for production. Terraform variables provide easy to use and supported customizations for clusters. Advanced options are available for customizing the architecture or hosts.
+
+## Variables
+
+Typhoon modules accept Terraform input variables for customizing clusters in meritorious ways (e.g. `worker_count`, etc). Variables are carefully considered to provide essentials, while limiting complexity and test matrix burden. See each platform's tutorial for options.
+
+## Addons
+
+Clusters are kept to a minimal Kubernetes control plane by offering components like Nginx Ingress Controller, Prometheus, Grafana, and Heapster as optional post-install [addons](https://github.com/poseidon/typhoon/tree/master/addons). Customize addons by modifying a copy of our addon manifests.
+
+## Hosts
+
+### Container Linux
+
+!!! danger
+    Container Linux Configs provide powerful host customization abilities. You are responsible for the additional configs defined for hosts.
+
+Container Linux Configs (CLCs) declare how a Container Linux instance's disk should be provisioned on first boot from disk. CLCs define disk partitions, filesystems, files, systemd units, dropins, networkd configs, mount units, raid arrays, and users. Typhoon creates controller and worker instances with base Container Linux Configs to create a minimal, secure Kubernetes cluster on each platform.
+
+Typhoon AWS, Google Cloud, and Digital Ocean give users the ability to provide CLC *snippets* - valid Container Linux Configs that are validated and additively merged into the Typhoon base config during `terraform plan`. This allows advanced host customizations and experimentation.
+
+#### Examples
+
+Container Linux [docs](https://coreos.com/os/docs/latest/clc-examples.html) show many simple config examples. Ensure a file `/opt/hello` is created with permissions 0644. 
+
+```
+# custom-files
+storage:
+  files:
+    - path: /opt/hello
+      filesystem: root
+      contents:
+        inline: |
+          Hello World
+      mode: 0644
+```
+
+Ensure a systemd unit `hello.service` is created and a dropin `50-etcd-cluster.conf` is added for `etcd-member.service`.
+
+```
+# custom-units
+systemd:
+  units:
+    - name: hello.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Hello World
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/bin/echo Hello World!
+        [Install]
+        WantedBy=multi-user.target
+    - name: etcd-member.service
+      enable: true
+      dropins:
+        - name: 50-etcd-cluster.conf
+          contents: |
+            Environment="ETCD_LOG_PACKAGE_LEVELS=etcdserver=WARNING,security=DEBUG"
+```
+
+#### Specification
+
+View the Container Linux Config [format](https://coreos.com/os/docs/1576.4.0/configuration.html) to read about each field.
+
+#### Usage
+
+Write Container Linux Configs *snippets* as files in the repository where you keep Terraform configs for clusters (perhaps in a `clc` or `snippets` subdirectory). You may organize snippets in multiple files as desired, provided they are each valid.
+
+Define an [AWS](https://typhoon.psdn.io/aws/#cluster), [Google Cloud](https://typhoon.psdn.io/google-cloud/#cluster), or [Digital Ocean](https://typhoon.psdn.io/digital-ocean/#cluster) cluster and fill in the optional `controller_clc_snippets` or `worker_clc_snippets` fields.
+
+```
+module "digital-ocean-nemo" {
+  ...
+
+  controller_count        = 1
+  worker_count            = 2
+  controller_clc_snippets = [
+    "${file("./custom-files")}",
+    "${file("./custom-units")}",
+  ]
+  worker_clc_snippets = [
+    "${file("./custom-files")}",
+    "${file("./custom-units")}",
+  ]
+  ...
+}
+```
+
+Plan the resources to be created.
+
+```
+$ terraform plan
+Plan: 54 to add, 0 to change, 0 to destroy.
+```
+
+Most syntax errors in CLCs can be caught during planning. For example, mangle the indentation in one of the CLC files:
+
+```
+$ terraform plan
+...
+error parsing Container Linux Config: error: yaml: line 3: did not find expected '-' indicator
+```
+
+Undo the mangle. Apply the changes to create the cluster per the tutorial.
+
+```
+$ terraform apply
+```
+
+Container Linux Configs (and the CoreOS Ignition system) create immutable infrastructure. Disk provisioning is performed only on first boot from disk. That means if you change a snippet used by an instance, Terraform will (correctly) try to destroy and recreate that instance. Be careful!
+
+!!! danger
+    Destroying and recreating controller instances is destructive! etcd runs on controller instances and stores data there. Do not modify controller snippets. See [blue/green](https://typhoon.psdn.io/topics/maintenance/#upgrades) clusters.
+
+## Architecture
+
+To customize clusters in ways that aren't supported by input variables, fork Typhoon and maintain a repository with customizations. Reference the repository by changing the username.
+
+```
+module "digital-ocean-nemo" {
+  source = "git::https://github.com/USERNAME/typhoon//digital-ocean/container-linux/kubernetes?ref=myspecialcase"
+  ...
+}
+```
 
 To customize lower-level Kubernetes control plane bootstrapping, see the [poseidon/bootkube-terraform](https://github.com/poseidon/bootkube-terraform) Terraform module.
 
diff --git a/docs/aws.md b/docs/aws.md
@@ -260,6 +260,8 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
 | cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
+| controller_clc_snippets | Controller Container Linux Config snippets | [] | |
+| worker_clc_snippets | Worker Container Linux Config snippets | [] | |
 
 Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/).
 

diff --git a/docs/digital-ocean.md b/docs/digital-ocean.md
@@ -269,6 +269,8 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
 | cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
+| controller_clc_snippets | Controller Container Linux Config snippets | [] | |
+| worker_clc_snippets | Worker Container Linux Config snippets | [] | |
 
 Check the list of valid [droplet types](https://developers.digitalocean.com/documentation/changelog/api-v2/new-size-slugs-for-droplet-plan-changes/) or use `doctl compute size list`.
 

diff --git a/docs/google-cloud.md b/docs/google-cloud.md
@@ -262,6 +262,8 @@ resource "google_dns_managed_zone" "zone-for-clusters" {
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
 | cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
+| controller_clc_snippets | Controller Container Linux Config snippets | [] | |
+| worker_clc_snippets | Worker Container Linux Config snippets | [] | |
 
 Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types).
 

diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md
@@ -205,7 +205,7 @@ You should now be able to run `terraform plan` without errors. When you choose,
 
 ## terraform-provider-ct v0.2.1
 
-Typhoon recommends updating the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin installed on your system from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1). The release contains an important feature that will be used in future Typhoon releases.
+Typhoon requires updating the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin installed on your system from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1).
 
 Check your `~/.terraformrc` to find your current `terraform-provider-ct` plugin.
 
@@ -236,4 +236,3 @@ Verify Terraform does not produce a diff related to Container Linux provisioning
 terraform plan
 ```
 
-You're prepared for future Typhoon releases.
diff --git a/google-cloud/container-linux/kubernetes/cluster.tf b/google-cloud/container-linux/kubernetes/cluster.tf
@@ -17,6 +17,7 @@ module "controllers" {
   ssh_authorized_key    = "${var.ssh_authorized_key}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+  clc_snippets          = "${var.controller_clc_snippets}"
 }
 
 module "workers" {
@@ -37,4 +38,5 @@ module "workers" {
   ssh_authorized_key    = "${var.ssh_authorized_key}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+  clc_snippets          = "${var.worker_clc_snippets}"
 }
diff --git a/google-cloud/container-linux/kubernetes/controllers/controllers.tf b/google-cloud/container-linux/kubernetes/controllers/controllers.tf
@@ -87,4 +87,5 @@ data "ct_config" "controller_ign" {
   count        = "${var.count}"
   content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
+  snippets     = ["${var.clc_snippets}"]
 }
diff --git a/google-cloud/container-linux/kubernetes/controllers/variables.tf b/google-cloud/container-linux/kubernetes/controllers/variables.tf
@@ -3,9 +3,9 @@ variable "cluster_name" {
   description = "Unique cluster name"
 }
 
-variable "ssh_authorized_key" {
+variable "region" {
   type        = "string"
-  description = "SSH public key for logging in as user 'core'"
+  description = "Google Cloud region (e.g. us-central1, see `gcloud compute regions list`)."
 }
 
 variable "network" {
@@ -30,11 +30,6 @@ variable "count" {
   description = "Number of controller compute instances the instance group should manage"
 }
 
-variable "region" {
-  type        = "string"
-  description = "Google Cloud region (e.g. us-central1, see `gcloud compute regions list`)."
-}
-
 variable "machine_type" {
   type        = "string"
   description = "Machine type for compute instances (e.g. gcloud compute machine-types list)"
@@ -51,12 +46,22 @@ variable "disk_size" {
   description = "The size of the disk in gigabytes."
 }
 
-// configuration
+# configuration
 
 variable "networking" {
   description = "Choice of networking provider (flannel or calico)"
   type        = "string"
-  default     = "flannel"
+  default     = "calico"
+}
+
+variable "kubeconfig" {
+  type        = "string"
+  description = "Generated Kubelet kubeconfig"
+}
+
+variable "ssh_authorized_key" {
+  type        = "string"
+  description = "SSH public key for logging in as user 'core'"
 }
 
 variable "service_cidr" {
@@ -75,7 +80,8 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
-variable "kubeconfig" {
-  type        = "string"
-  description = "Generated Kubelet kubeconfig"
+variable "clc_snippets" {
+  type        = "list"
+  description = "Container Linux Config snippets"
+  default     = []
 }
diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf
@@ -52,6 +52,18 @@ variable "worker_preemptible" {
   description = "If enabled, Compute Engine will terminate workers randomly within 24 hours"
 }
 
+variable "controller_clc_snippets" {
+  type        = "list"
+  description = "Controller Container Linux Config snippets"
+  default     = []
+}
+
+variable "worker_clc_snippets" {
+  type        = "list"
+  description = "Worker Container Linux Config snippets"
+  default     = []
+}
+
 # bootkube assets
 
 variable "asset_dir" {