diff --git a/Documentation/dev/node-bootstrap-flow.md b/Documentation/dev/node-bootstrap-flow.md
new file mode 100644
index 0000000000..fc44997163
--- /dev/null
+++ b/Documentation/dev/node-bootstrap-flow.md
@@ -0,0 +1,158 @@
+# Node bootstrapping flow
+
+This is a development document which describes the bootstrapping flow for ContainerLinux nodes provisioned by the tectonic-installer as part of a Tectonic cluster.
+
+## Overview
+
+When a cluster node is being bootstrapped from scratch, it goes through several phases in the following order:
+
+1. first-boot OS configuration, via ignition (systemd units, node configuration, etc)
+1. provisioning of additional assets (k8s manifests, TLS material), via either of:
+ * pushing from terraform file/remote-exec (SSH)
+ * pulling from private cloud stores (S3 buckets)
+1. system-wide updates via `k8s-node-bootstrap.service`, which includes:
+ * determining current kubernetes cluster version (when joining an existing cluster)
+ * triggering a ContainerLinux update, via update-engine (optional)
+ * downloading and deploying proper docker addon version, via tectonic-torcx
+ * writing the `kubelet.env` file
+1. if needed, a node reboot is triggered to apply systemd-wide changes
+1. `kubelet.service` picks up the `kubelet.env` file and actually starts the kubelet as a rkt-fly service.
+
+Additionally, only on one of the master nodes the following kubernetes bootstrapping happens:
+
+1. `bootkube.service` is started after `kubelet.service` start
+1. a static bootstrapping control-plane is deployed
+1. a fully self-hosted control-plane starts and takes over the previous one
+1. `bootkube.service` is completed with success
+1. `tectonic.service` is started
+1. a self-hosted tectonic control-plane is deployed
+1. `tectonic.service` is completed with success
+
+## Systemd units
+
+The following systemd units are deployed to a node by tectonic-installer and take part in the bootstrapping process:
+
+* `k8s-node-bootstrap.service` ensures node and assets freshness. It is automatically started on boot, can crash-loop, and it runs only during bootstrap
+* `kubelet.service` is the main kubelet deamon. It is automatically started on boot, it is crash-looping until `kubelet.env` is populated, and it runs on each boot
+
+Additionally, only on one of the master nodes the following kubernetes bootstrapping happens:
+
+* `bootkube.service` deploys the initial bootstrapping control-plane. It is started only after `kubelet.service` _is started_. It is a oneshot unit and cannot crash, and it runs only during bootstrap
+* `bootkube.path` waits for bootkube assets/scripts to exist on disk and triggers `bootkube.service`
+* `tectonic.service` deploys tectonic control-plane. It is started only after `bootkube.service` _has completed_. It is a oneshot unit and cannot crash, and it runs only during bootstrap
+* `bootkube.path` waits for tectonic assets/scripts to exist on disk and triggers `tectonic.service`
+
+## Service ordering
+
+Service ordering is enforced via systemd dependencies. This is the rationale for the settings, with relevant snippets:
+
+### `k8s-node-bootstrap.service`
+
+```
+ConditionPathExists=!/etc/kubernetes/kubelet.env
+Before=kubelet.service
+Restart=on-failure
+ExecStartPre=[...]
+ExecStart=/usr/bin/echo "node components bootstrapped"
+WantedBy=multi-user.target kubelet.service
+```
+
+This service is enabled by default and can crash-loop until success.
+Main logic happens in `Pre`, before the unit is marked as started, to block further services (a synchronous reboot can happen here).
+
+In particular, this blocks kubelet from starting by:
+ * a `WantedBy=` and `Before=`
+ * writing the actual `kubelet.env` file on success.
+
+It is skipped on further boots, as the condition-path exists.
+
+### `kubelet.service`
+
+```
+EnvironmentFile=/etc/kubernetes/kubelet.env
+ExecStart=/usr/lib/coreos/kubelet-wrapper [...]
+Restart=always
+WantedBy=multi-user.target
+```
+
+This service is enabled by default and can crash-loop until success.
+On first boot, it is initially blocked by `k8s-node-bootstrap.service`.
+It crash-loop until the `kubelet.env` file exists.
+It is started on every boot.
+
+### `bootkube.path` and `bootkube.service`
+
+```
+ConditionPathExists=!/opt/tectonic/init_bootkube.done
+Wants=kubelet.service
+After=kubelet.service
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/usr/bin/bash /opt/tectonic/bootkube.sh
+ExecStartPost=/bin/touch /opt/tectonic/init_bootkube.done
+```
+
+Bootkube service unit is not enabled by default. It is instead triggered by a path unit, which waits for assets written synchronously by terraform.
+
+This service waits for kubelet to be *started* via systemd dependency.
+It is a oneshot service, thus marked as started only once the script return with success.
+It is skipped on further boots, as the condition-path exists.
+
+### `tectonic.path` and `tectonic.service`
+
+```
+ConditionPathExists=!/opt/tectonic/init_tectonic.done
+Requires=bootkube.service
+After=bootkube.service
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/usr/bin/bash /opt/tectonic/tectonic-rkt.sh
+ExecStartPost=/bin/touch /opt/tectonic/init_tectonic.done
+```
+
+Tectonic service unit is not enabled by default. It is instead triggered by a path unit, which waits for assets written synchronously by terraform.
+
+This service waits for bootkube process to be *completed* via systemd dependency.
+It is a oneshot service, thus marked as started only once the script return with success.
+It is skipped on further boots, as the condition-path exists.
+
+## Diagram
+
+This is a visual simplified representation of the overall bootstrapping flow.
+
+```bob
+Legend:
+ * TF -> terraform provisioner
+ * IGN -> ignition
+ * knb.s -> k8s-node-bootstrap.service
+ * k.s -> kubelet.service
+ * b.p -> bootkube.path
+ * b.s -> bootkube.service
+ * t.p -> tectonic.path
+ * t.s -> tectonic.service
+
+.------------------------------------------------------------------------------------------------------------------.
+| |
+| Provision cloud/userdata +----------+ Provision files |
+| ,----------------------------------------------o| TF |o-----------------.------------------------. |
+| | +----------+ | | |
+| | v v |
+| | +----------+ +-----+ +-------+ |
+| | .--->| (reboot) |----. | b.p | | t.p | |
+| | | +----------+ | +-----+ +-------+ |
+| V | | o o |
+| +-------+ | v Before +------------+ Before | Trigger Trigger | |
+| | IGN | | *---------->| k.s |o--------. | | |
+| +-------+ o ^ +------------+ | v v |
+| | +----------+ | ^ | | +-----+ Before +-------+ |
+| '------>| knb.s |o--------------' | v '--->| b.s |o--------------->| t.s | |
+| Enable +----------+ '------' +-----+ +-------+ |
+| ^ | |
+| | v |
+| '----' o o |
+| | | |
+| * First boot | * Each boot | * First boot |
+| * All nodes | * All nodes | * Bootkube master |
+| | | |
+'----------------------------------------------o----------------------------o--------------------------------------'
+```
diff --git a/Documentation/variables/config.md b/Documentation/variables/config.md
index 15cb87fac5..2802b86f86 100644
--- a/Documentation/variables/config.md
+++ b/Documentation/variables/config.md
@@ -9,6 +9,7 @@ This document gives an overview of variables used in all platforms of the Tecton
| tectonic_admin_email | The e-mail address used to: 1. login as the admin user to the Tectonic Console. 2. generate DNS zones for some providers.
Note: This field MUST be in all lower-case e-mail address format and set manually prior to creating the cluster. | string | - |
| tectonic_admin_password_hash | The bcrypt hash of admin user password to login to the Tectonic Console. Use the bcrypt-hash tool (https://github.com/coreos/bcrypt-tool/releases/tag/v1.0.0) to generate it.
Note: This field MUST be set manually prior to creating the cluster. | string | - |
| tectonic_base_domain | The base DNS domain of the cluster. It must NOT contain a trailing period. Some DNS providers will automatically add this if necessary.
Example: `openstack.dev.coreos.systems`.
Note: This field MUST be set manually prior to creating the cluster. This applies only to cloud platforms.
[Azure-specific NOTE] To use Azure-provided DNS, `tectonic_base_domain` should be set to `""` If using DNS records, ensure that `tectonic_base_domain` is set to a properly configured external DNS zone. Instructions for configuring delegated domains for Azure DNS can be found here: https://docs.microsoft.com/en-us/azure/dns/dns-delegate-domain-azure-dns | string | - |
+| tectonic_bootstrap_upgrade_cl | (internal) Whether to trigger a ContainerLinux upgrade on node bootstrap. | string | `true` |
| tectonic_ca_cert | (optional) The content of the PEM-encoded CA certificate, used to generate Tectonic Console's server certificate. If left blank, a CA certificate will be automatically generated. | string | `` |
| tectonic_ca_key | (optional) The content of the PEM-encoded CA key, used to generate Tectonic Console's server certificate. This field is mandatory if `tectonic_ca_cert` is set. | string | `` |
| tectonic_ca_key_alg | (optional) The algorithm used to generate tectonic_ca_key. The default value is currently recommended. This field is mandatory if `tectonic_ca_cert` is set. | string | `RSA` |
diff --git a/config.tf b/config.tf
index 2798fa57a9..c80d3a0230 100644
--- a/config.tf
+++ b/config.tf
@@ -82,6 +82,7 @@ variable "tectonic_container_images" {
tectonic_etcd_operator = "quay.io/coreos/tectonic-etcd-operator:v0.0.2"
tectonic_prometheus_operator = "quay.io/coreos/tectonic-prometheus-operator:v1.6.0"
tectonic_cluo_operator = "quay.io/coreos/tectonic-cluo-operator:v0.2.1"
+ tectonic_torcx = "quay.io/coreos/tectonic-torcx:installer-latest"
}
}
@@ -445,3 +446,9 @@ WARNING: Enabling an alpha feature means that future updates may become unsuppor
This should only be enabled on clusters that are meant to be short-lived to begin validating the alpha feature.
EOF
}
+
+variable "tectonic_bootstrap_upgrade_cl" {
+ type = "string"
+ default = "true"
+ description = "(internal) Whether to trigger a ContainerLinux upgrade on node bootstrap."
+}
diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf
index 06271b9107..b96f55fd1e 100644
--- a/modules/aws/master-asg/ignition.tf
+++ b/modules/aws/master-asg/ignition.tf
@@ -1,16 +1,17 @@
data "ignition_config" "main" {
files = [
+ "${data.ignition_file.detect_master.id}",
+ "${data.ignition_file.init_assets.id}",
+ "${var.ign_installer_kubelet_env_id}",
"${var.ign_max_user_watches_id}",
"${var.ign_s3_puller_id}",
- "${data.ignition_file.init_assets.id}",
- "${data.ignition_file.detect_master.id}",
]
systemd = ["${compact(list(
var.ign_docker_dropin_id,
var.ign_locksmithd_service_id,
var.ign_kubelet_service_id,
- var.ign_s3_kubelet_env_service_id,
+ var.ign_k8s_node_bootstrap_service_id,
data.ignition_systemd_unit.init_assets.id,
var.ign_bootkube_service_id,
var.ign_tectonic_service_id,
diff --git a/modules/aws/master-asg/resources/services/init-assets.service b/modules/aws/master-asg/resources/services/init-assets.service
index 889adc9e9d..47a2299881 100644
--- a/modules/aws/master-asg/resources/services/init-assets.service
+++ b/modules/aws/master-asg/resources/services/init-assets.service
@@ -1,7 +1,7 @@
[Unit]
Description=Download Tectonic Assets
ConditionPathExists=!/opt/init_assets.done
-Before=bootkube.service kubelet-env.service
+Before=bootkube.service k8s-node-bootstrap.service
[Service]
Type=oneshot
@@ -16,4 +16,4 @@ ExecStartPost=/bin/touch /opt/init_assets.done
[Install]
WantedBy=multi-user.target
-RequiredBy=bootkube.service kubelet-env.service
+RequiredBy=bootkube.service k8s-node-bootstrap.service
diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf
index b4fa9b8a47..1982de400c 100644
--- a/modules/aws/master-asg/variables.tf
+++ b/modules/aws/master-asg/variables.tf
@@ -62,10 +62,6 @@ variable "extra_tags" {
default = {}
}
-variable "ign_s3_kubelet_env_service_id" {
- type = "string"
-}
-
variable "ign_s3_puller_id" {
type = "string"
}
diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf
index f097a90044..b41b983615 100644
--- a/modules/aws/worker-asg/ignition.tf
+++ b/modules/aws/worker-asg/ignition.tf
@@ -1,13 +1,14 @@
data "ignition_config" "main" {
files = [
+ "${var.ign_installer_kubelet_env_id}",
"${var.ign_max_user_watches_id}",
"${var.ign_s3_puller_id}",
]
systemd = [
"${var.ign_docker_dropin_id}",
- "${var.ign_locksmithd_service_id}",
+ "${var.ign_k8s_node_bootstrap_service_id}",
"${var.ign_kubelet_service_id}",
- "${var.ign_s3_kubelet_env_service_id}",
+ "${var.ign_locksmithd_service_id}",
]
}
diff --git a/modules/aws/worker-asg/variables.tf b/modules/aws/worker-asg/variables.tf
index 0f78dcf30a..98f69743ef 100644
--- a/modules/aws/worker-asg/variables.tf
+++ b/modules/aws/worker-asg/variables.tf
@@ -78,7 +78,3 @@ variable "worker_iam_role" {
variable "ign_s3_puller_id" {
type = "string"
}
-
-variable "ign_s3_kubelet_env_service_id" {
- type = "string"
-}
diff --git a/modules/azure/master-as/ignition-master.tf b/modules/azure/master-as/ignition-master.tf
index 94ffa7c6be..56f26e44a8 100644
--- a/modules/azure/master-as/ignition-master.tf
+++ b/modules/azure/master-as/ignition-master.tf
@@ -1,7 +1,7 @@
data "ignition_config" "master" {
files = [
"${data.ignition_file.kubeconfig.id}",
- "${var.ign_kubelet_env_id}",
+ "${var.ign_installer_kubelet_env_id}",
"${var.ign_azure_udev_rules_id}",
"${var.ign_max_user_watches_id}",
"${data.ignition_file.cloud_provider_config.id}",
@@ -10,6 +10,7 @@ data "ignition_config" "master" {
systemd = ["${compact(list(
var.ign_docker_dropin_id,
var.ign_locksmithd_service_id,
+ var.ign_k8s_node_bootstrap_service_id,
var.ign_kubelet_service_id,
var.ign_tx_off_service_id,
var.ign_bootkube_service_id,
diff --git a/modules/azure/master-as/variables.tf b/modules/azure/master-as/variables.tf
index 1c78f2c368..51d22a03aa 100644
--- a/modules/azure/master-as/variables.tf
+++ b/modules/azure/master-as/variables.tf
@@ -23,10 +23,6 @@ variable "ign_azure_udev_rules_id" {
type = "string"
}
-variable "ign_kubelet_env_id" {
- type = "string"
-}
-
variable "ign_tx_off_service_id" {
type = "string"
}
diff --git a/modules/azure/worker-as/ignition-worker.tf b/modules/azure/worker-as/ignition-worker.tf
index d792d4a880..8bd3dc8291 100644
--- a/modules/azure/worker-as/ignition-worker.tf
+++ b/modules/azure/worker-as/ignition-worker.tf
@@ -1,7 +1,7 @@
data "ignition_config" "worker" {
files = [
"${data.ignition_file.kubeconfig.id}",
- "${var.ign_kubelet_env_id}",
+ "${var.ign_installer_kubelet_env_id}",
"${var.ign_azure_udev_rules_id}",
"${var.ign_max_user_watches_id}",
"${data.ignition_file.cloud-provider-config.id}",
@@ -10,6 +10,7 @@ data "ignition_config" "worker" {
systemd = [
"${var.ign_docker_dropin_id}",
"${var.ign_locksmithd_service_id}",
+ "${var.ign_k8s_node_bootstrap_service_id}",
"${var.ign_kubelet_service_id}",
"${var.ign_tx_off_service_id}",
]
diff --git a/modules/azure/worker-as/variables.tf b/modules/azure/worker-as/variables.tf
index edcef948be..1ada97e23a 100644
--- a/modules/azure/worker-as/variables.tf
+++ b/modules/azure/worker-as/variables.tf
@@ -29,10 +29,6 @@ variable "ign_azure_udev_rules_id" {
type = "string"
}
-variable "ign_kubelet_env_id" {
- type = "string"
-}
-
variable "ign_tx_off_service_id" {
type = "string"
}
diff --git a/modules/ignition/assets.tf b/modules/ignition/assets.tf
index 9f7390062b..2698914c9c 100644
--- a/modules/ignition/assets.tf
+++ b/modules/ignition/assets.tf
@@ -48,21 +48,23 @@ data "ignition_systemd_unit" "kubelet" {
content = "${data.template_file.kubelet.rendered}"
}
-data "template_file" "kubelet_env_service" {
- template = "${file("${path.module}/resources/services/kubelet-env.service")}"
+data "template_file" "k8s_node_bootstrap" {
+ template = "${file("${path.module}/resources/services/k8s-node-bootstrap.service")}"
vars {
- kube_version_image_url = "${replace(var.container_images["kube_version"],var.image_re,"$1")}"
- kube_version_image_tag = "${replace(var.container_images["kube_version"],var.image_re,"$2")}"
- kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}"
- kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}"
+ bootstrap_upgrade_cl = "${var.bootstrap_upgrade_cl}"
+ kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}"
+ tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}"
+ tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}"
+ torcx_skip_setup = "${var.tectonic_vanilla_k8s ? "true" : "false" }"
+ torcx_store_url = "${var.torcx_store_url}"
}
}
-data "ignition_systemd_unit" "kubelet_env" {
- name = "kubelet-env.service"
+data "ignition_systemd_unit" "k8s_node_bootstrap" {
+ name = "k8s-node-bootstrap.service"
enable = true
- content = "${data.template_file.kubelet_env_service.rendered}"
+ content = "${data.template_file.k8s_node_bootstrap.rendered}"
}
data "template_file" "s3_puller" {
@@ -88,7 +90,7 @@ data "ignition_systemd_unit" "locksmithd" {
mask = true
}
-data "template_file" "kubelet_env" {
+data "template_file" "installer_kubelet_env" {
template = "${file("${path.module}/resources/kubernetes/kubelet.env")}"
vars {
@@ -97,13 +99,13 @@ data "template_file" "kubelet_env" {
}
}
-data "ignition_file" "kubelet_env" {
+data "ignition_file" "installer_kubelet_env" {
filesystem = "root"
- path = "/etc/kubernetes/kubelet.env"
+ path = "/etc/kubernetes/installer/kubelet.env"
mode = 0644
content {
- content = "${data.template_file.kubelet_env.rendered}"
+ content = "${data.template_file.installer_kubelet_env.rendered}"
}
}
diff --git a/modules/ignition/outputs.import b/modules/ignition/outputs.import
index 90bfd3139d..6b5116fbbd 100644
--- a/modules/ignition/outputs.import
+++ b/modules/ignition/outputs.import
@@ -15,3 +15,11 @@ variable "ign_kubelet_service_id" {
variable "ign_locksmithd_service_id" {
type = "string"
}
+
+variable "ign_installer_kubelet_env_id" {
+ type = "string"
+}
+
+variable "ign_k8s_node_bootstrap_service_id" {
+ type = "string"
+}
diff --git a/modules/ignition/outputs.tf b/modules/ignition/outputs.tf
index 80a662b0a6..5665b83746 100644
--- a/modules/ignition/outputs.tf
+++ b/modules/ignition/outputs.tf
@@ -22,12 +22,12 @@ output "kubelet_service_rendered" {
value = "${data.template_file.kubelet.rendered}"
}
-output "kubelet_env_service_id" {
- value = "${data.ignition_systemd_unit.kubelet_env.id}"
+output "k8s_node_bootstrap_service_id" {
+ value = "${data.ignition_systemd_unit.k8s_node_bootstrap.id}"
}
-output "kubelet_env_service_rendered" {
- value = "${data.template_file.kubelet_env_service.rendered}"
+output "k8s_node_bootstrap_service_rendered" {
+ value = "${data.template_file.k8s_node_bootstrap.rendered}"
}
output "s3_puller_id" {
@@ -42,12 +42,12 @@ output "locksmithd_service_id" {
value = "${data.ignition_systemd_unit.locksmithd.id}"
}
-output "kubelet_env_id" {
- value = "${data.ignition_file.kubelet_env.id}"
+output "installer_kubelet_env_id" {
+ value = "${data.ignition_file.installer_kubelet_env.id}"
}
-output "kubelet_env_rendered" {
- value = "${data.template_file.kubelet_env.rendered}"
+output "installer_kubelet_env_rendered" {
+ value = "${data.template_file.installer_kubelet_env.rendered}"
}
output "tx_off_service_id" {
diff --git a/modules/ignition/resources/services/k8s-node-bootstrap.service b/modules/ignition/resources/services/k8s-node-bootstrap.service
new file mode 100644
index 0000000000..aab1559b6a
--- /dev/null
+++ b/modules/ignition/resources/services/k8s-node-bootstrap.service
@@ -0,0 +1,36 @@
+[Unit]
+Description=Bootstrap Kubernetes Node Components
+ConditionPathExists=!/etc/kubernetes/kubelet.env
+Before=kubelet.service
+
+[Service]
+Type=simple
+RemainAfterExit=true
+Restart=on-failure
+RestartSec=10
+TimeoutStartSec=1h
+ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes
+${kubeconfig_fetch_cmd}
+ExecStartPre=/usr/bin/docker run --rm \
+ --tmpfs /tmp \
+ -v /usr/share:/usr/share:ro \
+ -v /usr/lib/os-release:/usr/lib/os-release:ro \
+ -v /usr/share/ca-certificates/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro \
+ -v /var/lib/torcx:/var/lib/torcx \
+ -v /var/run/dbus:/var/run/dbus \
+ -v /run/metadata:/run/metadata:ro \
+ -v /run/torcx:/run/torcx:ro \
+ -v /run/systemd:/run/systemd \
+ -v /etc/coreos:/etc/coreos:ro \
+ -v /etc/torcx:/etc/torcx \
+ -v /etc/kubernetes:/etc/kubernetes \
+ ${tectonic_torcx_image_url}:${tectonic_torcx_image_tag} \
+ /tectonic-torcx-bootstrap \
+ --upgrade-os=${bootstrap_upgrade_cl} \
+ --torcx-manifest-url="${torcx_store_url}" \
+ --torcx-skip-setup=${torcx_skip_setup} \
+ --verbose=debug
+ExecStart=/usr/bin/echo "node components bootstrapped"
+
+[Install]
+WantedBy=multi-user.target kubelet.service
diff --git a/modules/ignition/resources/services/kubelet-env.service b/modules/ignition/resources/services/kubelet-env.service
deleted file mode 100644
index bdbec3f50b..0000000000
--- a/modules/ignition/resources/services/kubelet-env.service
+++ /dev/null
@@ -1,15 +0,0 @@
-[Unit]
-Description=Determine the Kubelet Image Version
-ConditionPathExists=!/etc/kubernetes/kubelet.env
-
-[Service]
-
-ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes
-${kubeconfig_fetch_cmd}
-ExecStartPre=/usr/bin/bash -c "docker run --rm -v /etc/kubernetes:/etc/kubernetes ${kube_version_image_url}:${kube_version_image_tag} --kubeconfig=/etc/kubernetes/kubeconfig > /etc/kubernetes/kube.version"
-ExecStart=/usr/bin/bash -c "echo KUBELET_IMAGE_URL=${kubelet_image_url} > /etc/kubernetes/kubelet.env; echo KUBELET_IMAGE_TAG=$(tr '+' '_' < /etc/kubernetes/kube.version) >> /etc/kubernetes/kubelet.env; rm /etc/kubernetes/kube.version"
-Restart=on-failure
-RestartSec=10
-
-[Install]
-WantedBy=multi-user.target
diff --git a/modules/ignition/variables.tf b/modules/ignition/variables.tf
index 55f4f00f5a..6e3cffc12b 100644
--- a/modules/ignition/variables.tf
+++ b/modules/ignition/variables.tf
@@ -48,3 +48,21 @@ variable "cloud_provider_config" {
description = "(optional) The cloud provider config to be used for the kubelet."
default = ""
}
+
+variable "bootstrap_upgrade_cl" {
+ type = "string"
+ description = "(optional) Whether to trigger a ContainerLinux OS upgrade during the bootstrap process."
+ default = "true"
+}
+
+variable "torcx_store_url" {
+ type = "string"
+ description = "(optional) URL template for torcx store. Leave empty to use the default CoreOS endpoint."
+ default = ""
+}
+
+variable "tectonic_vanilla_k8s" {
+ description = < /dev/null; do sleep 1; done'
[Install]
RequiredBy=kubelet.service
+ - name: k8s-node-bootstrap.service
+ enable: true
+ contents: {{.ign_k8s_node_bootstrap_service_json}}
- name: kubelet.service
enable: true
contents: {{.ign_kubelet_service_json}}
@@ -63,11 +66,6 @@ systemd:
{{end}}
storage:
files:
- - path: /etc/kubernetes/kubelet.env
- filesystem: root
- mode: 0644
- contents:
- inline: {{.ign_kubelet_env_json}}
- path: /etc/hostname
filesystem: root
mode: 0644
@@ -79,6 +77,11 @@ storage:
mode: 0644
contents:
inline: {{.ign_max_user_watches_json}}
+ - path: /etc/kubernetes/installer/kubelet.env
+ filesystem: root
+ mode: 0644
+ contents:
+ inline: {{.ign_installer_kubelet_env_json}}
passwd:
users:
- name: core
diff --git a/platforms/metal/cl/bootkube-worker.yaml.tmpl b/platforms/metal/cl/bootkube-worker.yaml.tmpl
index cb4bdd050d..620c296bc0 100644
--- a/platforms/metal/cl/bootkube-worker.yaml.tmpl
+++ b/platforms/metal/cl/bootkube-worker.yaml.tmpl
@@ -21,21 +21,20 @@ systemd:
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
[Install]
RequiredBy=kubelet.service
- - name: kubelet-env.service
+ - name: k8s-node-bootstrap.service
enable: true
- contents: {{.ign_kubelet_env_service_json}}
+ contents: {{.ign_k8s_node_bootstrap_service_json}}
- name: kubelet.service
enable: true
contents: {{.ign_kubelet_service_json}}
storage:
files:
- - path: /etc/kubernetes/.empty
+ - path: /etc/kubernetes/installer/kubelet.env
filesystem: root
mode: 0644
contents:
- inline: |
- empty
+ inline: {{.ign_installer_kubelet_env_json}}
- path: /etc/hostname
filesystem: root
mode: 0644
diff --git a/platforms/metal/matchers.tf b/platforms/metal/matchers.tf
index d83b044f5e..c1c2c97a6e 100644
--- a/platforms/metal/matchers.tf
+++ b/platforms/metal/matchers.tf
@@ -22,12 +22,14 @@ resource "matchbox_group" "coreos_install" {
module "ignition_masters" {
source = "../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/master"
- kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/master"
+ kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
resource "matchbox_group" "controller" {
@@ -60,30 +62,31 @@ resource "matchbox_group" "controller" {
etcd_tls_enabled = "${var.tectonic_etcd_tls_enabled}"
# extra data
- etcd_image_tag = "v${var.tectonic_versions["etcd"]}"
- kubelet_image_url = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$1")}"
- kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}"
-
- ign_bootkube_path_unit_json = "${jsonencode(module.bootkube.systemd_path_unit_rendered)}"
- ign_bootkube_service_json = "${jsonencode(module.bootkube.systemd_service_rendered)}"
- ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}"
- ign_kubelet_env_json = "${jsonencode(module.ignition_masters.kubelet_env_rendered)}"
- ign_kubelet_service_json = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}"
- ign_max_user_watches_json = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}"
- ign_tectonic_path_unit_json = "${jsonencode(module.tectonic.systemd_path_unit_rendered)}"
- ign_tectonic_service_json = "${jsonencode(module.tectonic.systemd_service_rendered)}"
+ etcd_image_tag = "v${var.tectonic_versions["etcd"]}"
+
+ ign_bootkube_path_unit_json = "${jsonencode(module.bootkube.systemd_path_unit_rendered)}"
+ ign_bootkube_service_json = "${jsonencode(module.bootkube.systemd_service_rendered)}"
+ ign_docker_dropin_json = "${jsonencode(module.ignition_masters.docker_dropin_rendered)}"
+ ign_installer_kubelet_env_json = "${jsonencode(module.ignition_masters.installer_kubelet_env_rendered)}"
+ ign_k8s_node_bootstrap_service_json = "${jsonencode(module.ignition_masters.k8s_node_bootstrap_service_rendered)}"
+ ign_kubelet_service_json = "${jsonencode(module.ignition_masters.kubelet_service_rendered)}"
+ ign_max_user_watches_json = "${jsonencode(module.ignition_masters.max_user_watches_rendered)}"
+ ign_tectonic_path_unit_json = "${jsonencode(module.tectonic.systemd_path_unit_rendered)}"
+ ign_tectonic_service_json = "${jsonencode(module.tectonic.systemd_service_rendered)}"
}
}
module "ignition_workers" {
source = "../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/node"
- kubelet_node_taints = ""
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/node"
+ kubelet_node_taints = ""
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
resource "matchbox_group" "worker" {
@@ -105,9 +108,10 @@ resource "matchbox_group" "worker" {
kubelet_image_tag = "${replace(var.tectonic_container_images["hyperkube"],var.tectonic_image_re,"$2")}"
kube_version_image = "${var.tectonic_container_images["kube_version"]}"
- ign_docker_dropin_json = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}"
- ign_kubelet_env_service_json = "${jsonencode(module.ignition_workers.kubelet_env_service_rendered)}"
- ign_kubelet_service_json = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}"
- ign_max_user_watches_json = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}"
+ ign_docker_dropin_json = "${jsonencode(module.ignition_workers.docker_dropin_rendered)}"
+ ign_installer_kubelet_env_json = "${jsonencode(module.ignition_workers.installer_kubelet_env_rendered)}"
+ ign_k8s_node_bootstrap_service_json = "${jsonencode(module.ignition_workers.k8s_node_bootstrap_service_rendered)}"
+ ign_kubelet_service_json = "${jsonencode(module.ignition_workers.kubelet_service_rendered)}"
+ ign_max_user_watches_json = "${jsonencode(module.ignition_workers.max_user_watches_rendered)}"
}
}
diff --git a/platforms/openstack/neutron/main.tf b/platforms/openstack/neutron/main.tf
index 9209e517db..fbaa0ee25e 100644
--- a/platforms/openstack/neutron/main.tf
+++ b/platforms/openstack/neutron/main.tf
@@ -178,12 +178,14 @@ data "null_data_source" "local" {
module "ignition_masters" {
source = "../../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/master"
- kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/master"
+ kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
module "master_nodes" {
@@ -200,26 +202,29 @@ EOF
instance_count = "${var.tectonic_master_count}"
kubeconfig_content = "${module.bootkube.kubeconfig}"
- ign_bootkube_path_unit_id = "${module.bootkube.systemd_path_unit_id}"
- ign_bootkube_service_id = "${module.bootkube.systemd_service_id}"
- ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}"
- ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}"
- ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}"
- ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
- ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}"
- ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
- ign_tectonic_service_id = "${module.tectonic.systemd_service_id}"
+ ign_bootkube_path_unit_id = "${module.bootkube.systemd_path_unit_id}"
+ ign_bootkube_service_id = "${module.bootkube.systemd_service_id}"
+ ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}"
+ ign_installer_kubelet_env_id = "${module.ignition_masters.installer_kubelet_env_id}"
+ ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+ ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}"
+ ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
+ ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}"
+ ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+ ign_tectonic_service_id = "${module.tectonic.systemd_service_id}"
}
module "ignition_workers" {
source = "../../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/node"
- kubelet_node_taints = ""
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/node"
+ kubelet_node_taints = ""
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
module "worker_nodes" {
@@ -236,11 +241,12 @@ EOF
instance_count = "${var.tectonic_worker_count}"
kubeconfig_content = "${module.bootkube.kubeconfig}"
- ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}"
- ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}"
- ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}"
- ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}"
- ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}"
+ ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}"
+ ign_installer_kubelet_env_id = "${module.ignition_workers.installer_kubelet_env_id}"
+ ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+ ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}"
+ ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}"
+ ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}"
}
module "secrets" {
diff --git a/platforms/vmware/main.tf b/platforms/vmware/main.tf
index fdd64eb368..f1cedd7da3 100644
--- a/platforms/vmware/main.tf
+++ b/platforms/vmware/main.tf
@@ -39,12 +39,14 @@ module "etcd" {
module "ignition_masters" {
source = "../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/master"
- kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/master"
+ kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule"
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
module "masters" {
@@ -72,27 +74,29 @@ module "masters" {
private_key = "${var.tectonic_vmware_ssh_private_key_path}"
image_re = "${var.tectonic_image_re}"
- ign_bootkube_path_unit_id = "${module.bootkube.systemd_path_unit_id}"
- ign_bootkube_service_id = "${module.bootkube.systemd_service_id}"
- ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}"
- ign_kubelet_env_id = "${module.ignition_masters.kubelet_env_id}"
- ign_kubelet_env_service_id = "${module.ignition_masters.kubelet_env_service_id}"
- ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}"
- ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
- ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}"
- ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
- ign_tectonic_service_id = "${module.tectonic.systemd_service_id}"
+ ign_bootkube_path_unit_id = "${module.bootkube.systemd_path_unit_id}"
+ ign_bootkube_service_id = "${module.bootkube.systemd_service_id}"
+ ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}"
+ ign_installer_kubelet_env_id = "${module.ignition_masters.installer_kubelet_env_id}"
+ ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}"
+ ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}"
+ ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}"
+ ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}"
+ ign_tectonic_path_unit_id = "${var.tectonic_vanilla_k8s ? "" : module.tectonic.systemd_path_unit_id}"
+ ign_tectonic_service_id = "${module.tectonic.systemd_service_id}"
}
module "ignition_workers" {
source = "../../modules/ignition"
- container_images = "${var.tectonic_container_images}"
- image_re = "${var.tectonic_image_re}"
- kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
- kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
- kubelet_node_label = "node-role.kubernetes.io/node"
- kubelet_node_taints = ""
+ bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}"
+ container_images = "${var.tectonic_container_images}"
+ image_re = "${var.tectonic_image_re}"
+ kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+ kubelet_cni_bin_dir = "${var.tectonic_calico_network_policy ? "/var/lib/cni/bin" : "" }"
+ kubelet_node_label = "node-role.kubernetes.io/node"
+ kubelet_node_taints = ""
+ tectonic_vanilla_k8s = "${var.tectonic_vanilla_k8s}"
}
module "workers" {
@@ -120,10 +124,10 @@ module "workers" {
private_key = "${var.tectonic_vmware_ssh_private_key_path}"
image_re = "${var.tectonic_image_re}"
- ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}"
- ign_kubelet_env_id = "${module.ignition_workers.kubelet_env_id}"
- ign_kubelet_env_service_id = "${module.ignition_workers.kubelet_env_service_id}"
- ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}"
- ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}"
- ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}"
+ ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}"
+ ign_installer_kubelet_env_id = "${module.ignition_workers.installer_kubelet_env_id}"
+ ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}"
+ ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}"
+ ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}"
+ ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}"
}
diff --git a/tests/smoke/bare-metal/smoke.sh b/tests/smoke/bare-metal/smoke.sh
index 00b9064737..b0b9059311 100755
--- a/tests/smoke/bare-metal/smoke.sh
+++ b/tests/smoke/bare-metal/smoke.sh
@@ -206,6 +206,11 @@ kill_terraform_and_cleanup() {
}
kubelet_up() {
+ ssh -q -i ${ROOT}/matchbox/tests/smoke/fake_rsa \
+ -o StrictHostKeyChecking=no \
+ -o UserKnownHostsFile=/dev/null \
+ -o PreferredAuthentications=publickey \
+ core@$1 /usr/bin/systemctl status k8s-node-bootstrap kubelet
curl --silent --fail -m 1 "http://$1:10255/healthz" > /dev/null
}