run TNC as a static pod before cluster having state

modules/bootkube-ut2/assets.tf

@@ -46,25 +46,7 @@ resource "template_dir" "bootkube" {
     etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
     etcd_client_key  = "${base64encode(var.etcd_client_key_pem)}"
 
-    http_proxy               = "${var.http_proxy}"
-    https_proxy              = "${var.https_proxy}"
-    no_proxy                 = "${join(",", var.no_proxy)}"
-    kubelet_image_url        = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}"
-    kubelet_image_tag        = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}"
-    iscsi_enabled            = "${var.iscsi_enabled}"
-    kubeconfig_fetch_cmd     = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}"
-    tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}"
-    tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}"
-    torcx_skip_setup         = "false"
-    torcx_store_url          = "${var.torcx_store_url}"
-    bootstrap_upgrade_cl     = "${var.bootstrap_upgrade_cl}"
-    master_node_label        = "${var.kubelet_master_node_label}"
-    worker_node_label        = "${var.kubelet_worker_node_label}"
-    node_taints_param        = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}"
-    cluster_dns_ip           = "${var.kube_dns_service_ip}"
-    cloud_provider           = "${var.cloud_provider}"
-    debug_config             = "${var.kubelet_debug_config}"
-    cluster_name             = "${var.cluster_name}"
+    tnc_config = "${indent(4, chomp(data.template_file.tnc_config.rendered))}"
   }
 }
 
@@ -169,3 +151,37 @@ data "ignition_systemd_unit" "bootkube_path_unit" {
   enabled = true
   content = "${data.template_file.bootkube_path_unit.rendered}"
 }
+
+# TNC
+resource "local_file" "tnc_pod_config" {
+  content  = "${data.template_file.tnc_config.rendered}"
+  filename = "./generated/tnc-config"
+}
+
+data "template_file" "tnc_config" {
+  template = "${file("${path.module}/resources/tnc-config")}"
+
+  vars {
+    cloud_provider_config = "${var.cloud_provider_config}"
+
+    http_proxy               = "${var.http_proxy}"
+    https_proxy              = "${var.https_proxy}"
+    no_proxy                 = "${join(",", var.no_proxy)}"
+    kubelet_image_url        = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}"
+    kubelet_image_tag        = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}"
+    iscsi_enabled            = "${var.iscsi_enabled}"
+    kubeconfig_fetch_cmd     = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}"
+    tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}"
+    tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}"
+    torcx_skip_setup         = "false"
+    torcx_store_url          = "${var.torcx_store_url}"
+    bootstrap_upgrade_cl     = "${var.bootstrap_upgrade_cl}"
+    master_node_label        = "${var.kubelet_master_node_label}"
+    worker_node_label        = "${var.kubelet_worker_node_label}"
+    node_taints_param        = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}"
+    cluster_dns_ip           = "${var.kube_dns_service_ip}"
+    cloud_provider           = "${var.cloud_provider}"
+    debug_config             = "${var.kubelet_debug_config}"
+    cluster_name             = "${var.cluster_name}"
+  }
+}

modules/bootkube-ut2/resources/bootkube.sh

@@ -10,6 +10,7 @@ set -e
     --output=/assets
 
 mkdir -p /etc/kubernetes/manifests/
+mv /opt/tectonic/manifests/tectonic-node-controller-pod.yaml /etc/kubernetes/manifests/
 
 # shellcheck disable=SC2154
 /usr/bin/docker run \

modules/bootkube-ut2/resources/manifests/tectonic-node-controller-config.yaml

@@ -5,23 +5,4 @@ metadata:
   namespace: kube-system
 data:
   tnc-config: |
-    HTTPProxy: "${http_proxy}"
-    HTTPSProxy: "${https_proxy}"
-    NoProxy: "${no_proxy}"
-    KubeletImageUrl: "${kubelet_image_url}"
-    KubeletImageTag: "${kubelet_image_tag}"
-    IscsiEnabled: "${iscsi_enabled}"
-    KubeconfigFetchCmd: "${kubeconfig_fetch_cmd}"
-    TectonicTorcxImageURL: "${tectonic_torcx_image_url}"
-    TectonicTorcxImageTag: "${tectonic_torcx_image_tag}"
-    BootstrapUpgradeCl: "${bootstrap_upgrade_cl}"
-    TorcxStoreULL: "${torcx_store_url}"
-    TorcxSkipSetup: "${torcx_skip_setup}"
-    MasterNodeLabel: "${master_node_label}"
-    WorkerNodeLabel: "${worker_node_label}"
-    NodeTaintsParam: "${node_taints_param}"
-    ClusterDNSIP: "${cluster_dns_ip}"
-    CloudProvider: "${cloud_provider}"
-    CloudProviderConfig: "${cloud_provider_config}"
-    DebugConfig: "${debug_config}"
-    ClusterName: "${cluster_name}"
+    ${tnc_config}

modules/bootkube-ut2/resources/manifests/tectonic-node-controller-pod.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: tectonic-node-controller
+  namespace: kube-system
+  labels:
+    k8s-app: tectonic-node-controller
+spec:
+  containers:
+  - name: tectonic-node-controller
+    image: ${tnc_bootstrap_image}
+    args:
+    - --config=/etc/cluster-config/tnc-config
+    - --port=49500
+    - --debug
+    resources:
+      limits:
+        cpu: 20m
+        memory: 50Mi
+      requests:
+        cpu: 20m
+        memory: 50Mi
+    volumeMounts:
+    - name: cluster-config
+      mountPath: /etc/cluster-config/tnc-config
+  hostNetwork: true
+  restartPolicy: Always
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+  volumes:
+  - name: cluster-config
+    hostPath:
+      path: /opt/tectonic/tnc-config

modules/bootkube-ut2/resources/tnc-config

@@ -0,0 +1,20 @@
+HTTPProxy: "${http_proxy}"
+HTTPSProxy: "${https_proxy}"
+NoProxy: "${no_proxy}"
+KubeletImageUrl: "${kubelet_image_url}"
+KubeletImageTag: "${kubelet_image_tag}"
+IscsiEnabled: "${iscsi_enabled}"
+KubeconfigFetchCmd: "${kubeconfig_fetch_cmd}"
+TectonicTorcxImageURL: "${tectonic_torcx_image_url}"
+TectonicTorcxImageTag: "${tectonic_torcx_image_tag}"
+BootstrapUpgradeCl: "${bootstrap_upgrade_cl}"
+TorcxStoreULL: "${torcx_store_url}"
+TorcxSkipSetup: "${torcx_skip_setup}"
+MasterNodeLabel: "${master_node_label}"
+WorkerNodeLabel: "${worker_node_label}"
+NodeTaintsParam: "${node_taints_param}"
+ClusterDNSIP: "${cluster_dns_ip}"
+CloudProvider: "${cloud_provider}"
+CloudProviderConfig: "${cloud_provider_config}"
+DebugConfig: "${debug_config}"
+ClusterName: "${cluster_name}"

modules/tectonic/resources/tectonic-wrapper.sh

@@ -8,3 +8,6 @@ set -e
     --entrypoint=/bin/sh \
     ${hyperkube_image} \
     /assets/tectonic.sh /assets/auth/kubeconfig /assets
+
+# remove TNC static pod
+rm -f /etc/kubernetes/manifests/tectonic-node-controller-pod.yaml

tests/smoke/cluster_test.go

@@ -43,6 +43,7 @@ var (
 	defaultIgnoredManifests = []string{
 		"bootstrap",
 		"kco-config.yaml",
+		"tectonic-node-controller-pod.yaml",
 	}
 
 	// equivalentKindRemapping is used by resourceIdentifier to map different