*: use HTTPS when serving Ignition configs

config.tf

@@ -61,18 +61,18 @@ variable "tectonic_container_images" {
     awscli                               = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
     gcloudsdk                            = "google/cloud-sdk:178.0.0-alpine"
     bootkube                             = "quay.io/coreos/bootkube:v0.10.0"
-    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
+    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
     etcd_cert_signer                     = "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6"
     etcd                                 = "quay.io/coreos/etcd:v3.2.14"
     hyperkube                            = "openshift/origin-node:latest"
-    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
-    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
-    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
-    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
+    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
+    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
+    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
+    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
     tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.3.1"
-    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
-    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
-    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:c7add4ccd821710b4b5d012c6af9fd4f8e9a288b"
+    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
+    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
+    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:8563b8b3355a44eb17c2981da3fb42b4c2453646"
   }
 }
 

glide.yaml

@@ -17,4 +17,6 @@ import:
 - package: golang.org/x/crypto
   version: df8d4716b3472e4a531c33cedbe537dae921a1a9
 - package: github.com/coreos/ignition
-  version: v0.23.0
+  version: v0.26.0
+- package: github.com/vincent-petithory/dataurl
+  version: 9a301d65acbb728fcc3ace14f45f511a4cfeea9c

installer/pkg/config-generator/BUILD.bazel

@@ -11,14 +11,15 @@ go_library(
     deps = [
         "//installer/pkg/config:go_default_library",
         "//vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library",
-        "//vendor/github.com/coreos/ignition/config/v2_0:go_default_library",
-        "//vendor/github.com/coreos/ignition/config/v2_0/types:go_default_library",
+        "//vendor/github.com/coreos/ignition/config/v2_2:go_default_library",
+        "//vendor/github.com/coreos/ignition/config/v2_2/types:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/kube-addon:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/kube-core:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/tectonic-network:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/tectonic-node-controller:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/tectonic-utility:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
+        "//vendor/github.com/vincent-petithory/dataurl:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )

installer/pkg/config-generator/ignition.go

@@ -5,21 +5,24 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"path/filepath"
 
-	ignconfig "github.com/coreos/ignition/config/v2_0"
-	ignconfigtypes "github.com/coreos/ignition/config/v2_0/types"
+	ignconfig "github.com/coreos/ignition/config/v2_2"
+	ignconfigtypes "github.com/coreos/ignition/config/v2_2/types"
 	"github.com/coreos/tectonic-installer/installer/pkg/config"
+	"github.com/vincent-petithory/dataurl"
 )
 
 var (
-	ignVersion   = ignconfigtypes.IgnitionVersion{Major: 2, Minor: 0, Patch: 0}
+	ignVersion   = "2.2.0"
 	ignFilesPath = map[string]string{
 		"master": config.IgnitionMaster,
 		"worker": config.IgnitionWorker,
 		"etcd":   config.IgnitionEtcd,
 	}
+	caPath = "generated/tls/root-ca.crt"
 )
 
 func (c *ConfigGenerator) poolToRoleMap() map[string]string {
@@ -51,6 +54,11 @@ func (c *ConfigGenerator) GenerateIgnConfig(clusterDir string) error {
 		// add loop over count if role is etcd
 		c.embedAppendBlock(ignCfg, role)
 
+		ca := filepath.Join(clusterDir, caPath)
+		if err = c.appendCertificateAuthority(ignCfg, ca); err != nil {
+			return err
+		}
+
 		// agentless platforms (e.g. libvirt) need to embed the ssh key
 		c.embedUserBlock(ignCfg)
 
@@ -91,36 +99,59 @@ func (c *ConfigGenerator) embedAppendBlock(ignCfg *ignconfigtypes.Config, role s
 	ignCfg.Ignition.Config.Append = append(ignCfg.Ignition.Config.Append, appendBlock)
 }
 
+func (c *ConfigGenerator) appendCertificateAuthority(ignCfg *ignconfigtypes.Config, caPath string) error {
+	ca, err := ioutil.ReadFile(caPath)
+	if err != nil {
+		return err
+	}
+
+	ignCfg.Ignition.Security.TLS.CertificateAuthorities = append(ignCfg.Ignition.Security.TLS.CertificateAuthorities, ignconfigtypes.CaReference{
+		Source: dataurl.EncodeBytes(ca),
+	})
+
+	return nil
+}
+
 func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
-	if c.Platform == "libvirt" {
-		userBlock := ignconfigtypes.User{
+	if c.Platform.String() == config.PlatformLibvirt.String() {
+		userBlock := ignconfigtypes.PasswdUser{
 			Name: "core",
-			SSHAuthorizedKeys: []string{
-				c.Libvirt.SSHKey,
+			SSHAuthorizedKeys: []ignconfigtypes.SSHAuthorizedKey{
+				ignconfigtypes.SSHAuthorizedKey(c.Libvirt.SSHKey),
 			},
 		}
 
 		ignCfg.Passwd.Users = append(ignCfg.Passwd.Users, userBlock)
 	}
 }
 
-func (c *ConfigGenerator) getTNCURL(role string) ignconfigtypes.Url {
-	var url ignconfigtypes.Url
+func (c *ConfigGenerator) getTNCURL(role string) string {
+	var u string
 
 	// cloud platforms put this behind a load balancer which remaps ports;
 	// libvirt doesn't do that - use the tnc port directly
 	port := 80
-	if c.Platform == "libvirt" {
+	if c.Platform.String() == config.PlatformLibvirt.String() {
 		port = 49500
 	}
+
+	// XXX: The bootstrap node on AWS uses a CNAME to redirect TNC-bound
+	// traffic to S3. Because of this, HTTPS cannot be used.
+	scheme := "https"
+	if c.Platform.String() == config.PlatformAWS.String() && role == "master" {
+		scheme = "http"
+	}
+
 	if role == "master" || role == "worker" {
-		url = ignconfigtypes.Url{
-			Scheme: "http",
-			Host:   fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
-			Path:   fmt.Sprintf("/config/%s", role),
-		}
+		u = func() *url.URL {
+			return &url.URL{
+				Scheme: scheme,
+				Host:   fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
+				Path:   fmt.Sprintf("/config/%s", role),
+			}
+		}().String()
 	}
-	return url
+	return u
 }
 
 func ignCfgToFile(ignCfg ignconfigtypes.Config, filePath string) error {

installer/pkg/config/BUILD.bazel

@@ -15,7 +15,7 @@ go_library(
         "//installer/pkg/config/libvirt:go_default_library",
         "//installer/pkg/validate:go_default_library",
         "//vendor/github.com/Sirupsen/logrus:go_default_library",
-        "//vendor/github.com/coreos/ignition/config/v2_0:go_default_library",
+        "//vendor/github.com/coreos/ignition/config/v2_2:go_default_library",
         "//vendor/github.com/coreos/tectonic-config/config/tectonic-network:go_default_library",
         "//vendor/gopkg.in/yaml.v2:go_default_library",
     ],

installer/pkg/config/validate.go

@@ -11,7 +11,7 @@ import (
 	"github.com/coreos/tectonic-installer/installer/pkg/validate"
 
 	log "github.com/Sirupsen/logrus"
-	ignconfig "github.com/coreos/ignition/config/v2_0"
+	ignconfig "github.com/coreos/ignition/config/v2_2"
 	"github.com/coreos/tectonic-config/config/tectonic-network"
 )
 

modules/bootkube/manifests.tf

@@ -14,6 +14,7 @@ variable "manifest_names" {
     "pull.json",
     "tectonic-network-operator.yaml",
     "tectonic-node-controller-operator.yaml",
+    "tnc-tls-secret.yaml",
   ]
 }
 
@@ -52,6 +53,9 @@ data "template_file" "manifest_file_list" {
     etcd_ca_cert     = "${base64encode(var.etcd_ca_cert_pem)}"
     etcd_client_cert = "${base64encode(var.etcd_client_cert_pem)}"
     etcd_client_key  = "${base64encode(var.etcd_client_key_pem)}"
+
+    tnc_tls_cert = "${base64encode(var.tnc_cert_pem)}"
+    tnc_tls_key  = "${base64encode(var.tnc_key_pem)}"
   }
 }
 

modules/bootkube/resources/bootkube.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -e
 
+echo "Rendering Kubernetes core manifests..."
+
 # shellcheck disable=SC2154
 /usr/bin/docker run \
     --volume "$(pwd)":/assets \
@@ -9,6 +11,8 @@ set -e
     --config=/assets/kco-config.yaml \
     --output=/assets
 
+echo "Rendering TNC manifests..."
+
 # shellcheck disable=SC2154
 /usr/bin/docker run \
     --user 0 \
@@ -28,6 +32,8 @@ cp "$(pwd)/tnc-bootstrap/tectonic-node-controller-config.yaml" /etc/kubernetes/t
 # docker container.
 # See https://github.com/kubernetes/kubernetes/issues/43292
 
+echo "Starting etcd certificate signer..."
+
 # shellcheck disable=SC2154,SC2034
 signer_id=$(/usr/bin/docker run -d \
     --tmpfs /tmp \
@@ -44,24 +50,48 @@ signer_id=$(/usr/bin/docker run -d \
     --peercertdur=26280h \
     --servercertdur=26280h)
 
+echo "Waiting for etcd cluster..."
+
 # Wait for the etcd cluster to come up.
 export ETCDCTL_API=3
-# shellcheck disable=SC2154,SC2086
-/usr/bin/etcdctl \
-    --dial-timeout=10m \
-    --cacert=/opt/tectonic/tls/etcd-client-ca.crt \
-    --cert=/opt/tectonic/tls/etcd-client.crt \
-    --key=/opt/tectonic/tls/etcd-client.key \
-    --endpoints=${etcd_cluster} \
-    endpoint health
+
+i=0
+while true; do
+    set +e
+    # shellcheck disable=SC2154,SC2086
+    /usr/bin/etcdctl \
+        --dial-timeout=10m \
+        --cacert=/opt/tectonic/tls/etcd-client-ca.crt \
+        --cert=/opt/tectonic/tls/etcd-client.crt \
+        --key=/opt/tectonic/tls/etcd-client.key \
+        --endpoints=${etcd_cluster} \
+        endpoint health
+    status=$?
+    set -e
+
+    if [ "$status" -eq 0 ]; then
+      break
+    fi
+
+    i=$((i+1))
+    [ $i -eq 10 ] && echo "etcdctl failed too many times." && exit 1
+
+    echo "etcdctl failed. Retrying in 5 seconds..."
+    sleep 5
+done
+
 export ETCDCTL_API=
 
+echo "etcd cluster up. Killing etcd certificate signer..."
+
 # shellcheck disable=SC2154,SC1083
 /usr/bin/docker kill $${signer_id}
 rm /etc/kubernetes/manifests/tectonic-node-controller-pod.yaml
 
 cp -r "$(pwd)/bootstrap-configs" /etc/kubernetes/bootstrap-configs
 
+echo "Starting bootkube..."
+
 # shellcheck disable=SC2154
 /usr/bin/docker run \
     --volume "$(pwd)":/assets \

modules/bootkube/variables.tf

@@ -123,6 +123,14 @@ variable "kubelet_key_pem" {
   description = "The kubelet key in PEM format."
 }
 
+variable "tnc_cert_pem" {
+  type = "string"
+}
+
+variable "tnc_key_pem" {
+  type = "string"
+}
+
 variable "oidc_ca_cert" {
   type = "string"
 }

modules/tectonic/manifests.tf

@@ -13,7 +13,6 @@ variable "manifest_names" {
     "rbac/role-user.yaml",
     "secrets/ca-cert.yaml",
     "secrets/ingress-tls.yaml",
-    "secrets/tnc-tls.yaml",
     "secrets/license.json",
     "secrets/pull.json",
     "security/priviledged-scc-tectonic.yaml",
@@ -74,9 +73,6 @@ data "template_file" "manifest_file_list" {
     ingress_tls_key         = "${base64encode(var.ingress_key_pem)}"
     ingress_tls_bundle      = "${base64encode(var.ingress_bundle_pem)}"
 
-    tnc_tls_cert = "${base64encode(var.tnc_cert_pem)}"
-    tnc_tls_key  = "${base64encode(var.tnc_key_pem)}"
-
     platform = "${var.platform}"
   }
 }

modules/tectonic/resources/tectonic.sh

@@ -144,7 +144,6 @@ echo "Creating Tectonic Secrets"
 kubectl create -f secrets/pull.json
 kubectl create -f secrets/license.json
 kubectl create -f secrets/ingress-tls.yaml
-kubectl create -f secrets/tnc-tls.yaml
 kubectl create -f secrets/ca-cert.yaml
 kubectl create -f ingress/pull.json
 

modules/tectonic/variables.tf

@@ -70,11 +70,3 @@ variable "ingress_key_pem" {
 variable "ingress_bundle_pem" {
   type = "string"
 }
-
-variable "tnc_cert_pem" {
-  type = "string"
-}
-
-variable "tnc_key_pem" {
-  type = "string"
-}