diff --git a/config.tf b/config.tf index 973f2aca68..ec9a5221a8 100644 --- a/config.tf +++ b/config.tf @@ -71,7 +71,7 @@ variable "tectonic_container_images" { awscli = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600" gcloudsdk = "google/cloud-sdk:178.0.0-alpine" bootkube = "quay.io/coreos/bootkube:v0.10.0" - tnc_bootstrap = "quay.io/coreos/tectonic-node-controller-dev:76a584680b7f39aa7b3c40cd742c736b30b5a89a" + tnc_bootstrap = "quay.io/coreos/tectonic-node-controller-bootstrap-dev:f6d5e710a97a8cd6f4cd2963f4426131f854a869" etcd = "quay.io/coreos/etcd:v3.2.14" hyperkube = "quay.io/coreos/hyperkube:v1.9.1_coreos.0" kube_core_renderer = "quay.io/coreos/kube-core-renderer-dev:4ed85ee12e167da71e7d5f06ffdb94d1ce21f540" diff --git a/installer/pkg/workflow/destroy.go b/installer/pkg/workflow/destroy.go index 5b8fc0da71..d010da69af 100644 --- a/installer/pkg/workflow/destroy.go +++ b/installer/pkg/workflow/destroy.go @@ -20,6 +20,9 @@ func destroyAssetsStep(m *metadata) error { } func destroyBootstrapStep(m *metadata) error { + if err := runDestroyStep(m.clusterDir, etcdStep); err != nil { + return err + } return runDestroyStep(m.clusterDir, bootstrapStep) } diff --git a/installer/pkg/workflow/install.go b/installer/pkg/workflow/install.go index 8d45bc2ac0..b17f0bbcfb 100644 --- a/installer/pkg/workflow/install.go +++ b/installer/pkg/workflow/install.go @@ -61,11 +61,13 @@ func installBootstrapStep(m *metadata) error { return err } - if err := waitForTNC(m); err != nil { + destroyCNAME(m.clusterDir) + + if err := runInstallStep(m.clusterDir, etcdStep); err != nil { return err } - return destroyCNAME(m.clusterDir) + return waitForTNC(m) } func installJoinStep(m *metadata) error { diff --git a/installer/pkg/workflow/utils.go b/installer/pkg/workflow/utils.go index dd02c39da8..0f3a005340 100644 --- a/installer/pkg/workflow/utils.go +++ b/installer/pkg/workflow/utils.go @@ -24,6 +24,7 @@ const ( stepsBaseDir = "steps" assetsStep = "assets" bootstrapStep = "bootstrap" + etcdStep = "etcd" joinStep = "joining" configFileName = "config.yaml" internalFileName = "internal.yaml" diff --git a/modules/aws/etcd/ignition_s3.tf b/modules/aws/etcd/ignition.tf similarity index 56% rename from modules/aws/etcd/ignition_s3.tf rename to modules/aws/etcd/ignition.tf index 98cfa44453..6d6183fdd9 100644 --- a/modules/aws/etcd/ignition_s3.tf +++ b/modules/aws/etcd/ignition.tf @@ -2,10 +2,17 @@ locals { ignition_etcd_keys = ["ignition_etcd_0.json", "ignition_etcd_1.json", "ignition_etcd_2.json"] } -data "ignition_config" "s3" { +data "ignition_config" "tnc" { count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}" - replace { + append { + source = "${format("http://${var.cluster_name}-tnc.${var.base_domain}/ignition?role=etcd&etcd_index=%d", count.index)}" + + # TODO: add verification + } + + # Used for loading certificates + append { source = "${format("s3://%s/%s", var.s3_bucket, local.ignition_etcd_keys[count.index])}" # TODO: add verification diff --git a/modules/aws/etcd/nodes.tf b/modules/aws/etcd/nodes.tf index 56c6c4a5d5..8c78a64271 100644 --- a/modules/aws/etcd/nodes.tf +++ b/modules/aws/etcd/nodes.tf @@ -106,7 +106,7 @@ resource "aws_instance" "etcd_node" { instance_type = "${var.ec2_type}" key_name = "${var.ssh_key}" subnet_id = "${element(var.subnets, count.index)}" - user_data = "${data.ignition_config.s3.*.rendered[count.index]}" + user_data = "${data.ignition_config.tnc.*.rendered[count.index]}" vpc_security_group_ids = ["${var.sg_ids}"] lifecycle { diff --git a/modules/bootkube/assets.tf b/modules/bootkube/assets.tf index 39b9ececbb..fc65948404 100644 --- a/modules/bootkube/assets.tf +++ b/modules/bootkube/assets.tf @@ -159,30 +159,40 @@ resource "local_file" "tnc_pod_config" { filename = "./generated/tnc-config" } +data "template_file" "initial_cluster" { + count = "${length(var.etcd_endpoints)}" + template = "${var.etcd_endpoints[count.index]}=https://${var.etcd_endpoints[count.index]}:2380" +} + data "template_file" "tnc_config" { template = "${file("${path.module}/resources/tnc-config")}" vars { cloud_provider_config = "${var.cloud_provider_config}" - http_proxy = "${var.http_proxy}" - https_proxy = "${var.https_proxy}" - no_proxy = "${join(",", var.no_proxy)}" - kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}" - kubelet_image_tag = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}" - iscsi_enabled = "${var.iscsi_enabled}" - kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" - tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}" - tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}" - torcx_skip_setup = "false" - torcx_store_url = "${var.torcx_store_url}" - bootstrap_upgrade_cl = "${var.bootstrap_upgrade_cl}" - master_node_label = "${var.kubelet_master_node_label}" - worker_node_label = "${var.kubelet_worker_node_label}" - node_taints_param = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}" - cluster_dns_ip = "${var.kube_dns_service_ip}" - cloud_provider = "${var.cloud_provider}" - debug_config = "${var.kubelet_debug_config}" - cluster_name = "${var.cluster_name}" + http_proxy = "${var.http_proxy}" + https_proxy = "${var.https_proxy}" + no_proxy = "${join(",", var.no_proxy)}" + kubelet_image_url = "${replace(var.container_images["hyperkube"],var.image_re,"$1")}" + kubelet_image_tag = "${replace(var.container_images["hyperkube"],var.image_re,"$2")}" + iscsi_enabled = "${var.iscsi_enabled}" + kubeconfig_fetch_cmd = "${var.kubeconfig_fetch_cmd != "" ? "ExecStartPre=${var.kubeconfig_fetch_cmd}" : ""}" + tectonic_torcx_image_url = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$1")}" + tectonic_torcx_image_tag = "${replace(var.container_images["tectonic_torcx"],var.image_re,"$2")}" + torcx_skip_setup = "false" + torcx_store_url = "${var.torcx_store_url}" + bootstrap_upgrade_cl = "${var.bootstrap_upgrade_cl}" + master_node_label = "${var.kubelet_master_node_label}" + worker_node_label = "${var.kubelet_worker_node_label}" + node_taints_param = "${var.kubelet_node_taints != "" ? "--register-with-taints=${var.kubelet_node_taints}" : ""}" + cluster_dns_ip = "${var.kube_dns_service_ip}" + cloud_provider = "${var.cloud_provider}" + debug_config = "${var.kubelet_debug_config}" + cluster_name = "${var.cluster_name}" + base_domain = "${var.base_domain}" + etcd_initial_cluster_list = "${length(var.etcd_endpoints) > 0 ? format("--initial-cluster=%s", join(",", data.template_file.initial_cluster.*.rendered)) : ""}" + etcd_image = "${var.container_images["etcd"]}" + etcd_metadata_env = "${var.etcd_metadata_env}" + etcd_metadata_deps = "${var.etcd_metadata_deps}" } } diff --git a/modules/bootkube/resources/tnc-config b/modules/bootkube/resources/tnc-config index 945943738b..5ba5114fde 100644 --- a/modules/bootkube/resources/tnc-config +++ b/modules/bootkube/resources/tnc-config @@ -18,3 +18,8 @@ CloudProvider: "${cloud_provider}" CloudProviderConfig: "${cloud_provider_config}" DebugConfig: "${debug_config}" ClusterName: "${cluster_name}" +BaseDomain: "${base_domain}" +EtcdInitialCluster: "${etcd_initial_cluster_list}" +EtcdImage: "${etcd_image}" +EtcdMetadataEnv: "${etcd_metadata_env}" +EtcdMetadataDeps: "${etcd_metadata_deps}" diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index b11957f84b..4de27a9ab4 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -255,3 +255,18 @@ EOF type = "string" } + +variable "base_domain" { + type = "string" + description = "The cluster base domain" +} + +variable "etcd_metadata_env" { + type = "string" + description = "The cluster base domain" +} + +variable "etcd_metadata_deps" { + type = "string" + description = "The cluster base domain" +} diff --git a/modules/dns/route53/etcd.tf b/modules/dns/route53/etcd.tf deleted file mode 100644 index 3f51356f60..0000000000 --- a/modules/dns/route53/etcd.tf +++ /dev/null @@ -1,8 +0,0 @@ -resource "aws_route53_record" "etcd_a_nodes" { - count = "${var.etcd_count}" - type = "A" - ttl = "60" - zone_id = "${local.zone_id}" - name = "${var.cluster_name}-etcd-${count.index}" - records = ["${var.etcd_ip_addresses[count.index]}"] -} diff --git a/modules/dns/route53/outputs.tf b/modules/dns/route53/outputs.tf index 1d92ac716e..22e418db2f 100644 --- a/modules/dns/route53/outputs.tf +++ b/modules/dns/route53/outputs.tf @@ -1,12 +1,3 @@ -output "etcd_a_nodes" { - value = "${aws_route53_record.etcd_a_nodes.*.fqdn}" -} - -# We have to do this join() & split() 'trick' because the ternary operator can't output lists. -output "etcd_endpoints" { - value = ["${split(",", length(var.external_endpoints) == 0 ? join(",", aws_route53_record.etcd_a_nodes.*.fqdn) : join(",", var.external_endpoints))}"] -} - output "worker_nodes" { value = "${aws_route53_record.worker_nodes.*.fqdn}" } diff --git a/modules/dns/route53/variables.tf b/modules/dns/route53/variables.tf index 30d5bf6b99..838f1dcbab 100644 --- a/modules/dns/route53/variables.tf +++ b/modules/dns/route53/variables.tf @@ -30,11 +30,6 @@ variable "etcd_count" { type = "string" } -variable "etcd_ip_addresses" { - description = "List of string IPs for etcd nodes" - type = "list" -} - variable "master_ip_addresses" { description = "List of string IPs for masters" type = "list" diff --git a/steps/assets/ignition-etcd.tf b/steps/assets/ignition-etcd.tf index 03e3812e14..4f604b7ab2 100644 --- a/steps/assets/ignition-etcd.tf +++ b/steps/assets/ignition-etcd.tf @@ -6,37 +6,5 @@ locals { data "ignition_config" "etcd" { count = "${local.etcd_instance_count}" - systemd = [ - "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}", - "${module.ignition_bootstrap.etcd_dropin_id_list[count.index]}", - ] - - files = ["${compact(list( - module.ignition_bootstrap.profile_env_id, - module.ignition_bootstrap.systemd_default_env_id, - ))}", - "${module.ignition_bootstrap.etcd_crt_id_list}", - ] -} - -data "ignition_systemd_unit" "locksmithd" { - count = "${local.etcd_instance_count}" - - name = "locksmithd.service" - enabled = true - - dropin = [ - { - name = "40-etcd-lock.conf" - - content = <