From 8343ccfecf1682a607b83f0a0fd66be3d7649f35 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Tue, 23 Oct 2018 16:09:24 -0700 Subject: [PATCH 1/3] data/data: add SRV records for etcd members Add SRV records for bootstrapping etcd using DNS. xref: https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/clustering.md#dns-discovery --- data/data/aws/main.tf | 8 ++++++++ data/data/libvirt/main.tf | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf index 2dccc59bac7..1fa579d0fa5 100644 --- a/data/data/aws/main.tf +++ b/data/data/aws/main.tf @@ -117,6 +117,14 @@ resource "aws_route53_record" "etcd_a_nodes" { records = ["${module.masters.ip_addresses[count.index]}"] } +resource "aws_route53_record" "etcd_cluster" { + type = "SRV" + ttl = "60" + zone_id = "${local.private_zone_id}" + name = "_etcd-server-ssl._tcp" + records = ["${formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn)}"] +} + resource "aws_route53_zone" "tectonic_int" { count = "${local.private_endpoints ? "${var.tectonic_aws_external_private_zone == "" ? 1 : 0 }" : 0}" vpc_id = "${module.vpc.vpc_id}" diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf index df475cf29cc..a3493e986e3 100644 --- a/data/data/libvirt/main.tf +++ b/data/data/libvirt/main.tf @@ -50,6 +50,10 @@ resource "libvirt_network" "tectonic_net" { dns = [{ local_only = true + srvs = ["${flatten(list( + data.libvirt_network_dns_srv_template.etcd_cluster.*.rendered, + ))}"] + hosts = ["${flatten(list( data.libvirt_network_dns_host_template.bootstrap.*.rendered, data.libvirt_network_dns_host_template.masters.*.rendered, @@ -116,3 +120,13 @@ data "libvirt_network_dns_host_template" "workers" { ip = "${var.tectonic_libvirt_worker_ips[count.index]}" hostname = "${var.tectonic_cluster_name}" } + +data "libvirt_network_dns_srv_template" "etcd_cluster" { + count = "${var.tectonic_master_count}" + service = "etcd-server-ssl" + protocol = "tcp" + domain = "${var.tectonic_base_domain}" + port = 2380 + weight = 10 + target = "${var.tectonic_cluster_name}-etcd-${count.index}.${var.tectonic_base_domain}" +} From 4fbf790d858a03d25c909e3b68aef3b848b6f4e8 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Fri, 26 Oct 2018 10:16:11 -0700 Subject: [PATCH 2/3] data/data: convert master ign list to single ign --- data/data/aws/main.tf | 2 +- data/data/aws/master/main.tf | 2 +- data/data/aws/master/variables.tf | 4 +- data/data/config.tf | 6 +-- data/data/libvirt/main.tf | 5 +- data/data/openstack/main.tf | 2 +- data/data/openstack/masters/main.tf | 2 +- data/data/openstack/masters/variables.tf | 4 +- pkg/asset/cluster/tfvars.go | 9 +--- pkg/asset/ignition/machine/master.go | 61 +++++++++++------------- pkg/tfvars/tfvars.go | 10 ++-- 11 files changed, 49 insertions(+), 58 deletions(-) diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf index 1fa579d0fa5..c018f9858e4 100644 --- a/data/data/aws/main.tf +++ b/data/data/aws/main.tf @@ -55,7 +55,7 @@ module "masters" { root_volume_type = "${var.tectonic_aws_master_root_volume_type}" subnet_ids = "${module.vpc.master_subnet_ids}" ec2_ami = "${var.tectonic_aws_ec2_ami_override}" - user_data_igns = ["${var.ignition_masters}"] + user_data_ign = "${var.ignition_master}" } module "iam" { diff --git a/data/data/aws/master/main.tf b/data/data/aws/master/main.tf index 59f18da3f33..0ba8110b07c 100644 --- a/data/data/aws/master/main.tf +++ b/data/data/aws/master/main.tf @@ -81,7 +81,7 @@ resource "aws_instance" "master" { iam_instance_profile = "${aws_iam_instance_profile.master.name}" instance_type = "${var.ec2_type}" subnet_id = "${element(var.subnet_ids, count.index)}" - user_data = "${var.user_data_igns[count.index]}" + user_data = "${var.user_data_ign}" vpc_security_group_ids = ["${var.master_sg_ids}"] associate_public_ip_address = "${var.public_endpoints}" diff --git a/data/data/aws/master/variables.tf b/data/data/aws/master/variables.tf index 958192da9f7..d2a17aacbbd 100644 --- a/data/data/aws/master/variables.tf +++ b/data/data/aws/master/variables.tf @@ -93,6 +93,6 @@ variable "kubeconfig_content" { default = "" } -variable "user_data_igns" { - type = "list" +variable "user_data_ign" { + type = "string" } diff --git a/data/data/config.tf b/data/data/config.tf index 42678979f60..83986fde951 100644 --- a/data/data/config.tf +++ b/data/data/config.tf @@ -47,9 +47,9 @@ Note: This field MUST be set manually prior to creating the cluster. EOF } -variable "ignition_masters" { - type = "list" - default = [] +variable "ignition_master" { + type = "string" + default = "" description = < Date: Fri, 26 Oct 2018 10:26:02 -0700 Subject: [PATCH 3/3] asset: update master ign file list to single master ign file --- pkg/asset/ignition/machine/master.go | 2 +- pkg/asset/ignition/machine/master_test.go | 4 +--- pkg/asset/ignition/machine/node.go | 9 ++++----- pkg/asset/ignition/machine/worker.go | 2 +- pkg/asset/machines/aws/master.go | 2 +- pkg/asset/machines/libvirt/master.go | 2 +- pkg/asset/machines/master.go | 14 +++++-------- pkg/asset/machines/openstack/master.go | 2 +- pkg/asset/manifests/tectonic.go | 24 +++++++++++------------ 9 files changed, 27 insertions(+), 34 deletions(-) diff --git a/pkg/asset/ignition/machine/master.go b/pkg/asset/ignition/machine/master.go index a4d876bbed5..a1b1d2b6e30 100644 --- a/pkg/asset/ignition/machine/master.go +++ b/pkg/asset/ignition/machine/master.go @@ -38,7 +38,7 @@ func (a *Master) Generate(dependencies asset.Parents) error { rootCA := &tls.RootCA{} dependencies.Get(installConfig, rootCA) - a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master", "") + a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "master") data, err := json.Marshal(a.Config) if err != nil { diff --git a/pkg/asset/ignition/machine/master_test.go b/pkg/asset/ignition/machine/master_test.go index 950ae921a97..21d9c82fdd8 100644 --- a/pkg/asset/ignition/machine/master_test.go +++ b/pkg/asset/ignition/machine/master_test.go @@ -55,9 +55,7 @@ func TestMasterGenerate(t *testing.T) { err = master.Generate(parents) assert.NoError(t, err, "unexpected error generating master asset") expectedIgnitionConfigNames := []string{ - "master-0.ign", - "master-1.ign", - "master-2.ign", + "master.ign", } actualFiles := master.Files() actualIgnitionConfigNames := make([]string, len(actualFiles)) diff --git a/pkg/asset/ignition/machine/node.go b/pkg/asset/ignition/machine/node.go index 9cdefca7978..0bebbcd5480 100644 --- a/pkg/asset/ignition/machine/node.go +++ b/pkg/asset/ignition/machine/node.go @@ -12,7 +12,7 @@ import ( // pointerIgnitionConfig generates a config which references the remote config // served by the machine config server. -func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string, query string) *ignition.Config { +func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *ignition.Config { return &ignition.Config{ Ignition: ignition.Ignition{ Version: ignition.MaxVersion.String(), @@ -20,10 +20,9 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro Append: []ignition.ConfigReference{{ Source: func() *url.URL { return &url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s-api.%s:49500", installConfig.ObjectMeta.Name, installConfig.BaseDomain), - Path: fmt.Sprintf("/config/%s", role), - RawQuery: query, + Scheme: "https", + Host: fmt.Sprintf("%s-api.%s:49500", installConfig.ObjectMeta.Name, installConfig.BaseDomain), + Path: fmt.Sprintf("/config/%s", role), } }().String(), }}, diff --git a/pkg/asset/ignition/machine/worker.go b/pkg/asset/ignition/machine/worker.go index 32a5ee07a9f..ad0a7fe7e14 100644 --- a/pkg/asset/ignition/machine/worker.go +++ b/pkg/asset/ignition/machine/worker.go @@ -38,7 +38,7 @@ func (a *Worker) Generate(dependencies asset.Parents) error { rootCA := &tls.RootCA{} dependencies.Get(installConfig, rootCA) - a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "worker", "") + a.Config = pointerIgnitionConfig(installConfig.Config, rootCA.Cert(), "worker") data, err := json.Marshal(a.Config) if err != nil { diff --git a/pkg/asset/machines/aws/master.go b/pkg/asset/machines/aws/master.go index ee40ac74f4b..89dbf3d99c1 100644 --- a/pkg/asset/machines/aws/master.go +++ b/pkg/asset/machines/aws/master.go @@ -65,7 +65,7 @@ items: values: - "{{$c.ClusterName}}_master_sg" userDataSecret: - name: "master-user-data-{{$index}}" + name: "master-user-data" versions: kubelet: "" controlPlane: "" diff --git a/pkg/asset/machines/libvirt/master.go b/pkg/asset/machines/libvirt/master.go index f63fbe66abb..2c1132ac048 100644 --- a/pkg/asset/machines/libvirt/master.go +++ b/pkg/asset/machines/libvirt/master.go @@ -40,7 +40,7 @@ items: kind: LibvirtMachineProviderConfig domainMemory: 2048 domainVcpu: 2 - ignKey: /var/lib/libvirt/images/master-{{$index}}.ign + ignKey: /var/lib/libvirt/images/master.ign volume: poolName: default baseVolumeID: /var/lib/libvirt/images/coreos_base diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go index 54e18a0806e..ae39be35a46 100644 --- a/pkg/asset/machines/master.go +++ b/pkg/asset/machines/master.go @@ -19,8 +19,8 @@ import ( // Master generates the machines for the `master` machine pool. type Master struct { - MachinesRaw []byte - UserDataSecretsRaw []byte + MachinesRaw []byte + UserDataSecretRaw []byte } var _ asset.Asset = (*Master)(nil) @@ -45,15 +45,11 @@ func (m *Master) Generate(dependencies asset.Parents) error { mign := &machine.Master{} dependencies.Get(installconfig, mign) - userDataContent := map[string][]byte{} - for i, file := range mign.FileList { - userDataContent[fmt.Sprintf("master-user-data-%d", i)] = file.Data - } - var err error - m.UserDataSecretsRaw, err = userDataList(userDataContent) + userDataMap := map[string][]byte{"master-user-data": mign.File.Data} + m.UserDataSecretRaw, err = userDataList(userDataMap) if err != nil { - return errors.Wrap(err, "failed to create user-data secrets for master machines") + return errors.Wrap(err, "failed to create user-data secret for worker machines") } ic := installconfig.Config diff --git a/pkg/asset/machines/openstack/master.go b/pkg/asset/machines/openstack/master.go index b365d57c2b6..ae8a5528e79 100644 --- a/pkg/asset/machines/openstack/master.go +++ b/pkg/asset/machines/openstack/master.go @@ -62,7 +62,7 @@ items: values: - "{{$c.ClusterName}}_master_sg" userDataSecret: - name: master-user-data-{{$index}} + name: master-user-data versions: kubelet: "" controlPlane: "" diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index f892b7d0b70..b33803d74e8 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -64,18 +64,18 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { } assetData := map[string][]byte{ - "99_binding-discovery.yaml": []byte(content.BindingDiscovery), - "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), - "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), - "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, - "99_openshift-cluster-api_master-machines.yaml": master.MachinesRaw, - "99_openshift-cluster-api_master-user-data-secrets.yaml": master.UserDataSecretsRaw, - "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, - "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, - "99_role-admin.yaml": []byte(content.RoleAdmin), - "99_role-user.yaml": []byte(content.RoleUser), - "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), - "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), + "99_binding-discovery.yaml": []byte(content.BindingDiscovery), + "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), + "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), + "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, + "99_openshift-cluster-api_master-machines.yaml": master.MachinesRaw, + "99_openshift-cluster-api_master-user-data-secret.yaml": master.UserDataSecretRaw, + "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, + "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, + "99_role-admin.yaml": []byte(content.RoleAdmin), + "99_role-user.yaml": []byte(content.RoleUser), + "99_tectonic-system-00-binding-admin.yaml": []byte(content.BindingAdmin), + "99_tectonic-system-02-pull.json": applyTemplateData(content.PullTectonicSystem, templateData), } // addon goes to openshift system