From 23539b9d8bbca6bab206cfa856757384f8a9d6b3 Mon Sep 17 00:00:00 2001 From: Alex Somesan Date: Wed, 19 Apr 2017 20:44:45 +0200 Subject: [PATCH 1/2] AWS: VPC support subnets with custom CIDRs and AZs per workers / masters --- modules/aws/vpc/existing-vpc.tf | 4 ++-- modules/aws/vpc/variables.tf | 22 +++++++++++++++++++- modules/aws/vpc/vpc-private.tf | 24 ++++++++++++++-------- modules/aws/vpc/vpc-public.tf | 24 ++++++++++++++-------- platforms/aws/main.tf | 36 +++++++++++++++++++++++++++++++-- platforms/aws/variables.tf | 12 ++++++++++- 6 files changed, 100 insertions(+), 22 deletions(-) diff --git a/modules/aws/vpc/existing-vpc.tf b/modules/aws/vpc/existing-vpc.tf index 713ee694eb..1cd150cafd 100644 --- a/modules/aws/vpc/existing-vpc.tf +++ b/modules/aws/vpc/existing-vpc.tf @@ -2,11 +2,11 @@ # whenever an external VPC is specified # data "aws_subnet" "external_worker" { - count = "${var.external_vpc_id == "" ? 0 : var.az_count}" + count = "${var.external_vpc_id == "" ? 0 : length(var.external_worker_subnets)}" id = "${var.external_worker_subnets[count.index]}" } data "aws_subnet" "external_master" { - count = "${var.external_vpc_id == "" ? 0 : var.az_count}" + count = "${var.external_vpc_id == "" ? 0 : length(var.external_master_subnets)}" id = "${var.external_master_subnets[count.index]}" } diff --git a/modules/aws/vpc/variables.tf b/modules/aws/vpc/variables.tf index 054c63c556..27a71cd1d3 100644 --- a/modules/aws/vpc/variables.tf +++ b/modules/aws/vpc/variables.tf @@ -1,4 +1,8 @@ -variable "az_count" { +variable "master_az_count" { + type = "string" +} + +variable "worker_az_count" { type = "string" } @@ -32,3 +36,19 @@ variable "enable_etcd_sg" { description = "If set to true, security groups for etcd nodes are being created" default = true } + +variable "master_subnets" { + type = "list" +} + +variable "worker_subnets" { + type = "list" +} + +variable "master_azs" { + type = "list" +} + +variable "worker_azs" { + type = "list" +} diff --git a/modules/aws/vpc/vpc-private.tf b/modules/aws/vpc/vpc-private.tf index f149926561..7a9b69edfa 100644 --- a/modules/aws/vpc/vpc-private.tf +++ b/modules/aws/vpc/vpc-private.tf @@ -1,5 +1,5 @@ resource "aws_route_table" "private_routes" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}" vpc_id = "${data.aws_vpc.cluster_vpc.id}" tags = "${merge(map( @@ -9,7 +9,7 @@ resource "aws_route_table" "private_routes" { } resource "aws_route" "to_nat_gw" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}" route_table_id = "${aws_route_table.private_routes.*.id[count.index]}" destination_cidr_block = "0.0.0.0/0" nat_gateway_id = "${aws_nat_gateway.nat_gw.*.id[count.index]}" @@ -17,20 +17,28 @@ resource "aws_route" "to_nat_gw" { } resource "aws_subnet" "worker_subnet" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" - cidr_block = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index + var.az_count)}" - vpc_id = "${data.aws_vpc.cluster_vpc.id}" - availability_zone = "${data.aws_availability_zones.azs.names[count.index]}" + count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}" + + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + cidr_block = "${length(var.worker_subnets) > 1 ? + "${element(var.worker_subnets, count.index)}" : + "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index + var.worker_az_count)}" + }" + + availability_zone = "${var.worker_azs[count.index]}" tags = "${merge(map( - "Name", "worker-${data.aws_availability_zones.azs.names[count.index]}", + "Name", "worker-${ "${length(var.worker_azs)}" > 0 ? + "${var.worker_azs[count.index]}" : + "${data.aws_availability_zones.azs.names[count.index]}" }", "KubernetesCluster", "${var.cluster_name}", "kubernetes.io/role/internal-elb", "" ), var.extra_tags)}" } resource "aws_route_table_association" "worker_routing" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}" route_table_id = "${aws_route_table.private_routes.*.id[count.index]}" subnet_id = "${aws_subnet.worker_subnet.*.id[count.index]}" } diff --git a/modules/aws/vpc/vpc-public.tf b/modules/aws/vpc/vpc-public.tf index 46d47c1447..41b43452e3 100644 --- a/modules/aws/vpc/vpc-public.tf +++ b/modules/aws/vpc/vpc-public.tf @@ -27,30 +27,38 @@ resource "aws_route" "igw_route" { } resource "aws_subnet" "master_subnet" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" - cidr_block = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index)}" - vpc_id = "${data.aws_vpc.cluster_vpc.id}" - availability_zone = "${data.aws_availability_zones.azs.names[count.index]}" + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" + + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + cidr_block = "${length(var.master_subnets) > 1 ? + "${element(var.master_subnets, count.index)}" : + "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index)}" + }" + + availability_zone = "${var.master_azs[count.index]}" tags = "${merge(map( - "Name", "master-${data.aws_availability_zones.azs.names[count.index]}", + "Name", "master-${ "${length(var.master_azs)}" > 0 ? + "${var.master_azs[count.index]}" : + "${data.aws_availability_zones.azs.names[count.index]}" }", "KubernetesCluster", "${var.cluster_name}" ), var.extra_tags)}" } resource "aws_route_table_association" "route_net" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" route_table_id = "${aws_route_table.default.id}" subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" } resource "aws_eip" "nat_eip" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" vpc = true } resource "aws_nat_gateway" "nat_gw" { - count = "${var.external_vpc_id == "" ? var.az_count : 0}" + count = "${var.external_vpc_id == "" ? var.master_az_count : 0}" allocation_id = "${aws_eip.nat_eip.*.id[count.index]}" subnet_id = "${aws_subnet.master_subnet.*.id[count.index]}" } diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index d28d00744b..3eea435b3b 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -3,7 +3,6 @@ data "aws_availability_zones" "azs" {} module "vpc" { source = "../../modules/aws/vpc" - az_count = "${var.tectonic_aws_az_count}" cidr_block = "${var.tectonic_aws_vpc_cidr_block}" cluster_name = "${var.tectonic_cluster_name}" @@ -12,13 +11,46 @@ module "vpc" { external_worker_subnets = ["${compact(var.tectonic_aws_external_worker_subnet_ids)}"] extra_tags = "${var.tectonic_aws_extra_tags}" enable_etcd_sg = "${length(compact(var.tectonic_etcd_servers)) == 0 ? 1 : 0}" + + # VPC layout settings. + # + # The following parameters control the layout of the VPC accross availability zones. + # Two modes are available: + # A. Explicitly configure a list of AZs + associated subnet CIDRs + # B. Let the module calculate subnets accross a set number of AZs + # + # To enable mode A, make sure "tectonic_aws_az_count" variable IS NOT SET to any value + # and instead configure a set of AZs + CIDRs for masters and workers using the + # "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" variables. + # + # To enable mode B, make sure that "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" + # ARE NOT SET. Instead, set the desired number of VPC AZs using "tectonic_aws_az_count" variable. + + # These counts could be deducted by length(keys(var.tectonic_aws_master_subnets)) + # but there is a restriction on passing computed values as counts. This approach works around that. + master_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_master_subnets))}" : var.tectonic_aws_az_count}" + worker_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_worker_subnets))}" : var.tectonic_aws_az_count}" + # The appending of the "padding" element is required as workaround since the function + # element() won't work on empty lists. See https://github.com/hashicorp/terraform/issues/11210 + master_subnets = "${concat(values(var.tectonic_aws_master_subnets),list("padding"))}" + worker_subnets = "${concat(values(var.tectonic_aws_worker_subnets),list("padding"))}" + # The split() / join() trick works around the limitation of tenrary operator expressions + # only being able to return strings. + master_azs = ["${ split("|", "${length(keys(var.tectonic_aws_master_subnets))}" > 0 ? + join("|", keys(var.tectonic_aws_master_subnets)) : + join("|", data.aws_availability_zones.azs.names) + )}"] + worker_azs = ["${ split("|", "${length(keys(var.tectonic_aws_worker_subnets))}" > 0 ? + join("|", keys(var.tectonic_aws_worker_subnets)) : + join("|", data.aws_availability_zones.azs.names) + )}"] } module "etcd" { source = "../../modules/aws/etcd" instance_count = "${var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : var.tectonic_aws_az_count == 5 ? 5 : 3}" - az_count = "${var.tectonic_aws_az_count}" + az_count = "${length(data.aws_availability_zones.azs.names)}" ec2_type = "${var.tectonic_aws_etcd_ec2_type}" sg_ids = ["${module.vpc.etcd_sg_id}"] diff --git a/platforms/aws/variables.tf b/platforms/aws/variables.tf index ad7ce66c65..c3d5f9dbc5 100644 --- a/platforms/aws/variables.tf +++ b/platforms/aws/variables.tf @@ -29,7 +29,7 @@ variable "tectonic_aws_vpc_cidr_block" { variable "tectonic_aws_az_count" { type = "string" - default = "3" + default = "" description = "Number of Availability Zones your EC2 instances will be deployed across. This should be less than or equal to the total number available in the region. Be aware that some regions only have 2." } @@ -127,3 +127,13 @@ variable "tectonic_aws_worker_root_volume_iops" { default = "100" description = "The amount of provisioned IOPS for the root block device of worker nodes." } + +variable "tectonic_aws_master_subnets" { + type = "map" + default = {} +} + +variable "tectonic_aws_worker_subnets" { + type = "map" + default = {} +} From 72163e2690ff850e50c2b07cf17c0242cc1901f1 Mon Sep 17 00:00:00 2001 From: Alex Somesan Date: Thu, 20 Apr 2017 18:50:47 +0200 Subject: [PATCH 2/2] Rename custom subnet vars more appropriately --- platforms/aws/main.tf | 22 +++++++++++----------- platforms/aws/variables.tf | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index 3eea435b3b..399368f9af 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -21,27 +21,27 @@ module "vpc" { # # To enable mode A, make sure "tectonic_aws_az_count" variable IS NOT SET to any value # and instead configure a set of AZs + CIDRs for masters and workers using the - # "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" variables. + # "tectonic_aws_master_custom_subnets" and "tectonic_aws_worker_custom_subnets" variables. # - # To enable mode B, make sure that "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" + # To enable mode B, make sure that "tectonic_aws_master_custom_subnets" and "tectonic_aws_worker_custom_subnets" # ARE NOT SET. Instead, set the desired number of VPC AZs using "tectonic_aws_az_count" variable. - # These counts could be deducted by length(keys(var.tectonic_aws_master_subnets)) + # These counts could be deducted by length(keys(var.tectonic_aws_master_custom_subnets)) # but there is a restriction on passing computed values as counts. This approach works around that. - master_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_master_subnets))}" : var.tectonic_aws_az_count}" - worker_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_worker_subnets))}" : var.tectonic_aws_az_count}" + master_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_master_custom_subnets))}" : var.tectonic_aws_az_count}" + worker_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_worker_custom_subnets))}" : var.tectonic_aws_az_count}" # The appending of the "padding" element is required as workaround since the function # element() won't work on empty lists. See https://github.com/hashicorp/terraform/issues/11210 - master_subnets = "${concat(values(var.tectonic_aws_master_subnets),list("padding"))}" - worker_subnets = "${concat(values(var.tectonic_aws_worker_subnets),list("padding"))}" + master_subnets = "${concat(values(var.tectonic_aws_master_custom_subnets),list("padding"))}" + worker_subnets = "${concat(values(var.tectonic_aws_worker_custom_subnets),list("padding"))}" # The split() / join() trick works around the limitation of tenrary operator expressions # only being able to return strings. - master_azs = ["${ split("|", "${length(keys(var.tectonic_aws_master_subnets))}" > 0 ? - join("|", keys(var.tectonic_aws_master_subnets)) : + master_azs = ["${ split("|", "${length(keys(var.tectonic_aws_master_custom_subnets))}" > 0 ? + join("|", keys(var.tectonic_aws_master_custom_subnets)) : join("|", data.aws_availability_zones.azs.names) )}"] - worker_azs = ["${ split("|", "${length(keys(var.tectonic_aws_worker_subnets))}" > 0 ? - join("|", keys(var.tectonic_aws_worker_subnets)) : + worker_azs = ["${ split("|", "${length(keys(var.tectonic_aws_worker_custom_subnets))}" > 0 ? + join("|", keys(var.tectonic_aws_worker_custom_subnets)) : join("|", data.aws_availability_zones.azs.names) )}"] } diff --git a/platforms/aws/variables.tf b/platforms/aws/variables.tf index c3d5f9dbc5..9f9047d144 100644 --- a/platforms/aws/variables.tf +++ b/platforms/aws/variables.tf @@ -128,12 +128,12 @@ variable "tectonic_aws_worker_root_volume_iops" { description = "The amount of provisioned IOPS for the root block device of worker nodes." } -variable "tectonic_aws_master_subnets" { +variable "tectonic_aws_master_custom_subnets" { type = "map" default = {} } -variable "tectonic_aws_worker_subnets" { +variable "tectonic_aws_worker_custom_subnets" { type = "map" default = {} }