AWS: VPC subnets with custom CIDRs and AZs per workers / masters

modules/aws/vpc/existing-vpc.tf

@@ -2,11 +2,11 @@
 # whenever an external VPC is specified
 #
 data "aws_subnet" "external_worker" {
-  count = "${var.external_vpc_id == "" ? 0 : var.az_count}"
+  count = "${var.external_vpc_id == "" ? 0 : length(var.external_worker_subnets)}"
   id    = "${var.external_worker_subnets[count.index]}"
 }
 
 data "aws_subnet" "external_master" {
-  count = "${var.external_vpc_id == "" ? 0 : var.az_count}"
+  count = "${var.external_vpc_id == "" ? 0 : length(var.external_master_subnets)}"
   id    = "${var.external_master_subnets[count.index]}"
 }

modules/aws/vpc/variables.tf

@@ -1,4 +1,8 @@
-variable "az_count" {
+variable "master_az_count" {
+  type = "string"
+}
+
+variable "worker_az_count" {
   type = "string"
 }
 
@@ -32,3 +36,19 @@ variable "enable_etcd_sg" {
   description = "If set to true, security groups for etcd nodes are being created"
   default     = true
 }
+
+variable "master_subnets" {
+  type = "list"
+}
+
+variable "worker_subnets" {
+  type = "list"
+}
+
+variable "master_azs" {
+  type = "list"
+}
+
+variable "worker_azs" {
+  type = "list"
+}

modules/aws/vpc/vpc-private.tf

@@ -1,5 +1,5 @@
 resource "aws_route_table" "private_routes" {
-  count  = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count  = "${var.external_vpc_id == "" ? var.worker_az_count : 0}"
   vpc_id = "${data.aws_vpc.cluster_vpc.id}"
 
   tags = "${merge(map(
@@ -9,28 +9,36 @@ resource "aws_route_table" "private_routes" {
 }
 
 resource "aws_route" "to_nat_gw" {
-  count                  = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count                  = "${var.external_vpc_id == "" ? var.worker_az_count : 0}"
   route_table_id         = "${aws_route_table.private_routes.*.id[count.index]}"
   destination_cidr_block = "0.0.0.0/0"
   nat_gateway_id         = "${aws_nat_gateway.nat_gw.*.id[count.index]}"
   depends_on             = ["aws_route_table.private_routes"]
 }
 
 resource "aws_subnet" "worker_subnet" {
-  count             = "${var.external_vpc_id == "" ? var.az_count : 0}"
-  cidr_block        = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index + var.az_count)}"
-  vpc_id            = "${data.aws_vpc.cluster_vpc.id}"
-  availability_zone = "${data.aws_availability_zones.azs.names[count.index]}"
+  count = "${var.external_vpc_id == "" ? var.worker_az_count : 0}"
+
+  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
+
+  cidr_block = "${length(var.worker_subnets) > 1 ?
+    "${element(var.worker_subnets, count.index)}" :
+    "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index + var.worker_az_count)}"
+  }"
+
+  availability_zone = "${var.worker_azs[count.index]}"
 
   tags = "${merge(map(
-      "Name", "worker-${data.aws_availability_zones.azs.names[count.index]}",
+      "Name", "worker-${ "${length(var.worker_azs)}" > 0 ? 
+    "${var.worker_azs[count.index]}" : 
+    "${data.aws_availability_zones.azs.names[count.index]}" }",
       "KubernetesCluster", "${var.cluster_name}",
       "kubernetes.io/role/internal-elb", ""
     ), var.extra_tags)}"
 }
 
 resource "aws_route_table_association" "worker_routing" {
-  count          = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count          = "${var.external_vpc_id == "" ? var.worker_az_count : 0}"
   route_table_id = "${aws_route_table.private_routes.*.id[count.index]}"
   subnet_id      = "${aws_subnet.worker_subnet.*.id[count.index]}"
 }

modules/aws/vpc/vpc-public.tf

@@ -27,30 +27,38 @@ resource "aws_route" "igw_route" {
 }
 
 resource "aws_subnet" "master_subnet" {
-  count             = "${var.external_vpc_id == "" ? var.az_count : 0}"
-  cidr_block        = "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index)}"
-  vpc_id            = "${data.aws_vpc.cluster_vpc.id}"
-  availability_zone = "${data.aws_availability_zones.azs.names[count.index]}"
+  count = "${var.external_vpc_id == "" ? var.master_az_count : 0}"
+
+  vpc_id = "${data.aws_vpc.cluster_vpc.id}"
+
+  cidr_block = "${length(var.master_subnets) > 1 ?
+    "${element(var.master_subnets, count.index)}" :
+    "${cidrsubnet(data.aws_vpc.cluster_vpc.cidr_block, 4, count.index)}"
+  }"
+
+  availability_zone = "${var.master_azs[count.index]}"
 
   tags = "${merge(map(
-      "Name", "master-${data.aws_availability_zones.azs.names[count.index]}",
+      "Name", "master-${ "${length(var.master_azs)}" > 0 ? 
+     "${var.master_azs[count.index]}" : 
+     "${data.aws_availability_zones.azs.names[count.index]}" }",
       "KubernetesCluster", "${var.cluster_name}"
     ), var.extra_tags)}"
 }
 
 resource "aws_route_table_association" "route_net" {
-  count          = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count          = "${var.external_vpc_id == "" ? var.master_az_count : 0}"
   route_table_id = "${aws_route_table.default.id}"
   subnet_id      = "${aws_subnet.master_subnet.*.id[count.index]}"
 }
 
 resource "aws_eip" "nat_eip" {
-  count = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count = "${var.external_vpc_id == "" ? var.master_az_count : 0}"
   vpc   = true
 }
 
 resource "aws_nat_gateway" "nat_gw" {
-  count         = "${var.external_vpc_id == "" ? var.az_count : 0}"
+  count         = "${var.external_vpc_id == "" ? var.master_az_count : 0}"
   allocation_id = "${aws_eip.nat_eip.*.id[count.index]}"
   subnet_id     = "${aws_subnet.master_subnet.*.id[count.index]}"
 }

platforms/aws/main.tf

@@ -3,7 +3,6 @@ data "aws_availability_zones" "azs" {}
 module "vpc" {
   source = "../../modules/aws/vpc"
 
-  az_count     = "${var.tectonic_aws_az_count}"
   cidr_block   = "${var.tectonic_aws_vpc_cidr_block}"
   cluster_name = "${var.tectonic_cluster_name}"
 
@@ -12,13 +11,46 @@ module "vpc" {
   external_worker_subnets = ["${compact(var.tectonic_aws_external_worker_subnet_ids)}"]
   extra_tags              = "${var.tectonic_aws_extra_tags}"
   enable_etcd_sg          = "${length(compact(var.tectonic_etcd_servers)) == 0 ? 1 : 0}"
+
+  # VPC layout settings.
+  #
+  # The following parameters control the layout of the VPC accross availability zones.
+  # Two modes are available:
+  # A. Explicitly configure a list of AZs + associated subnet CIDRs
+  # B. Let the module calculate subnets accross a set number of AZs
+  #
+  # To enable mode A, make sure "tectonic_aws_az_count" variable IS NOT SET to any value 
+  # and instead configure a set of AZs + CIDRs for masters and workers using the
+  # "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" variables.
+  #
+  # To enable mode B, make sure that "tectonic_aws_master_subnets" and "tectonic_aws_worker_subnets" 
+  # ARE NOT SET. Instead, set the desired number of VPC AZs using "tectonic_aws_az_count" variable.
+
+  # These counts could be deducted by length(keys(var.tectonic_aws_master_subnets)) 
+  # but there is a restriction on passing computed values as counts. This approach works around that.
+  master_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_master_subnets))}" : var.tectonic_aws_az_count}"
+  worker_az_count = "${var.tectonic_aws_az_count == "" ? "${length(keys(var.tectonic_aws_worker_subnets))}" : var.tectonic_aws_az_count}"
+  # The appending of the "padding" element is required as workaround since the function
+  # element() won't work on empty lists. See https://github.com/hashicorp/terraform/issues/11210
+  master_subnets = "${concat(values(var.tectonic_aws_master_subnets),list("padding"))}"
+  worker_subnets = "${concat(values(var.tectonic_aws_worker_subnets),list("padding"))}"
+  # The split() / join() trick works around the limitation of tenrary operator expressions 
+  # only being able to return strings.
+  master_azs = ["${ split("|", "${length(keys(var.tectonic_aws_master_subnets))}" > 0 ?
+    join("|", keys(var.tectonic_aws_master_subnets)) :
+    join("|", data.aws_availability_zones.azs.names)
+  )}"]
+  worker_azs = ["${ split("|", "${length(keys(var.tectonic_aws_worker_subnets))}" > 0 ?
+    join("|", keys(var.tectonic_aws_worker_subnets)) :
+    join("|", data.aws_availability_zones.azs.names)
+  )}"]
 }
 
 module "etcd" {
   source = "../../modules/aws/etcd"
 
   instance_count = "${var.tectonic_etcd_count > 0 ? var.tectonic_etcd_count : var.tectonic_aws_az_count == 5 ? 5 : 3}"
-  az_count       = "${var.tectonic_aws_az_count}"
+  az_count       = "${length(data.aws_availability_zones.azs.names)}"
   ec2_type       = "${var.tectonic_aws_etcd_ec2_type}"
   sg_ids         = ["${module.vpc.etcd_sg_id}"]
 

platforms/aws/variables.tf

@@ -29,7 +29,7 @@ variable "tectonic_aws_vpc_cidr_block" {
 
 variable "tectonic_aws_az_count" {
   type        = "string"
-  default     = "3"
+  default     = ""
   description = "Number of Availability Zones your EC2 instances will be deployed across. This should be less than or equal to the total number available in the region. Be aware that some regions only have 2."
 }
 
@@ -127,3 +127,13 @@ variable "tectonic_aws_worker_root_volume_iops" {
   default     = "100"
   description = "The amount of provisioned IOPS for the root block device of worker nodes."
 }
+
+variable "tectonic_aws_master_subnets" {
+  type    = "map"
+  default = {}
+}
+
+variable "tectonic_aws_worker_subnets" {
+  type    = "map"
+  default = {}
+}