Combine NLBs to use one NLB per cluster

* Simplify clusters to come with a single NLB * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver) * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller) * Reduce cost of default clusters by 1 NLB ($18.14/month) * Keep using CNAME records to the `ingress_dns_name` NLB and the nginx-ingress addon for Ingress (up to a few million RPS) * Users with heavy traffic (many million RPS) can create their own separate NLB(s) for Ingress and use the new output worker target groups * Fix issue where additional worker pools come with an extraneous network load balancer
poseidon · Jun 22, 2018 · 316f06d · 316f06d
1 parent f4d3059
commit 316f06d
Show file tree

Hide file tree

Showing 9 changed files with 113 additions and 99 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -10,7 +10,13 @@ Notable changes between versions.
 #### AWS
 
 * Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
-  * Update NLB, security groups, and generated kubeconfig's
+* Combine apiserver and ingress NLBs ([#249](https://github.com/poseidon/typhoon/pull/249))
+  * Simplify clusters to come with one NLB. Reduce cost by ~$18/month per cluster.
+  * Users may keep using CNAME records to `ingress_dns_name` and the `nginx-ingress` addon for Ingress (up to a few million RPS)
+  * Users with heavy traffic (many million RPS) should create a separate NLB(s) for Ingress instead
+  * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver)
+  * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller)
+* Worker pools (advanced) no longer include an extraneous load balancer
 
 #### Bare-Metal
 

diff --git a/aws/container-linux/kubernetes/apiserver.tf → aws/container-linux/kubernetes/nlb.tf b/aws/container-linux/kubernetes/apiserver.tf → aws/container-linux/kubernetes/nlb.tf
@@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_lb.apiserver.dns_name}"
-    zone_id                = "${aws_lb.apiserver.zone_id}"
+    name                   = "${aws_lb.nlb.dns_name}"
+    zone_id                = "${aws_lb.nlb.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Network Load Balancer for apiservers
-resource "aws_lb" "apiserver" {
-  name               = "${var.cluster_name}-apiserver"
+# Network Load Balancer for apiservers and ingress
+resource "aws_lb" "nlb" {
+  name               = "${var.cluster_name}-nlb"
   load_balancer_type = "network"
   internal           = false
 
@@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" {
   enable_cross_zone_load_balancing = true
 }
 
-# Forward TCP traffic to controllers
+# Forward TCP apiserver traffic to controllers
 resource "aws_lb_listener" "apiserver-https" {
-  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  load_balancer_arn = "${aws_lb.nlb.arn}"
   protocol          = "TCP"
   port              = "6443"
 
@@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
   }
 }
 
+# Forward HTTP ingress traffic to workers
+resource "aws_lb_listener" "ingress-http" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 80
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_http_arn}"
+  }
+}
+
+# Forward HTTPS ingress traffic to workers
+resource "aws_lb_listener" "ingress-https" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 443
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_https_arn}"
+  }
+}
+
 # Target group of controllers
 resource "aws_lb_target_group" "controllers" {
   name        = "${var.cluster_name}-controllers"

diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf
@@ -1,8 +1,18 @@
 output "ingress_dns_name" {
-  value       = "${module.workers.ingress_dns_name}"
+  value       = "${aws_lb.nlb.dns_name}"
   description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
 }
 
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${module.workers.target_group_http_arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${module.workers.target_group_https_arn}"
+}
+
 # Outputs for worker pools
 
 output "vpc_id" {

diff --git a/aws/container-linux/kubernetes/workers/ingress.tf b/aws/container-linux/kubernetes/workers/ingress.tf
@@ -1,39 +1,4 @@
-# Network Load Balancer for Ingress
-resource "aws_lb" "ingress" {
-  name               = "${var.name}-ingress"
-  load_balancer_type = "network"
-  internal           = false
-
-  subnets = ["${var.subnet_ids}"]
-
-  enable_cross_zone_load_balancing = true
-}
-
-# Forward HTTP traffic to workers
-resource "aws_lb_listener" "ingress-http" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 80
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-http.arn}"
-  }
-}
-
-# Forward HTTPS traffic to workers
-resource "aws_lb_listener" "ingress-https" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 443
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-https.arn}"
-  }
-}
-
-# Network Load Balancer target groups of instances
+# Target groups of instances for use with load balancers
 
 resource "aws_lb_target_group" "workers-http" {
   name        = "${var.name}-workers-http"

diff --git a/aws/container-linux/kubernetes/workers/outputs.tf b/aws/container-linux/kubernetes/workers/outputs.tf
@@ -1,4 +1,9 @@
-output "ingress_dns_name" {
-  value       = "${aws_lb.ingress.dns_name}"
-  description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${aws_lb_target_group.workers-http.arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${aws_lb_target_group.workers-https.arn}"
 }
diff --git a/aws/fedora-atomic/kubernetes/apiserver.tf → aws/fedora-atomic/kubernetes/nlb.tf b/aws/fedora-atomic/kubernetes/apiserver.tf → aws/fedora-atomic/kubernetes/nlb.tf
@@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_lb.apiserver.dns_name}"
-    zone_id                = "${aws_lb.apiserver.zone_id}"
+    name                   = "${aws_lb.nlb.dns_name}"
+    zone_id                = "${aws_lb.nlb.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Network Load Balancer for apiservers
-resource "aws_lb" "apiserver" {
-  name               = "${var.cluster_name}-apiserver"
+# Network Load Balancer for apiservers and ingress
+resource "aws_lb" "nlb" {
+  name               = "${var.cluster_name}-nlb"
   load_balancer_type = "network"
   internal           = false
 
@@ -24,31 +24,55 @@ resource "aws_lb" "apiserver" {
   enable_cross_zone_load_balancing = true
 }
 
-# Forward TCP traffic to controllers
+# Forward TCP apiserver traffic to controllers
 resource "aws_lb_listener" "apiserver-https" {
-  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  load_balancer_arn = "${aws_lb.nlb.arn}"
   protocol          = "TCP"
-  port              = "443"
+  port              = "6443"
 
   default_action {
     type             = "forward"
     target_group_arn = "${aws_lb_target_group.controllers.arn}"
   }
 }
 
+# Forward HTTP ingress traffic to workers
+resource "aws_lb_listener" "ingress-http" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 80
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_http_arn}"
+  }
+}
+
+# Forward HTTPS ingress traffic to workers
+resource "aws_lb_listener" "ingress-https" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 443
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_https_arn}"
+  }
+}
+
 # Target group of controllers
 resource "aws_lb_target_group" "controllers" {
   name        = "${var.cluster_name}-controllers"
   vpc_id      = "${aws_vpc.network.id}"
   target_type = "instance"
 
   protocol = "TCP"
-  port     = 443
+  port     = 6443
 
   # TCP health check for apiserver
   health_check {
     protocol = "TCP"
-    port     = 443
+    port     = 6443
 
     # NLBs required to use same healthy and unhealthy thresholds
     healthy_threshold   = 3
@@ -65,5 +89,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
 
   target_group_arn = "${aws_lb_target_group.controllers.arn}"
   target_id        = "${element(aws_instance.controllers.*.id, count.index)}"
-  port             = 443
+  port             = 6443
 }
diff --git a/aws/fedora-atomic/kubernetes/outputs.tf b/aws/fedora-atomic/kubernetes/outputs.tf
@@ -1,8 +1,18 @@
 output "ingress_dns_name" {
-  value       = "${module.workers.ingress_dns_name}"
+  value       = "${aws_lb.nlb.dns_name}"
   description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
 }
 
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${module.workers.target_group_http_arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${module.workers.target_group_https_arn}"
+}
+
 # Outputs for worker pools
 
 output "vpc_id" {

diff --git a/aws/fedora-atomic/kubernetes/workers/ingress.tf b/aws/fedora-atomic/kubernetes/workers/ingress.tf
@@ -1,39 +1,4 @@
-# Network Load Balancer for Ingress
-resource "aws_lb" "ingress" {
-  name               = "${var.name}-ingress"
-  load_balancer_type = "network"
-  internal           = false
-
-  subnets = ["${var.subnet_ids}"]
-
-  enable_cross_zone_load_balancing = true
-}
-
-# Forward HTTP traffic to workers
-resource "aws_lb_listener" "ingress-http" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 80
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-http.arn}"
-  }
-}
-
-# Forward HTTPS traffic to workers
-resource "aws_lb_listener" "ingress-https" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 443
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-https.arn}"
-  }
-}
-
-# Network Load Balancer target groups of instances
+# Target groups of instances for use with load balancers
 
 resource "aws_lb_target_group" "workers-http" {
   name        = "${var.name}-workers-http"

diff --git a/aws/fedora-atomic/kubernetes/workers/outputs.tf b/aws/fedora-atomic/kubernetes/workers/outputs.tf
@@ -1,4 +1,9 @@
-output "ingress_dns_name" {
-  value       = "${aws_lb.ingress.dns_name}"
-  description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${aws_lb_target_group.workers-http.arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${aws_lb_target_group.workers-https.arn}"
 }