Skip to content

*: clean up AWS ELBs #242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 3 commits into from
Sep 21, 2018
Merged

*: clean up AWS ELBs #242

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
239373f
aws/ELBs: merge tnc with api_internal and cleanup
steveej Sep 19, 2018
a6231bc
*: subst tnc/TNC with mcs/MCS
steveej Sep 20, 2018
08708cc
*: remove stale mcso config references
steveej Sep 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions installer/pkg/config-generator/ignition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,19 +125,13 @@ func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {

func (c *ConfigGenerator) getTNCURL(role string, query string) string {

@abhinavdahiya abhinavdahiya Sep 15, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No longer TNC getMCSURL ?

@wking wking Sep 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm in favor of writing out anything project-specific. Folks likely have seen URL before, which would make this getMachineConfigServerURL.

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep the rename out of scope here see #242 (comment).

(I know I'm repeating myself here :-D)

var u string

// cloud platforms put this behind a load balancer which remaps ports;
// libvirt doesn't do that - use the tnc port directly
port := 80
if c.Platform == config.PlatformLibvirt {
port = 49500
}
port := 49500

if role == "master" || role == "worker" {
u = func() *url.URL {
return &url.URL{
Scheme: "https",
Host: fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
Host: fmt.Sprintf("%s-api.%s:%d", c.Name, c.BaseDomain, port),
Path: fmt.Sprintf("/config/%s", role),
RawQuery: query,
}
Expand Down
2 changes: 1 addition & 1 deletion installer/pkg/config-generator/tls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ func (c *ConfigGenerator) GenerateTLSConfig(clusterDir string) error {
}

// MachineConfigServer certs
mcsDomain := fmt.Sprintf("%s-tnc.%s", c.Name, c.BaseDomain)
mcsDomain := fmt.Sprintf("%s-api.%s", c.Name, c.BaseDomain)
cfg = &tls.CertCfg{
ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
DNSNames: []string{mcsDomain},
Expand Down
6 changes: 0 additions & 6 deletions modules/aws/master/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,6 @@ resource "aws_instance" "master" {
), var.extra_tags)}"
}

resource "aws_elb_attachment" "masters_tnc" {
count = "${var.private_endpoints ? var.instance_count : 0}"
elb = "${var.elb_tnc_id}"
instance = "${aws_instance.master.*.id[count.index]}"
}

resource "aws_elb_attachment" "masters_internal" {
count = "${var.private_endpoints ? var.instance_count : 0}"
elb = "${var.elb_api_internal_id}"
Expand Down
4 changes: 0 additions & 4 deletions modules/aws/master/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,6 @@ variable "public_endpoints" {
default = true
}

variable "elb_tnc_id" {
type = "string"
}

variable "elb_api_internal_id" {
type = "string"
}
Expand Down
49 changes: 16 additions & 33 deletions modules/aws/vpc/master-elb.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,3 @@
resource "aws_elb" "tnc" {
count = "${var.private_master_endpoints ? 1 : 0}"
name = "${var.cluster_name}-tnc"
subnets = ["${local.master_subnet_ids}"]
internal = true
security_groups = ["${aws_security_group.tnc.id}"]

idle_timeout = 3600
connection_draining = true
connection_draining_timeout = 300

listener {
instance_port = 49500
instance_protocol = "tcp"
lb_port = 80
lb_protocol = "tcp"
}

health_check {
healthy_threshold = 2
unhealthy_threshold = 2
timeout = 3
target = "TCP:49500"
interval = 5
}

tags = "${merge(map(
"Name", "${var.cluster_name}-int",
"kubernetes.io/cluster/${var.cluster_name}", "owned",
"tectonicClusterID", "${var.cluster_id}"
), var.extra_tags)}"
}

resource "aws_elb" "api_internal" {
count = "${var.private_master_endpoints ? 1 : 0}"
name = "${var.cluster_name}-int"
Expand All @@ -49,6 +16,13 @@ resource "aws_elb" "api_internal" {
lb_protocol = "tcp"
}

listener {
instance_port = 49500
instance_protocol = "tcp"
lb_port = 49500
lb_protocol = "tcp"
}

health_check {
healthy_threshold = 2
unhealthy_threshold = 2
Expand All @@ -57,6 +31,15 @@ resource "aws_elb" "api_internal" {
interval = 5
}

# TODO: we only have on health_check per ELB but need to check the following too
# health_check {

@abhinavdahiya abhinavdahiya Sep 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are these disabled?

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

An ELB can only have one health_check. We'll need to come up with a different solution for these.

@wking wking Sep 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

An ELB can only have one health_check.

So drop the commented-out code and file an issue? Do we need to continue to health-check 49500 independently?

@abhinavdahiya abhinavdahiya Sep 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add this as comment above.

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we like TODOs in the code? If not I'll opt for removing the code altogether

@crawford crawford Sep 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

49500 doesn't need a health check because Ignition uses a random resolver. If the endpoint is down, Ignition will try another IP address.

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what if all are down?

@crawford crawford Sep 21, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If they are all down, Ignition keeps retrying.

# healthy_threshold = 2
# unhealthy_threshold = 2
# timeout = 3
# target = "TCP:49500"
# interval = 5
# }

tags = "${merge(map(
"Name", "${var.cluster_name}-int",
"kubernetes.io/cluster/${var.cluster_name}", "owned",
Expand Down
18 changes: 3 additions & 15 deletions modules/aws/vpc/outputs.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,23 +42,19 @@ output "aws_elb_console_id" {
value = "${aws_elb.console.id}"
}

output "aws_elb_tnc_id" {
value = "${aws_elb.tnc.0.id}"
}

output "aws_lbs" {
value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id, aws_elb.tnc.*.id))}"]
value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id))}"]
}

output "aws_api_external_dns_name" {
output "aws_elb_api_external_dns_name" {

@wking wking Sep 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I don't know if we want to handle this incrementally or not, but aws_ is redundant for output variables in this AWS-only module. Also, I don't know if consumers will care if this is an elastic load balancer or not. So personally I'd prefer api_external_dns_name here.

But it's also nice to keep this diff small and the variables consistent, and aws_elb_api_external_... matches the pattern set by aws_elb_api_external_zone_id below.

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it's also nice to keep this diff small and the variables consistent, and aws_elb_api_external_... matches the pattern set by aws_elb_api_external_zone_id below.

That was my intention. Now it's consistent within the current scheme, which can be refactored followiing up to this change.

value = "${element(concat(aws_elb.api_external.*.dns_name, list("")), 0)}"
}

output "aws_elb_api_external_zone_id" {
value = "${element(concat(aws_elb.api_external.*.zone_id, list("")), 0)}"
}

output "aws_api_internal_dns_name" {
output "aws_elb_api_internal_dns_name" {
value = "${element(concat(aws_elb.api_internal.*.dns_name, list("")), 0)}"
}

Expand All @@ -73,11 +69,3 @@ output "aws_console_dns_name" {
output "aws_elb_console_zone_id" {
value = "${aws_elb.console.zone_id}"
}

output "aws_elb_tnc_dns_name" {
value = "${element(concat(aws_elb.tnc.*.dns_name, list("")), 0)}"
}

output "aws_elb_tnc_zone_id" {
value = "${element(concat(aws_elb.tnc.*.zone_id, list("")), 0)}"
}
50 changes: 10 additions & 40 deletions modules/aws/vpc/sg-elb.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,3 @@
resource "aws_security_group" "tnc" {
vpc_id = "${data.aws_vpc.cluster_vpc.id}"

tags = "${merge(map(
"Name", "${var.cluster_name}_tnc_sg",
"kubernetes.io/cluster/${var.cluster_name}", "owned",
"tectonicClusterID", "${var.cluster_id}"
), var.extra_tags)}"
}

resource "aws_security_group_rule" "tnc_egress" {
type = "egress"
security_group_id = "${aws_security_group.tnc.id}"

from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}

resource "aws_security_group_rule" "tnc_ingress_http" {
type = "ingress"
security_group_id = "${aws_security_group.tnc.id}"

protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
from_port = 80
to_port = 80
}

resource "aws_security_group_rule" "tnc_ingress_https" {
type = "ingress"
security_group_id = "${aws_security_group.tnc.id}"

protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
from_port = 443
to_port = 443
}

resource "aws_security_group" "api" {
vpc_id = "${data.aws_vpc.cluster_vpc.id}"

Expand Down Expand Up @@ -68,6 +28,16 @@ resource "aws_security_group_rule" "api_ingress_console" {
to_port = 6443
}

resource "aws_security_group_rule" "tnc_ingress" {

@wking wking Sep 19, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tnc -> mcs rebranding? Or ignition_ingress? Or something.

@steveej steveej Sep 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep the rename out of scope here see #242 (comment).

@crawford crawford Sep 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can put it in a different commit, but it belongs in this PR.

@steveej steveej Sep 20, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it the rename is not related to the logic of this PR which is to consolidate ELBs and not rename stuff, I added the commit anyway so y'all are happy ;-)

type = "ingress"
security_group_id = "${aws_security_group.api.id}"

protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
from_port = 49500
to_port = 49500
}

resource "aws_security_group" "console" {
vpc_id = "${data.aws_vpc.cluster_vpc.id}"

Expand Down
2 changes: 1 addition & 1 deletion pkg/asset/ignition/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
Source: func() *url.URL {
return &url.URL{
Scheme: "https",
Host: fmt.Sprintf("%s-tnc.%s:49500", installConfig.Name, installConfig.BaseDomain),
Host: fmt.Sprintf("%s-api.%s:49500", installConfig.Name, installConfig.BaseDomain),
Path: fmt.Sprintf("/config/%s", role),
RawQuery: query,
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/asset/tls/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,9 @@ func genIPAddressesForOpenshiftAPIServerCertKey(cfg *types.InstallConfig) ([]net
}

func genDNSNamesForTNCCertKey(cfg *types.InstallConfig) ([]string, error) {
return []string{fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil
return []string{fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil
}

func genSubjectForTNCCertKey(cfg *types.InstallConfig) (pkix.Name, error) {
return pkix.Name{CommonName: fmt.Sprintf("%s-tnc.%s", cfg.Name, cfg.BaseDomain)}, nil
return pkix.Name{CommonName: fmt.Sprintf("%s-api.%s", cfg.Name, cfg.BaseDomain)}, nil
}
17 changes: 2 additions & 15 deletions steps/infra/aws/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ provider "aws" {
module "masters" {
source = "../../../modules/aws/master"

elb_tnc_id = "${module.vpc.aws_elb_tnc_id}"
elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}"
elb_api_external_id = "${module.vpc.aws_elb_api_external_id}"
elb_console_id = "${module.vpc.aws_elb_console_id}"
Expand Down Expand Up @@ -51,9 +50,9 @@ module "iam" {
module "dns" {
source = "../../../modules/dns/route53"

api_external_elb_dns_name = "${module.vpc.aws_api_external_dns_name}"
api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}"
api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}"
api_internal_elb_dns_name = "${module.vpc.aws_api_internal_dns_name}"
api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}"
api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}"
api_ip_addresses = "${module.vpc.aws_lbs}"
base_domain = "${var.tectonic_base_domain}"
Expand Down Expand Up @@ -100,18 +99,6 @@ resource "aws_route53_record" "etcd_a_nodes" {
records = ["${module.masters.ip_addresses[count.index]}"]
}

resource "aws_route53_record" "tectonic_tnc_a" {
zone_id = "${local.private_zone_id}"
name = "${var.tectonic_cluster_name}-tnc.${var.tectonic_base_domain}"
type = "A"

alias {
name = "${module.vpc.aws_elb_tnc_dns_name}"
zone_id = "${module.vpc.aws_elb_tnc_zone_id}"
evaluate_target_health = true
}
}

resource "aws_route53_zone" "tectonic_int" {
count = "${local.private_endpoints ? "${var.tectonic_aws_external_private_zone == "" ? 1 : 0 }" : 0}"
vpc_id = "${module.vpc.vpc_id}"
Expand Down
1 change: 0 additions & 1 deletion steps/infra/libvirt/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ resource "libvirt_domain" "master" {
locals {
"hostnames" = [
"${var.tectonic_cluster_name}-api",
"${var.tectonic_cluster_name}-tnc",
]
}

Expand Down