Skip to content
Closed
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
00f216b
Add EKS Fargate support
itssimon May 5, 2020
8cd11c3
Add new inputs for EKS Fargate
itssimon May 6, 2020
c3e5c43
Add new outputs for EKS Fargate
itssimon May 6, 2020
ed47451
Merge branch 'master' into fargate
itssimon May 6, 2020
23dd5bf
Formatting
itssimon May 6, 2020
1843863
Merge branch 'fargate' of github.com:itssimon/terraform-aws-eks into …
itssimon May 6, 2020
026e422
Fix lint issues
itssimon May 6, 2020
064aca0
Remove cluster security group from workers again
itssimon May 6, 2020
799c9f9
Add security group rule
itssimon May 6, 2020
29509af
Merge branch 'master' into fargate
itssimon May 12, 2020
fe188c9
Move Fargate resources to submodule
itssimon May 12, 2020
3ef8991
Merge branch 'fargate' of github.com:itssimon/terraform-aws-eks into …
itssimon May 12, 2020
b770ef3
Requested changes in review
itssimon May 20, 2020
3cba018
Update according to review changes
itssimon May 20, 2020
1cbbaf4
Fix
itssimon May 20, 2020
ad96814
Clean up
itssimon May 20, 2020
6822b43
Use splat syntax for cluster name
itssimon May 20, 2020
0eef01f
Requested changes
itssimon May 23, 2020
40051b9
Fix
itssimon May 23, 2020
c054b7a
Remove security groups here (adding to main module)
itssimon May 26, 2020
9cc9cec
Update modules/fargate/fargate.tf
itssimon May 27, 2020
9553f5c
Update modules/fargate/fargate.tf
itssimon May 27, 2020
e4c0d49
Update modules/fargate/fargate.tf
itssimon May 27, 2020
0a8e0a6
Remove unnecessary variables
itssimon May 27, 2020
0885fdf
Simplify aws_auth_roles output
itssimon May 27, 2020
b6c8522
Fix formatting
itssimon May 27, 2020
7199e01
Add create_fargate_pod_execution_role input
itssimon May 28, 2020
47816b4
Remove race conditions
itssimon Jun 4, 2020
ba4aa63
Merge branch 'master' into fargate
itssimon Jun 4, 2020
85f1319
Undo adding depedency on aws-auth
itssimon Jun 4, 2020
cdbc078
Merge branch 'fargate' of github.com:itssimon/terraform-aws-eks into …
itssimon Jun 4, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | `"1.16"` | no |
| config\_output\_path | Where to save the Kubectl config file (if `write_kubeconfig = true`). Assumed to be a directory if the value ends with a forward slash `/`. | `string` | `"./"` | no |
| create\_eks | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no |
| create\_eks\_fargate | Controls if EKS Fargate resources should be created | `bool` | `false` | no |
| eks\_fargate\_profiles | EKS Fargate profiles | <pre>list(object({<br> namespace = string<br> labels = map(string)<br> }))</pre> | `[]` | no |
| eks\_oidc\_root\_ca\_thumbprint | Thumbprint of Root CA for EKS OIDC, Valid until 2037 | `string` | `"9e99a48a9960b14926bb7f3b02e22da2b0ab7280"` | no |
| enable\_irsa | Whether to create OpenID Connect Provider for EKS to enable IRSA | `bool` | `false` | no |
| iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no |
Expand Down Expand Up @@ -227,6 +229,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
| cluster\_security\_group\_id | Security group ID attached to the EKS cluster. On 1.14 or later, this is the 'Additional security groups' in the EKS console. |
| cluster\_version | The Kubernetes server version for the EKS cluster. |
| config\_map\_aws\_auth | A kubernetes configuration to authenticate to this EKS cluster. |
| fargate\_iam\_role\_arn | IAM role ARN for EKS Fargate pods |
| fargate\_iam\_role\_name | IAM role name for EKS Fargate pods |
| kubeconfig | kubectl config file contents for this EKS cluster. |
| kubeconfig\_filename | The filename of the generated kubectl config. |
| node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys |
Expand Down
13 changes: 11 additions & 2 deletions aws_auth.tf
Original file line number Diff line number Diff line change
Expand Up @@ -38,22 +38,31 @@ locals {
}
]

auth_fargate_roles = [
for index in range(0, var.create_eks && var.create_eks_fargate ? 1 : 0) : {
worker_role_arn = module.fargate.iam_role_arn
platform = "fargate"
}
]

# Convert to format needed by aws-auth ConfigMap
configmap_roles = [
for role in concat(
local.auth_launch_template_worker_roles,
local.auth_worker_roles,
local.auth_fargate_roles,
module.node_groups.aws_auth_roles,
) :
{
rolearn = role["worker_role_arn"]
username = "system:node:{{EC2PrivateDNSName}}"
username = role["platform"] == "fargate" ? "system:node:{{SessionName}}" : "system:node:{{EC2PrivateDNSName}}"
groups = tolist(concat(
[
"system:bootstrappers",
"system:nodes",
],
role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : []
role["platform"] == "windows" ? ["eks:kube-proxy-windows"] : [],
role["platform"] == "fargate" ? ["system:node-proxier"] : [],
))
}
]
Expand Down
10 changes: 10 additions & 0 deletions fargate.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
module "fargate" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This needs a dependency on the aws-auth configmap. Otherwise spinning up fresh clusters with fargate enabled may fail due to the race condition of who creates the configmap first

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you provide some guidance on how to best achieve that?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

node_groups.tf in the root of the module has one example. But honestly I think the rejected work in #867 is a more "terraform native" way of implementing it. Makes the module easier to use stand-alone as well. My mistake as I didn't think depending on vars had been implemented yet when I moved managed node groups to a module.

Copy link
Contributor Author

@itssimon itssimon Jun 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding this dependency creates a cycle. Are you sure the fargate module would cause the aws-auth ConfigMap to be created?

Error: Cycle: data.null_data_source.fargate, module.fargate.var.cluster_name, module.fargate.aws_iam_role.eks_fargate_pod, module.fargate.output.aws_auth_roles, local.configmap_roles, kubernetes_config_map.aws_auth

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, AWS automatically creates the aws-auth configmap when you create a fargate profile on a cluster that does not have the configmap. This is a race condition that caused some pain for early adopters of the managed node groups in this module.

The problem with the cycle is using the cluster_name from the null_resource to create the IAM role. This could be avoided by using a dependency variable instead of the null_resource. Then only the aws_eks_fargate_profile needs to block on the configmap.

source = "./modules/fargate"
create = var.create_eks && var.create_eks_fargate
cluster_name = aws_eks_cluster.this[0].name
profiles = var.eks_fargate_profiles
subnets = var.subnets
tags = var.tags
cluster_primary_security_group_id = element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0)
worker_security_group_id = local.worker_security_group_id
}
67 changes: 67 additions & 0 deletions modules/fargate/fargate.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Allow Fargate pods and EC2 workers to communicate

resource "aws_security_group_rule" "eks_fargate1" {
count = var.create ? 1 : 0
type = "ingress"
from_port = 0
to_port = 65535
protocol = "all"
security_group_id = var.worker_security_group_id
source_security_group_id = var.cluster_primary_security_group_id
}

resource "aws_security_group_rule" "eks_fargate2" {
count = var.create ? 1 : 0
type = "ingress"
from_port = 0
to_port = 65535
protocol = "all"
security_group_id = var.cluster_primary_security_group_id
source_security_group_id = var.worker_security_group_id
}


# EKS Fargate Pod Execution Role

data "aws_iam_policy_document" "eks_fargate_pod_assume_role" {
count = var.create ? 1 : 0
statement {
effect = "Allow"
actions = ["sts:AssumeRole"]

principals {
type = "Service"
identifiers = ["eks-fargate-pods.amazonaws.com"]
}
}
}

resource "aws_iam_role" "eks_fargate_pod" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add in the iam_path variable from the parent too

count = var.create ? 1 : 0
name = format("%s-fargate", var.cluster_name)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The cluster and worker roles are created using name_prefix by default. This allows for the same name cluster to be created in different regions as IAM is global. I have no idea if anybody actually does that. There is also the complexity around name_prefix vs name and var.workers_role_name. Do we want to replicate that here? @barryib

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the only open topic to resolve now. How would you like to handle the naming here @barryib ?

assume_role_policy = join("", data.aws_iam_policy_document.eks_fargate_pod_assume_role.*.json)
tags = merge(var.tags, { "kubernetes.io/cluster/${var.cluster_name}" = "owned" })
}

resource "aws_iam_role_policy_attachment" "eks_fargate_pod" {
count = var.create ? 1 : 0
policy_arn = "arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy"
role = join("", aws_iam_role.eks_fargate_pod.*.name)
}


# EKS Fargate profiles

resource "aws_eks_fargate_profile" "this" {
count = var.create ? local.profile_count : 0
cluster_name = var.cluster_name
fargate_profile_name = format("%s-fargate-%s", var.cluster_name, var.profiles[count.index].namespace)
pod_execution_role_arn = join("", aws_iam_role.eks_fargate_pod.*.arn)
subnet_ids = var.subnets
tags = merge(var.tags, { "kubernetes.io/cluster/${var.cluster_name}" = "owned" })

selector {
namespace = var.profiles[count.index].namespace
labels = var.profiles[count.index].labels
}
}
3 changes: 3 additions & 0 deletions modules/fargate/locals.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
locals {
profile_count = length(var.profiles)
}
9 changes: 9 additions & 0 deletions modules/fargate/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
output "iam_role_name" {
description = "IAM role name for EKS Fargate pods"
value = element(concat(aws_iam_role.eks_fargate_pod.*.name, list("")), 0)
}

output "iam_role_arn" {
description = "IAM role ARN for EKS Fargate pods"
value = element(concat(aws_iam_role.eks_fargate_pod.*.arn, list("")), 0)
}
39 changes: 39 additions & 0 deletions modules/fargate/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
variable "create" {
description = "Controls if EKS Fargate resources should be created."
type = bool
default = false
}

variable "cluster_name" {
description = "Name of parent cluster."
type = string
}

variable "profiles" {
description = "List of EKS Fargate profiles to create."
type = list(object({
namespace = string
labels = map(string)
}))
default = []
}

variable "subnets" {
description = "A list of subnets for the EKS Fargate profiles."
type = list(string)
}

variable "tags" {
description = "A map of tags to add to all resources."
type = map(string)
}

variable "cluster_primary_security_group_id" {
description = "Cluster primary security group ID created by the EKS cluster on 1.14 or later. Referred to as 'Cluster security group' in the EKS console."
type = string
}

variable "worker_security_group_id" {
description = "Security group ID attached to the EKS workers."
type = string
}
10 changes: 10 additions & 0 deletions outputs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,16 @@ output "worker_iam_role_arn" {
)[0]
}

output "fargate_iam_role_name" {
description = "IAM role name for EKS Fargate pods"
value = module.fargate.iam_role_name
}

output "fargate_iam_role_arn" {
description = "IAM role ARN for EKS Fargate pods"
value = module.fargate.iam_role_arn
}

output "node_groups" {
description = "Outputs from EKS node groups. Map of maps, keyed by var.node_groups keys"
value = module.node_groups.node_groups
Expand Down
15 changes: 15 additions & 0 deletions variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -332,3 +332,18 @@ variable "cluster_encryption_config" {
}))
default = []
}

variable "create_eks_fargate" {
description = "Controls if EKS Fargate resources should be created"
type = bool
default = false
}

variable "eks_fargate_profiles" {
description = "EKS Fargate profiles"
type = list(object({
namespace = string
labels = map(string)
}))
default = []
}