diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a9eedc8aa9..c6511bd262 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -56,7 +56,7 @@ jobs: name: Minimum version check runs-on: ubuntu-latest container: - image: hashicorp/terraform:0.12.2 + image: hashicorp/terraform:0.12.6 steps: - uses: actions/checkout@master - name: Validate Code diff --git a/CHANGELOG.md b/CHANGELOG.md index abee4af9cc..0fb7bfccf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,14 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [[v7.?.?](https://github.com/terraform-aws-modules/terraform-aws-eks/compare/v7.0.0...HEAD)] - 2019-??-??] - Test against minimum versions specified in `versions.tf` (by @dpiddockcmp) +- Support for AWS EKS Managed Node Groups. (by @wmorgan6796) - Updated instance_profile_names and instance_profile_arns outputs to also consider launch template as well as asg (by @ankitwal) - **Breaking:** Configure the aws-auth configmap using the terraform kubernetes providers. Read the [docs](docs/upgrading-to-aws-auth-kubernetes-provider.md) for more info (by @sdehaes) - Updated application of `aws-auth` configmap to create `kube_config.yaml` and `aws_auth_configmap.yaml` in sequence (and not parallel) to `kubectl apply` (by @knittingdev) - Exit with error code when `aws-auth` configmap is unable to be updated (by @knittingdev) - Fix deprecated interpolation-only expression (by @angelabad) - Fix broken terraform plan/apply on a cluster < 1.14 (by @hodduc) +- Updated required version of AWS Provider to >= v2.38.0 for Managed Node Groups (by @wmorgan6796) #### Important notes diff --git a/README.md b/README.md index 6e534eae84..76a385f329 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | map\_accounts | Additional AWS account numbers to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | list(string) | `[]` | no | | map\_roles | Additional IAM roles to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | | map\_users | Additional IAM users to add to the aws-auth configmap. See examples/basic/variables.tf for example format. | object | `[]` | no | +| node\_groups | A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys. | any | `[]` | no | | permissions\_boundary | If provided, all IAM roles will be created with this permissions boundary attached. | string | `"null"` | no | | subnets | A list of subnets to place the EKS cluster and workers within. | list(string) | n/a | yes | | tags | A map of tags to add to all resources. | map(string) | `{}` | no | diff --git a/examples/managed_node_groups/main.tf b/examples/managed_node_groups/main.tf new file mode 100644 index 0000000000..00b93aa9ff --- /dev/null +++ b/examples/managed_node_groups/main.tf @@ -0,0 +1,104 @@ +terraform { + required_version = ">= 0.12.6" +} + +provider "aws" { + version = ">= 2.28.1" + region = var.region +} + +provider "random" { + version = "~> 2.1" +} + +provider "local" { + version = "~> 1.2" +} + +provider "null" { + version = "~> 2.1" +} + +provider "template" { + version = "~> 2.1" +} + +data "aws_availability_zones" "available" { +} + +locals { + cluster_name = "test-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 2.6" + + name = "test-vpc" + cidr = "172.16.0.0/16" + azs = data.aws_availability_zones.available.names + private_subnets = ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"] + public_subnets = ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"] + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = "1" + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = "1" + } +} + +module "eks" { + source = "../.." + cluster_name = local.cluster_name + subnets = module.vpc.private_subnets + + tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + vpc_id = module.vpc.vpc_id + + node_groups = [ + { + name = "example" + + node_group_desired_capacity = 1 + node_group_max_capacity = 10 + node_group_min_capacity = 1 + + instance_type = "m5.large" + node_group_k8s_labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + node_group_additional_tags = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + } + ] + + map_roles = var.map_roles + map_users = var.map_users + map_accounts = var.map_accounts +} diff --git a/examples/managed_node_groups/outputs.tf b/examples/managed_node_groups/outputs.tf new file mode 100644 index 0000000000..a0788aff1d --- /dev/null +++ b/examples/managed_node_groups/outputs.tf @@ -0,0 +1,25 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane." + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids attached to the cluster control plane." + value = module.eks.cluster_security_group_id +} + +output "kubectl_config" { + description = "kubectl config as generated by the module." + value = module.eks.kubeconfig +} + +output "config_map_aws_auth" { + description = "A kubernetes configuration to authenticate to this EKS cluster." + value = module.eks.config_map_aws_auth +} + +output "region" { + description = "AWS region." + value = var.region +} + diff --git a/examples/managed_node_groups/variables.tf b/examples/managed_node_groups/variables.tf new file mode 100644 index 0000000000..7085aeabd4 --- /dev/null +++ b/examples/managed_node_groups/variables.tf @@ -0,0 +1,52 @@ +variable "region" { + default = "us-west-2" +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + + default = [ + "777777777777", + "888888888888", + ] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + + default = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + + default = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] +} diff --git a/local.tf b/local.tf index d3eccdbad2..ba0bb72f2d 100644 --- a/local.tf +++ b/local.tf @@ -16,8 +16,9 @@ locals { default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) + worker_group_count = length(var.worker_groups) + worker_group_launch_template_count = length(var.worker_groups_launch_template) + worker_group_managed_node_group_count = length(var.node_groups) default_ami_id_linux = data.aws_ami.eks_worker.id default_ami_id_windows = data.aws_ami.eks_worker_windows.id @@ -79,6 +80,15 @@ locals { spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + ami_type = "AL2_x86_64" # AMI Type to use for the Managed Node Groups. Can be either: AL2_x86_64 or AL2_x86_64_GPU + ami_release_version = "" # AMI Release Version of the Managed Node Groups + source_security_group_id = [] # Source Security Group IDs to allow SSH Access to the Nodes. NOTE: IF LEFT BLANK, AND A KEY IS SPECIFIED, THE SSH PORT WILL BE OPENNED TO THE WORLD + node_group_k8s_labels = {} # Kubernetes Labels to apply to the nodes within the Managed Node Group + node_group_desired_capacity = 1 # Desired capacity of the Node Group + node_group_min_capacity = 1 # Min capacity of the Node Group (Minimum value allowed is 1) + node_group_max_capacity = 3 # Max capacity of the Node Group + node_group_iam_role_arn = "" # IAM role to use for Managed Node Groups instead of default one created by the automation + node_group_additional_tags = {} # Additional tags to be applied to the Node Groups } workers_group_defaults = merge( @@ -123,4 +133,7 @@ locals { "t2.small", "t2.xlarge" ] + + node_groups = { for node_group in var.node_groups : node_group["name"] => node_group } + } diff --git a/node_groups.tf b/node_groups.tf new file mode 100644 index 0000000000..c47436fd03 --- /dev/null +++ b/node_groups.tf @@ -0,0 +1,103 @@ +resource "aws_iam_role" "node_groups" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name = "${var.workers_role_name != "" ? var.workers_role_name : aws_eks_cluster.this.name}-managed-node-groups" + assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path + force_detach_policies = true + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKSWorkerNodePolicy" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEKS_CNI_Policy" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_AmazonEC2ContainerRegistryReadOnly" { + count = local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_role_policy_attachment" "node_groups_additional_policies" { + for_each = toset(var.workers_additional_policies) + + role = aws_iam_role.node_groups[0].name + policy_arn = each.key +} + +resource "aws_iam_role_policy_attachment" "node_groups_autoscaling" { + count = var.manage_worker_autoscaling_policy && var.attach_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + policy_arn = aws_iam_policy.node_groups_autoscaling[0].arn + role = aws_iam_role.node_groups[0].name +} + +resource "aws_iam_policy" "node_groups_autoscaling" { + count = var.manage_worker_autoscaling_policy && local.worker_group_managed_node_group_count > 0 ? 1 : 0 + name_prefix = "eks-worker-autoscaling-${aws_eks_cluster.this.name}" + description = "EKS worker node autoscaling policy for cluster ${aws_eks_cluster.this.name}" + policy = data.aws_iam_policy_document.worker_autoscaling.json + path = var.iam_path +} + +resource "random_pet" "node_groups" { + for_each = local.node_groups + + separator = "-" + length = 2 + + keepers = { + instance_type = lookup(each.value, "instance_type", local.workers_group_defaults["instance_type"]) + + ec2_ssh_key = lookup(each.value, "key_name", local.workers_group_defaults["key_name"]) + + source_security_group_ids = join("-", compact( + lookup(each.value, "source_security_group_ids", local.workers_group_defaults["source_security_group_id"] + ))) + + node_group_name = join("-", [var.cluster_name, each.value["name"]]) + } +} + +resource "aws_eks_node_group" "workers" { + for_each = local.node_groups + + node_group_name = join("-", [var.cluster_name, each.key, random_pet.node_groups[each.key].id]) + + cluster_name = var.cluster_name + node_role_arn = lookup(each.value, "iam_role_arn", aws_iam_role.node_groups[0].arn) + subnet_ids = lookup(each.value, "subnets", local.workers_group_defaults["subnets"]) + + scaling_config { + desired_size = lookup(each.value, "node_group_desired_capacity", local.workers_group_defaults["asg_desired_capacity"]) + max_size = lookup(each.value, "node_group_max_capacity", local.workers_group_defaults["asg_max_size"]) + min_size = lookup(each.value, "node_group_min_capacity", local.workers_group_defaults["asg_min_size"]) + } + + ami_type = lookup(each.value, "ami_type", null) + disk_size = lookup(each.value, "root_volume_size", null) + instance_types = [lookup(each.value, "instance_type", null)] + labels = lookup(each.value, "node_group_k8s_labels", null) + release_version = lookup(each.value, "ami_release_version", null) + + # This sometimes breaks idempotency as described in https://github.com/terraform-providers/terraform-provider-aws/issues/11063 + remote_access { + ec2_ssh_key = lookup(each.value, "key_name", "") != "" ? each.value["key_name"] : null + source_security_group_ids = lookup(each.value, "key_name", "") != "" ? lookup(each.value, "source_security_group_ids", []) : null + } + + version = aws_eks_cluster.this.version + + tags = lookup(each.value, "node_group_additional_tags", null) + + lifecycle { + create_before_destroy = true + } +} diff --git a/variables.tf b/variables.tf index 06c9460fb8..498313b886 100644 --- a/variables.tf +++ b/variables.tf @@ -293,3 +293,9 @@ variable "attach_worker_cni_policy" { type = bool default = true } + +variable "node_groups" { + description = "A list of maps defining node group configurations to be defined using AWS EKS Managed Node Groups. See workers_group_defaults for valid keys." + type = any + default = [] +} \ No newline at end of file diff --git a/versions.tf b/versions.tf index 0a6046ef7f..61c59ea9e6 100644 --- a/versions.tf +++ b/versions.tf @@ -1,8 +1,8 @@ terraform { - required_version = ">= 0.12.2" + required_version = ">= 0.12.6" required_providers { - aws = ">= 2.31.0" + aws = ">= 2.38.0" local = ">= 1.2" null = ">= 2.1" template = ">= 2.1"