From c6b9e45f87d5596dd0a5c0519751aec55afd607c Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Fri, 15 Nov 2019 18:08:43 +0200 Subject: [PATCH] Added cluster autoscaling Updated docs Added tests for cluster autoscaling in node_pool fixture * Fix #93 --- autogen/cluster.tf | 13 ++++++++++ autogen/main.tf | 14 +++++++++++ autogen/variables.tf | 14 ++++++++++- cluster.tf | 1 + examples/node_pool/README.md | 1 + examples/node_pool/main.tf | 1 + examples/node_pool/variables.tf | 11 +++++++++ .../README.md | 1 + .../cluster.tf | 13 ++++++++++ .../main.tf | 14 +++++++++++ .../variables.tf | 12 ++++++++++ modules/beta-private-cluster/README.md | 1 + modules/beta-private-cluster/cluster.tf | 13 ++++++++++ modules/beta-private-cluster/main.tf | 14 +++++++++++ modules/beta-private-cluster/variables.tf | 12 ++++++++++ modules/beta-public-cluster/README.md | 1 + modules/beta-public-cluster/cluster.tf | 13 ++++++++++ modules/beta-public-cluster/main.tf | 14 +++++++++++ modules/beta-public-cluster/variables.tf | 12 ++++++++++ .../private-cluster-update-variant/cluster.tf | 1 + .../variables.tf | 1 - modules/private-cluster/cluster.tf | 1 + modules/private-cluster/variables.tf | 1 - test/fixtures/node_pool/example.tf | 10 ++++++++ test/integration/node_pool/controls/gcloud.rb | 24 +++++++++++++++++++ variables.tf | 1 - 26 files changed, 210 insertions(+), 4 deletions(-) diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 4791220591..eb0e46eba9 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -45,6 +45,7 @@ resource "google_container_cluster" "primary" { } } + {% if beta_cluster %} dynamic "release_channel" { for_each = local.release_channel @@ -62,6 +63,18 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service {% if beta_cluster %} + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/autogen/main.tf b/autogen/main.tf index 841444ea44..47aa3d9be8 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -50,6 +50,20 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal {% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) {% endif %} diff --git a/autogen/variables.tf b/autogen/variables.tf index ad5cc44e34..9ffa2efc97 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -178,8 +178,20 @@ variable "node_pools_metadata" { default-node-pool = {} } } - {% if beta_cluster %} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({key=string,value=string,effect=string}))) description = "Map of lists containing node taints by node-pool name" diff --git a/cluster.tf b/cluster.tf index 072a60fb14..d9646339ed 100644 --- a/cluster.tf +++ b/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 237b3f0b6f..7b8867b9c8 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,6 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 5bc0f53407..1730b6005a 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -36,6 +36,7 @@ module "gke" { create_service_account = false remove_default_node_pool = true disable_legacy_metadata_endpoints = false + cluster_autoscaling = var.cluster_autoscaling node_pools = [ { diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 040c78d2c4..82c02e5756 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -52,3 +52,14 @@ variable "compute_engine_service_account" { description = "Service account to associate to the nodes in the cluster" } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d2b5197726..2e1acbb4dd 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 10c206f35c..af32f478e1 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 5b235ce00f..1f0b3ca8f3 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d04ed5ac1f..8a592fa7bf 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 96ad9abf5a..c60157400e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c363dacf9f..42ef7389bf 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 5b235ce00f..1f0b3ca8f3 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d04ed5ac1f..8a592fa7bf 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fcf78cccc..af6d9c3c52 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -136,6 +136,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 1f5eee84aa..979cf8d23f 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9668b6f1ea..c44a3ad315 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a1057f1843..41d70b7a52 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index d7fc2dd736..0d04412da4 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 508a4f1b96..3398d01422 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index c8051255bf..fe225b9991 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 508a4f1b96..3398d01422 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 28d51f2357..2df8b89df2 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -26,5 +26,15 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account + + cluster_autoscaling = { + enabled = true + resource_limits = { + max_cpu_cores = 20 + min_cpu_cores = 5 + max_memory_gb = 30 + min_memory_gb = 10 + } + } } diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 69a15e8293..c08b61f1c6 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -33,6 +33,30 @@ end end + describe "cluster-autoscaling" do + it "has the expected cluster autoscaling settings" do + expect(data['autoscaling']).to eq({ + "autoprovisioningNodePoolDefaults" => { + "oauthScopes" => %w(https://www.googleapis.com/auth/logging.write https://www.googleapis.com/auth/monitoring), + "serviceAccount" => "default" + }, + "enableNodeAutoprovisioning" => true, + "resourceLimits" => [ + { + "maximum" => "20", + "minimum" => "5", + "resourceType" => "cpu" + }, + { + "maximum" => "30", + "minimum" => "10", + "resourceType" => "memory" + } + ] + }) + end + end + describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } diff --git a/variables.tf b/variables.tf index 58cf1f4685..bd110cbf42 100644 --- a/variables.tf +++ b/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name"