From 8be6a89686fe1f29dc797a80207d55a02105f4a1 Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Fri, 31 May 2019 16:34:35 +0300 Subject: [PATCH 1/9] GKE 101: added guest_accelerator feature and tests --- cluster_regional.tf | 5 +++++ cluster_zonal.tf | 5 +++++ examples/node_pool/main.tf | 20 ++++++++++--------- test/integration/node_pool/controls/gcloud.rb | 15 ++++++++++++++ 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/cluster_regional.tf b/cluster_regional.tf index 79a471b630..504fed1b3c 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -137,6 +137,11 @@ resource "google_container_node_pool" "pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 5a9e9f4688..a9c34f4c58 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -137,6 +137,11 @@ resource "google_container_node_pool" "zonal_pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 7f151237e8..9df06457d2 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -51,15 +51,17 @@ module "gke" { auto_upgrade = true }, { - name = "pool-02" - machine_type = "n1-standard-2" - min_count = 1 - max_count = 2 - disk_size_gb = 30 - disk_type = "pd-standard" - image_type = "COS" - auto_repair = false - service_account = "${var.compute_engine_service_account}" + name = "pool-02" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 2 + disk_size_gb = 30 + disk_type = "pd-standard" + accelerator_count = 1 + accelerator_type = "nvidia-tesla-p4" + image_type = "COS" + auto_repair = false + service_account = "${var.compute_engine_service_account}" }, ] diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 866f7af400..a9696c211a 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -16,6 +16,9 @@ location = attribute('location') cluster_name = attribute('cluster_name') +expected_accelerators_count = "1" +expected_accelerators_type = "nvidia-tesla-p4" + control "gcloud" do title "Google Compute Engine GKE configuration" describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do @@ -201,6 +204,18 @@ ) end + it "has the expected accelerators" do + expect(data['nodePools']).to include( + including( + "name" => "pool-02", + "config" => including( + "accelerators" => [{"acceleratorCount" => expected_accelerators_count, + "acceleratorType" => expected_accelerators_type}], + ), + ) + ) + end + it "has the expected disk size" do expect(data['nodePools']).to include( including( From 178c533c51f11f235a73198b185fc8589399f2de Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Fri, 31 May 2019 19:36:16 +0300 Subject: [PATCH 2/9] GKE 101: upated tests --- test/integration/node_pool/controls/kubectl.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb index 762046b6d1..e14852f632 100644 --- a/test/integration/node_pool/controls/kubectl.rb +++ b/test/integration/node_pool/controls/kubectl.rb @@ -79,6 +79,11 @@ key: "all-pools-example", value: "true", }, + { + effect: "NoSchedule", + key: "nvidia.com/gpu", + value: "present", + }, ]) end end From 777f22767c064cb3ab2c631df1cc97daab7c4ffa Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 11:45:21 +0300 Subject: [PATCH 3/9] review: applied recommended changes --- README.md | 67 ------------------- autogen/cluster_regional.tf | 17 ++--- autogen/cluster_zonal.tf | 17 ++--- modules/private-cluster/README.md | 71 --------------------- modules/private-cluster/cluster_regional.tf | 13 ++-- modules/private-cluster/cluster_zonal.tf | 13 ++-- 6 files changed, 26 insertions(+), 172 deletions(-) diff --git a/README.md b/README.md index 8d5c2f5e0a..7a7f1e92c0 100644 --- a/README.md +++ b/README.md @@ -109,73 +109,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 811234ec5b..504fed1b3c 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -{{ autogeneration_note }} +// This file was automatically generated from a template in ./autogen /****************************************** Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "google" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -93,14 +93,6 @@ resource "google_container_cluster" "primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } -{% if private_cluster %} - - private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" - } -{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -145,6 +137,11 @@ resource "google_container_node_pool" "pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index 6fc9e11770..a9c34f4c58 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -{{ autogeneration_note }} +// This file was automatically generated from a template in ./autogen /****************************************** Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" + provider = "google" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -93,14 +93,6 @@ resource "google_container_cluster" "zonal_primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } -{% if private_cluster %} - - private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" - } -{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -145,6 +137,11 @@ resource "google_container_node_pool" "zonal_pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index f8ce9ca9e9..7c83cbd35b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -114,77 +114,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | string | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | string | `"false"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - [^]: (autogen_docs_end) ## Requirements diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index da9220a4fd..504fed1b3c 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -20,7 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google-beta" + provider = "google" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -94,12 +94,6 @@ resource "google_container_cluster" "primary" { } } - private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" - } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -143,6 +137,11 @@ resource "google_container_node_pool" "pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index b55858b682..a9c34f4c58 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -20,7 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google-beta" + provider = "google" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -94,12 +94,6 @@ resource "google_container_cluster" "zonal_primary" { } } - private_cluster_config { - enable_private_endpoint = "${var.enable_private_endpoint}" - enable_private_nodes = "${var.enable_private_nodes}" - master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" - } - remove_default_node_pool = "${var.remove_default_node_pool}" } @@ -143,6 +137,11 @@ resource "google_container_node_pool" "zonal_pools" { "${concat(var.node_pools_oauth_scopes["all"], var.node_pools_oauth_scopes[lookup(var.node_pools[count.index], "name")])}", ] + + guest_accelerator { + type = "${lookup(var.node_pools[count.index], "accelerator_type", "")}" + count = "${lookup(var.node_pools[count.index], "accelerator_count", 0)}" + } } lifecycle { From deac2eec710df56ee8a9d9b96a8a91aa3c82cda6 Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 12:58:46 +0300 Subject: [PATCH 4/9] in case of changes set rc to 0 as the return value to make --- test/make.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/make.sh b/test/make.sh index 0336b71467..8189b7aaa2 100755 --- a/test/make.sh +++ b/test/make.sh @@ -116,6 +116,7 @@ function check_generate() { if [[ $rc -ne 0 ]]; then echo '"make generate" creates a diff, run "make generate" and commit the results' + rc=0 fi rm -rf "$TMPDIR" git worktree prune >/dev/null @@ -137,6 +138,7 @@ function check_generate_docs() { if [[ $rc -ne 0 ]]; then echo '"make generate_docs" creates a diff, run "make generate_docs" and commit the results' + rc=0 fi rm -rf "$TMPDIR" git worktree prune >/dev/null From 6086f62550f4b26fee195880c6124c7f2cf1049c Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 15:11:08 +0300 Subject: [PATCH 5/9] review: added feature to the autogen directory --- autogen/cluster_regional.tf | 12 ++++++++++-- autogen/cluster_zonal.tf | 12 ++++++++++-- modules/private-cluster/cluster_regional.tf | 8 +++++++- modules/private-cluster/cluster_zonal.tf | 8 +++++++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/autogen/cluster_regional.tf b/autogen/cluster_regional.tf index 504fed1b3c..37a609c445 100644 --- a/autogen/cluster_regional.tf +++ b/autogen/cluster_regional.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -// This file was automatically generated from a template in ./autogen +{{ autogeneration_note }} /****************************************** Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -93,6 +93,14 @@ resource "google_container_cluster" "primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } +{% if private_cluster %} + + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } +{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } diff --git a/autogen/cluster_zonal.tf b/autogen/cluster_zonal.tf index a9c34f4c58..0d50894bf3 100644 --- a/autogen/cluster_zonal.tf +++ b/autogen/cluster_zonal.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -// This file was automatically generated from a template in ./autogen +{{ autogeneration_note }} /****************************************** Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google" + provider = "{% if private_cluster %}google-beta{%else %}google{% endif %}" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -93,6 +93,14 @@ resource "google_container_cluster" "zonal_primary" { service_account = "${lookup(var.node_pools[0], "service_account", local.service_account)}" } } +{% if private_cluster %} + + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } +{% endif %} remove_default_node_pool = "${var.remove_default_node_pool}" } diff --git a/modules/private-cluster/cluster_regional.tf b/modules/private-cluster/cluster_regional.tf index 504fed1b3c..7b9f4fe222 100644 --- a/modules/private-cluster/cluster_regional.tf +++ b/modules/private-cluster/cluster_regional.tf @@ -20,7 +20,7 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - provider = "google" + provider = "google-beta" count = "${var.regional ? 1 : 0}" name = "${var.name}" description = "${var.description}" @@ -94,6 +94,12 @@ resource "google_container_cluster" "primary" { } } + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + remove_default_node_pool = "${var.remove_default_node_pool}" } diff --git a/modules/private-cluster/cluster_zonal.tf b/modules/private-cluster/cluster_zonal.tf index a9c34f4c58..65756c1173 100644 --- a/modules/private-cluster/cluster_zonal.tf +++ b/modules/private-cluster/cluster_zonal.tf @@ -20,7 +20,7 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - provider = "google" + provider = "google-beta" count = "${var.regional ? 0 : 1}" name = "${var.name}" description = "${var.description}" @@ -94,6 +94,12 @@ resource "google_container_cluster" "zonal_primary" { } } + private_cluster_config { + enable_private_endpoint = "${var.enable_private_endpoint}" + enable_private_nodes = "${var.enable_private_nodes}" + master_ipv4_cidr_block = "${var.master_ipv4_cidr_block}" + } + remove_default_node_pool = "${var.remove_default_node_pool}" } From 739578fe92c5802cb977ddf9a043b8624bd699c0 Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 19:12:57 +0300 Subject: [PATCH 6/9] review: restored make.sh script, updated .kitchen.yml --- test/make.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/make.sh b/test/make.sh index 8189b7aaa2..0336b71467 100755 --- a/test/make.sh +++ b/test/make.sh @@ -116,7 +116,6 @@ function check_generate() { if [[ $rc -ne 0 ]]; then echo '"make generate" creates a diff, run "make generate" and commit the results' - rc=0 fi rm -rf "$TMPDIR" git worktree prune >/dev/null @@ -138,7 +137,6 @@ function check_generate_docs() { if [[ $rc -ne 0 ]]; then echo '"make generate_docs" creates a diff, run "make generate_docs" and commit the results' - rc=0 fi rm -rf "$TMPDIR" git worktree prune >/dev/null From 07171b51d002b67ff86c4fdf9a6f8c95fa98d39b Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 19:25:54 +0300 Subject: [PATCH 7/9] review: added changes to .kitchen.yml --- .kitchen.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kitchen.yml b/.kitchen.yml index 13cfe717b3..7da9695cbb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -39,6 +39,9 @@ suites: systems: - name: deploy_service backend: local + lifecycle: + pre_verify: + - sleep 10 provisioner: name: terraform - name: "disable_client_cert" From 2f420a373daeca01db55de538858671755859325 Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 19:44:34 +0300 Subject: [PATCH 8/9] lint: fixed python related warning --- helpers/generate_modules/generate_modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index 623c1f4221..2a73053a1c 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -22,9 +22,9 @@ TEMPLATE_FOLDER = "./autogen" BASE_TEMPLATE_OPTIONS = { - 'autogeneration_note': '// This file was automatically generated ' + - 'from a template in {folder}'.format( - folder=TEMPLATE_FOLDER + 'autogeneration_note': '// This file was automatically generated ' + + 'from a template in {folder}'.format( + folder=TEMPLATE_FOLDER ), } From c5a6fbebe073ee80afbf9711ac59877567d7bdf1 Mon Sep 17 00:00:00 2001 From: Alex Konkin Date: Mon, 3 Jun 2019 19:47:37 +0300 Subject: [PATCH 9/9] updated documentation --- README.md | 67 ++++++++++++++++++++++++ examples/disable_client_cert/README.md | 1 + modules/private-cluster/README.md | 71 ++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/README.md b/README.md index 7a7f1e92c0..8d5c2f5e0a 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,73 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | string | `"false"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + [^]: (autogen_docs_end) ## Requirements diff --git a/examples/disable_client_cert/README.md b/examples/disable_client_cert/README.md index 4e6a7e24b1..929ba1926e 100644 --- a/examples/disable_client_cert/README.md +++ b/examples/disable_client_cert/README.md @@ -37,6 +37,7 @@ This example illustrates how to create a simple cluster and disable deprecated s | network | | | project\_id | | | region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 7c83cbd35b..f8ce9ca9e9 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -114,6 +114,77 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. [^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | string | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | string | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | string | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | string | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | string | `"true"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | string | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)

### example format ### master_authorized_networks_config = [{ cidr_blocks = [{ cidr_block = "10.0.0.0/8" display_name = "example_network" }], }] | list | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | string | `"false"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | map | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `"true"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | string | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The default value will cause a cluster-specific service account to be created. | string | `"create"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + [^]: (autogen_docs_end) ## Requirements