From 1d1b19032bd2403f6a133a07ab9eb79cf614b187 Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Wed, 2 Dec 2020 09:40:00 -0800 Subject: [PATCH 1/6] add datapath_provider var for beta clusters --- autogen/main/cluster.tf.tmpl | 3 +++ autogen/main/variables.tf.tmpl | 7 +++++++ modules/beta-private-cluster-update-variant/README.md | 1 + modules/beta-private-cluster-update-variant/cluster.tf | 1 + modules/beta-private-cluster-update-variant/variables.tf | 5 +++++ modules/beta-private-cluster/README.md | 1 + modules/beta-private-cluster/cluster.tf | 1 + modules/beta-private-cluster/variables.tf | 5 +++++ modules/beta-public-cluster-update-variant/README.md | 1 + modules/beta-public-cluster-update-variant/cluster.tf | 1 + modules/beta-public-cluster-update-variant/variables.tf | 5 +++++ modules/beta-public-cluster/README.md | 1 + modules/beta-public-cluster/cluster.tf | 1 + modules/beta-public-cluster/variables.tf | 5 +++++ 14 files changed, 38 insertions(+) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 0526d02a76..4bad6c8678 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -180,6 +180,9 @@ resource "google_container_cluster" "primary" { } {% endif %} } + {% if beta_cluster %} + datapath_provider = var.datapath_provider + {% endif %} {% if beta_cluster %} networking_mode = "VPC_NATIVE" diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 8d1267a29f..8cb32c0951 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -101,6 +101,13 @@ variable "network_policy_provider" { description = "The network policy provider." default = "CALICO" } +{% if beta_cluster %} +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} +{% endif %} variable "maintenance_start_time" { type = string diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 3d6ce9cf4a..de5a3cfbc9 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -158,6 +158,7 @@ Then perform the following commands on the root folder: | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index ddd9857f35..91d8a3f895 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -163,6 +163,7 @@ resource "google_container_cluster" "primary" { enabled = var.config_connector } } + datapath_provider = var.datapath_provider networking_mode = "VPC_NATIVE" ip_allocation_policy { diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 09efb45c8b..78f4663a3d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -101,6 +101,11 @@ variable "network_policy_provider" { description = "The network policy provider." default = "CALICO" } +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} variable "maintenance_start_time" { type = string diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index bb1aa43177..c927f5775b 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -136,6 +136,7 @@ Then perform the following commands on the root folder: | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 7bea7d0e2c..cadf8181b2 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -163,6 +163,7 @@ resource "google_container_cluster" "primary" { enabled = var.config_connector } } + datapath_provider = var.datapath_provider networking_mode = "VPC_NATIVE" ip_allocation_policy { diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 09efb45c8b..78f4663a3d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -101,6 +101,11 @@ variable "network_policy_provider" { description = "The network policy provider." default = "CALICO" } +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} variable "maintenance_start_time" { type = string diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 9a5ba632a1..778515925c 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -152,6 +152,7 @@ Then perform the following commands on the root folder: | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 4d130f1f3d..617eb213f6 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -163,6 +163,7 @@ resource "google_container_cluster" "primary" { enabled = var.config_connector } } + datapath_provider = var.datapath_provider networking_mode = "VPC_NATIVE" ip_allocation_policy { diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 9f23b5a2b7..dd2db9f08c 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -101,6 +101,11 @@ variable "network_policy_provider" { description = "The network policy provider." default = "CALICO" } +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} variable "maintenance_start_time" { type = string diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 9ae1a7c000..6fe00ec2a6 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -130,6 +130,7 @@ Then perform the following commands on the root folder: | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 4b3fd37cd5..7f0086462b 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -163,6 +163,7 @@ resource "google_container_cluster" "primary" { enabled = var.config_connector } } + datapath_provider = var.datapath_provider networking_mode = "VPC_NATIVE" ip_allocation_policy { diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 9f23b5a2b7..dd2db9f08c 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -101,6 +101,11 @@ variable "network_policy_provider" { description = "The network policy provider." default = "CALICO" } +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} variable "maintenance_start_time" { type = string From e8fc713eb2b49e0c11d648dd9e5530063011daed Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Wed, 2 Dec 2020 10:42:51 -0800 Subject: [PATCH 2/6] incorporate datapath_provider option --- examples/simple_regional_beta/main.tf | 4 ++++ examples/simple_regional_beta/variables.tf | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index 94d394b39a..d95fa328e5 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -51,6 +51,10 @@ module "gke" { # Disable workload identity identity_namespace = null node_metadata = "UNSPECIFIED" + + # Support Dataplane Setup + network_policy = var.network_policy + datapath_provider = var.datapath_provider } data "google_client_config" "default" { diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index cdedce33d2..4dc4f04f8c 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -123,3 +123,15 @@ variable "regional" { description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" default = true } + +variable "network_policy" { + type = bool + description = "Enable network policy addon" + default = true +} + +variable "datapath_provider" { + type = string + description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." + default = "DATAPATH_PROVIDER_UNSPECIFIED" +} From 648007cad59b4b42c8b33ffb1d63b3f79d1d895d Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Wed, 2 Dec 2020 10:43:18 -0800 Subject: [PATCH 3/6] update tests for datapath_provider beta cluster --- test/fixtures/beta_cluster/main.tf | 4 ++++ test/integration/beta_cluster/controls/gcloud.rb | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf index bd1a91b354..e229cdd654 100644 --- a/test/fixtures/beta_cluster/main.tf +++ b/test/fixtures/beta_cluster/main.tf @@ -61,6 +61,10 @@ module "this" { enable_binary_authorization = true enable_pod_security_policy = true + + // Dataplane-V2 Feature + network_policy = false + datapath_provider = "ADVANCED_DATAPATH" } data "google_client_config" "default" { diff --git a/test/integration/beta_cluster/controls/gcloud.rb b/test/integration/beta_cluster/controls/gcloud.rb index c7d67591e2..f76d7bcfcd 100644 --- a/test/integration/beta_cluster/controls/gcloud.rb +++ b/test/integration/beta_cluster/controls/gcloud.rb @@ -57,7 +57,7 @@ }, "kalmConfig" => {}, "configConnectorConfig" => {}, - "networkPolicyConfig" => {}, + "networkPolicyConfig" => {"disabled"=>true}, "istioConfig" => {"auth"=>"AUTH_MUTUAL_TLS"}, "cloudRunConfig" => including( "loadBalancerType" => "LOAD_BALANCER_TYPE_EXTERNAL", @@ -71,6 +71,12 @@ ) end + it "has the expected datapathProvider config" do + expect(data['networkConfig']).to include( + "datapathProvider" => "ADVANCED_DATAPATH" + ) + end + it "has the expected binaryAuthorization config" do expect(data['binaryAuthorization']).to eq({ "enabled" => true, From 745b46bbb29e274c7a467552b754d9aaae426d49 Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Wed, 2 Dec 2020 14:28:20 -0800 Subject: [PATCH 4/6] example README var updates --- examples/simple_regional_beta/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index d849d2bfff..c38da141bb 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -11,6 +11,7 @@ This example illustrates how to create a simple cluster with beta features. | cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | | enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | @@ -19,6 +20,7 @@ This example illustrates how to create a simple cluster with beta features. | ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | | istio | Boolean to enable / disable Istio | `bool` | `true` | no | | network | The VPC network to host the cluster in | `any` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `true` | no | | node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | project\_id | The project ID to host the cluster in | `any` | n/a | yes | | region | The region to host the cluster in | `any` | n/a | yes | From 4892479c97526f867eef16b6411a80dba6453b1e Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Thu, 4 Mar 2021 09:24:09 -0800 Subject: [PATCH 5/6] remove network_policy var usage --- examples/simple_regional_beta/main.tf | 5 ++--- examples/simple_regional_beta/variables.tf | 6 ------ test/fixtures/beta_cluster/main.tf | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index e78ac4ddef..e79e7275e4 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -61,7 +61,6 @@ module "gke" { identity_namespace = null node_metadata = "UNSPECIFIED" - # Support Dataplane Setup - network_policy = var.network_policy - datapath_provider = var.datapath_provider + # Enable Dataplane Setup + datapath_provider = "ADVANCED_DATAPATH" } diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index 4dc4f04f8c..8af6ea5afd 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -124,12 +124,6 @@ variable "regional" { default = true } -variable "network_policy" { - type = bool - description = "Enable network policy addon" - default = true -} - variable "datapath_provider" { type = string description = "The desired datapath provider for this cluster. By default, uses the IPTables-based kube-proxy implementation." diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf index e229cdd654..33832103d2 100644 --- a/test/fixtures/beta_cluster/main.tf +++ b/test/fixtures/beta_cluster/main.tf @@ -63,7 +63,6 @@ module "this" { enable_pod_security_policy = true // Dataplane-V2 Feature - network_policy = false datapath_provider = "ADVANCED_DATAPATH" } From d1ae09408938c2df6b3c28ca2049ac150f082ef7 Mon Sep 17 00:00:00 2001 From: ryan-atkins <> Date: Thu, 4 Mar 2021 09:27:14 -0800 Subject: [PATCH 6/6] finish docs updates --- examples/simple_regional_beta/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index c38da141bb..5b49a79de2 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -20,7 +20,6 @@ This example illustrates how to create a simple cluster with beta features. | ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | | istio | Boolean to enable / disable Istio | `bool` | `true` | no | | network | The VPC network to host the cluster in | `any` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `true` | no | | node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | project\_id | The project ID to host the cluster in | `any` | n/a | yes | | region | The region to host the cluster in | `any` | n/a | yes |