From d16d63992c129dd72726bcd1d3109302dec4c633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Wed, 21 Mar 2018 22:57:46 +0100 Subject: [PATCH 1/7] Add time partitioning field to google_bigquery_table resource --- google/resource_bigquery_table.go | 17 +++++++++++++++++ website/docs/r/bigquery_table.html.markdown | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 4887ffa8099..07e1b77d057 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -141,6 +141,14 @@ func resourceBigQueryTable() *schema.Resource { Required: true, ValidateFunc: validation.StringInSlice([]string{"DAY"}, false), }, + + // Type: [Optional] The field used to determine how to create a time-based + // partition. If time-based partitioning is enabled without this value, the + // table is partitioned based on the load time. + "field": { + Type: schema.TypeString, + Optional: true, + }, }, }, }, @@ -419,6 +427,10 @@ func expandTimePartitioning(configured interface{}) *bigquery.TimePartitioning { raw := configured.([]interface{})[0].(map[string]interface{}) tp := &bigquery.TimePartitioning{Type: raw["type"].(string)} + if v, ok := raw["field"]; ok { + tp.Field = v.(string) + } + if v, ok := raw["expiration_ms"]; ok { tp.ExpirationMs = int64(v.(int)) } @@ -428,6 +440,11 @@ func expandTimePartitioning(configured interface{}) *bigquery.TimePartitioning { func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interface{} { result := map[string]interface{}{"type": tp.Type} + result["field"] = tp.Type + + if tp.Type != "" { + result["field"] = tp.Type + } if tp.ExpirationMs != 0 { result["expiration_ms"] = tp.ExpirationMs diff --git a/website/docs/r/bigquery_table.html.markdown b/website/docs/r/bigquery_table.html.markdown index 3c16cdcccba..f1ab1fc6df5 100644 --- a/website/docs/r/bigquery_table.html.markdown +++ b/website/docs/r/bigquery_table.html.markdown @@ -81,6 +81,10 @@ The `time_partitioning` block supports: * `expiration_ms` - (Optional) Number of milliseconds for which to keep the storage for a partition. +* `field` - (Optional) The field used to determine how to create a time-based + partition. If time-based partitioning is enabled without this value, the + table is partitioned based on the load time. + * `type` - (Required) The only type supported is DAY, which will generate one partition per day based on data loading time. From 55ae14f63d3dbf28aacb2110154f8a0c07cdc3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Fri, 23 Mar 2018 23:55:33 +0100 Subject: [PATCH 2/7] Fix flatten time partitioning field to google_bigquery_table resource --- google/resource_bigquery_table.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index 07e1b77d057..e938cb86c5f 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -440,10 +440,9 @@ func expandTimePartitioning(configured interface{}) *bigquery.TimePartitioning { func flattenTimePartitioning(tp *bigquery.TimePartitioning) []map[string]interface{} { result := map[string]interface{}{"type": tp.Type} - result["field"] = tp.Type - if tp.Type != "" { - result["field"] = tp.Type + if tp.Field != "" { + result["field"] = tp.Field } if tp.ExpirationMs != 0 { From bde5269d73f2e62087ced0cc24709de2e2002ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Sat, 24 Mar 2018 00:33:08 +0100 Subject: [PATCH 3/7] Add resource bigquery table time partitioning field test --- google/resource_bigquery_table_test.go | 60 ++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index c09f4d418c1..a068b150e08 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -40,6 +40,28 @@ func TestAccBigQueryTable_Basic(t *testing.T) { }) } +func TestAccBigQueryTable_TimePartitioningField(t *testing.T) { + t.Parallel() + + datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckBigQueryTableDestroy, + Steps: []resource.TestStep{ + { + Config: testAccBigQueryTableWithTimePartitioningField(datasetID, tableID), + Check: resource.ComposeTestCheckFunc( + testAccBigQueryTableExists( + "google_bigquery_table.test"), + ), + }, + }, + }) +} + func TestAccBigQueryTable_View(t *testing.T) { t.Parallel() @@ -226,6 +248,44 @@ EOH }`, datasetID, tableID) } +func testAccBigQueryTableWithTimePartitioningField(datasetID, tableID string) string { + return fmt.Sprintf(` +resource "google_bigquery_dataset" "test" { + dataset_id = "%s" +} + +resource "google_bigquery_table" "test" { + table_id = "%s" + dataset_id = "${google_bigquery_dataset.test.dataset_id}" + + time_partitioning { + type = "DAY" + field = "ts" + } + + schema = < Date: Fri, 30 Mar 2018 00:54:13 +0200 Subject: [PATCH 4/7] Move resource bigquery table time partitioning field test to basic --- google/resource_bigquery_table_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index a068b150e08..513158c8cf0 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -36,21 +36,7 @@ func TestAccBigQueryTable_Basic(t *testing.T) { "google_bigquery_table.test"), ), }, - }, - }) -} -func TestAccBigQueryTable_TimePartitioningField(t *testing.T) { - t.Parallel() - - datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) - tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) - - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckBigQueryTableDestroy, - Steps: []resource.TestStep{ { Config: testAccBigQueryTableWithTimePartitioningField(datasetID, tableID), Check: resource.ComposeTestCheckFunc( From 8a46873a1277dfcb0bbba949aa0f4dadfa82f9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Fri, 30 Mar 2018 01:01:41 +0200 Subject: [PATCH 5/7] Add step to check that all the fields match --- google/resource_bigquery_table_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index 513158c8cf0..d9598ffb164 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -13,6 +13,7 @@ import ( func TestAccBigQueryTable_Basic(t *testing.T) { t.Parallel() + resourceName := "google_bigquery_table.test" datasetID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) tableID := fmt.Sprintf("tf_test_%s", acctest.RandString(10)) @@ -24,26 +25,29 @@ func TestAccBigQueryTable_Basic(t *testing.T) { { Config: testAccBigQueryTable(datasetID, tableID), Check: resource.ComposeTestCheckFunc( - testAccBigQueryTableExists( - "google_bigquery_table.test"), + testAccBigQueryTableExists(resourceName), ), }, { Config: testAccBigQueryTableUpdated(datasetID, tableID), Check: resource.ComposeTestCheckFunc( - testAccBigQueryTableExists( - "google_bigquery_table.test"), + testAccBigQueryTableExists(resourceName), ), }, { Config: testAccBigQueryTableWithTimePartitioningField(datasetID, tableID), Check: resource.ComposeTestCheckFunc( - testAccBigQueryTableExists( - "google_bigquery_table.test"), + testAccBigQueryTableExists(resourceName), ), }, + + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, }, }) } From 15c6b512db6cd594bfb9c1133116f5d641e62977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Fri, 30 Mar 2018 02:23:11 +0200 Subject: [PATCH 6/7] Mark resource bigquery table time partitioning field as ForceNew --- google/resource_bigquery_table.go | 1 + 1 file changed, 1 insertion(+) diff --git a/google/resource_bigquery_table.go b/google/resource_bigquery_table.go index e938cb86c5f..df2bc848dd8 100644 --- a/google/resource_bigquery_table.go +++ b/google/resource_bigquery_table.go @@ -148,6 +148,7 @@ func resourceBigQueryTable() *schema.Resource { "field": { Type: schema.TypeString, Optional: true, + ForceNew: true, }, }, }, From d0904be7dfff64684390c28e0f650a667df7421f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Matellanes?= Date: Fri, 30 Mar 2018 12:30:43 +0200 Subject: [PATCH 7/7] Add time partitioning field test to testAccBigQueryTable config --- google/resource_bigquery_table_test.go | 50 +++----------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/google/resource_bigquery_table_test.go b/google/resource_bigquery_table_test.go index d9598ffb164..aebc5057afd 100644 --- a/google/resource_bigquery_table_test.go +++ b/google/resource_bigquery_table_test.go @@ -36,13 +36,6 @@ func TestAccBigQueryTable_Basic(t *testing.T) { ), }, - { - Config: testAccBigQueryTableWithTimePartitioningField(datasetID, tableID), - Check: resource.ComposeTestCheckFunc( - testAccBigQueryTableExists(resourceName), - ), - }, - { ResourceName: resourceName, ImportState: true, @@ -209,10 +202,15 @@ resource "google_bigquery_table" "test" { time_partitioning { type = "DAY" + field = "ts" } schema = <