From 04d5bb15411a5528501eca803c7c2bdc03cb3d88 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Tue, 21 Jan 2025 14:45:51 -0800 Subject: [PATCH 1/6] feat(bigquery): Support resource tags for datasets in java client --- .../com/google/cloud/bigquery/Dataset.java | 6 +++ .../google/cloud/bigquery/DatasetInfo.java | 45 ++++++++++++++++ .../cloud/bigquery/DatasetInfoTest.java | 7 +++ .../google/cloud/bigquery/DatasetTest.java | 8 +++ .../cloud/bigquery/it/ITBigQueryTest.java | 53 +++++++++++++++++++ 5 files changed, 119 insertions(+) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java index 4fc857785..18606e701 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java @@ -170,6 +170,12 @@ public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) { return this; } + @Override + public Builder setResourceTags(Map resourceTags) { + infoBuilder.setResourceTags(resourceTags); + return this; + } + @Override public Dataset build() { return new Dataset(bigquery, infoBuilder); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java index f9b7f03e1..3c3bc616f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java @@ -76,6 +76,7 @@ public Dataset apply(DatasetInfo datasetInfo) { private final ExternalDatasetReference externalDatasetReference; private final String storageBillingModel; private final Long maxTimeTravelHours; + private final Map resourceTags; /** A builder for {@code DatasetInfo} objects. */ public abstract static class Builder { @@ -184,6 +185,19 @@ public abstract Builder setDefaultEncryptionConfiguration( */ public abstract Builder setDefaultCollation(String defaultCollation); + /** + * Optional. The tags attached to this + * dataset. Tag keys are globally unique. Tag key is expected to be in the namespaced format, + * for example "123456789012/environment" where 123456789012 is the ID of the parent + * organization or project resource for this tag key. Tag value is expected to be the short + * name, for example "Production". + * + * @see Tag + * definitions for more details. + * @param resourceTags resourceTags or {@code null} for none + */ + public abstract Builder setResourceTags(Map resourceTags); + /** Creates a {@code DatasetInfo} object. */ public abstract DatasetInfo build(); } @@ -208,6 +222,7 @@ static final class BuilderImpl extends Builder { private ExternalDatasetReference externalDatasetReference; private String storageBillingModel; private Long maxTimeTravelHours; + private Map resourceTags; BuilderImpl() {} @@ -230,6 +245,7 @@ static final class BuilderImpl extends Builder { this.externalDatasetReference = datasetInfo.externalDatasetReference; this.storageBillingModel = datasetInfo.storageBillingModel; this.maxTimeTravelHours = datasetInfo.maxTimeTravelHours; + this.resourceTags = datasetInfo.resourceTags; } BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) { @@ -270,6 +286,9 @@ public Acl apply(Dataset.Access accessPb) { } this.storageBillingModel = datasetPb.getStorageBillingModel(); this.maxTimeTravelHours = datasetPb.getMaxTimeTravelHours(); + if (datasetPb.getResourceTags() != null) { + this.resourceTags = datasetPb.getResourceTags(); + } } @Override @@ -388,6 +407,12 @@ public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) { return this; } + @Override + public Builder setResourceTags(Map resourceTags) { + this.resourceTags = resourceTags; + return this; + } + @Override public DatasetInfo build() { return new DatasetInfo(this); @@ -413,6 +438,7 @@ public DatasetInfo build() { externalDatasetReference = builder.externalDatasetReference; storageBillingModel = builder.storageBillingModel; maxTimeTravelHours = builder.maxTimeTravelHours; + resourceTags = builder.resourceTags; } /** Returns the dataset identity. */ @@ -554,6 +580,21 @@ public Long getMaxTimeTravelHours() { return maxTimeTravelHours; } + /** + * Optional. The tags attached to this + * dataset. Tag keys are globally unique. Tag key is expected to be in the namespaced format, for + * example "123456789012/environment" where 123456789012 is the ID of the parent organization or + * project resource for this tag key. Tag value is expected to be the short name, for example + * "Production". + * + * @see Tag + * definitions for more details. + * @return value or {@code null} for none + */ + public Map getResourceTags() { + return resourceTags; + } + /** * Returns information about the external metadata storage where the dataset is defined. Filled * out when the dataset type is EXTERNAL. @@ -588,6 +629,7 @@ public String toString() { .add("externalDatasetReference", externalDatasetReference) .add("storageBillingModel", storageBillingModel) .add("maxTimeTravelHours", maxTimeTravelHours) + .add("resourceTags", resourceTags) .toString(); } @@ -675,6 +717,9 @@ public Dataset.Access apply(Acl acl) { if (maxTimeTravelHours != null) { datasetPb.setMaxTimeTravelHours(maxTimeTravelHours); } + if (resourceTags != null) { + datasetPb.setResourceTags(resourceTags); + } return datasetPb; } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java index 49a392baf..e999b86e2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java @@ -62,6 +62,10 @@ public class DatasetInfoTest { private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final Long MAX_TIME_TRAVEL_HOURS_5_DAYS = 120L; private static final Long MAX_TIME_TRAVEL_HOURS_7_DAYS = 168L; + private static final Map RESOURCE_TAGS = + ImmutableMap.of( + "example-key1", "example-value1", + "example-key2", "example-value2"); private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE = ExternalDatasetReference.newBuilder() @@ -85,6 +89,7 @@ public class DatasetInfoTest { .setDefaultPartitionExpirationMs(DEFAULT_PARTITION__EXPIRATION) .setStorageBillingModel(STORAGE_BILLING_MODEL) .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS_7_DAYS) + .setResourceTags(RESOURCE_TAGS) .build(); private static final DatasetInfo DATASET_INFO_COMPLETE = DATASET_INFO @@ -183,6 +188,7 @@ public void testBuilder() { assertEquals( MAX_TIME_TRAVEL_HOURS_5_DAYS, DATASET_INFO_WITH_MAX_TIME_TRAVEL_5_DAYS.getMaxTimeTravelHours()); + assertEquals(RESOURCE_TAGS, DATASET_INFO.getResourceTags()); } @Test @@ -272,5 +278,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) { assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel()); assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours()); + assertEquals(expected.getResourceTags(), value.getResourceTags()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java index e2d7c635c..d138e3cb5 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java @@ -68,6 +68,10 @@ public class DatasetTest { private static final Field FIELD = Field.of("FieldName", LegacySQLTypeName.INTEGER); private static final String STORAGE_BILLING_MODEL = "LOGICAL"; private static final Long MAX_TIME_TRAVEL_HOURS = 168L; + private static final Map RESOURCE_TAGS = + ImmutableMap.of( + "example-key1", "example-value1", + "example-key2", "example-value2"); private static final StandardTableDefinition TABLE_DEFINITION = StandardTableDefinition.of(Schema.of(FIELD)); private static final ViewDefinition VIEW_DEFINITION = ViewDefinition.of("QUERY"); @@ -124,6 +128,7 @@ public void testBuilder() { .setLabels(LABELS) .setStorageBillingModel(STORAGE_BILLING_MODEL) .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) + .setResourceTags(RESOURCE_TAGS) .build(); assertEquals(DATASET_ID, builtDataset.getDatasetId()); assertEquals(ACCESS_RULES, builtDataset.getAcl()); @@ -139,6 +144,7 @@ public void testBuilder() { assertEquals(LABELS, builtDataset.getLabels()); assertEquals(STORAGE_BILLING_MODEL, builtDataset.getStorageBillingModel()); assertEquals(MAX_TIME_TRAVEL_HOURS, builtDataset.getMaxTimeTravelHours()); + assertEquals(RESOURCE_TAGS, builtDataset.getResourceTags()); } @Test @@ -348,6 +354,7 @@ public void testExternalDatasetReference() { .setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE) .setStorageBillingModel(STORAGE_BILLING_MODEL) .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) + .setResourceTags(RESOURCE_TAGS) .build(); assertEquals( EXTERNAL_DATASET_REFERENCE, @@ -379,5 +386,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) { assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference()); assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel()); assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours()); + assertEquals(expected.getResourceTags(), value.getResourceTags()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 0178ac10a..41450f877 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -223,6 +223,10 @@ public class ITBigQueryTest { ImmutableMap.of( "example-label1", "example-value1", "example-label2", "example-value2"); + private static final Map RESOURCE_TAGS = + ImmutableMap.of( + PROJECT_ID + "/example-key1", "example-value1", + PROJECT_ID + "/example-key2", "example-value2"); private static final Field TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) @@ -1278,6 +1282,7 @@ public void testUpdateDataset() { assertThat(dataset.getLabels()).containsExactly("a", "b"); assertThat(dataset.getStorageBillingModel()).isNull(); assertThat(dataset.getMaxTimeTravelHours()).isNull(); + assertThat(dataset.getResourceTags()).isNull(); Map updateLabels = new HashMap<>(); updateLabels.put("x", "y"); @@ -1290,17 +1295,48 @@ public void testUpdateDataset() { .setLabels(updateLabels) .setStorageBillingModel("LOGICAL") .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) + .setResourceTags(RESOURCE_TAGS) .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL"); assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(MAX_TIME_TRAVEL_HOURS); + assertThat(updatedDataset.getResourceTags()).isEqualTo(RESOURCE_TAGS); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); assertThat(dataset.delete()).isTrue(); } + @Test + public void testUpdateDatasetResourceTags() { + Dataset dataset = + bigquery.create( + DatasetInfo.newBuilder(OTHER_DATASET) + .setDescription("Some Description") + .setLabels(Collections.singletonMap("a", "b")) + .setResourceTags(RESOURCE_TAGS) + .build()); + + Map updatedResourceTags = + ImmutableMap.of( + PROJECT_ID + "updated-key1", + "updated-value1", + PROJECT_ID + "updated-key2", + "updated-value2"); + + Dataset updatedDataset = + bigquery.update(dataset.toBuilder().setResourceTags(updatedResourceTags).build()); + assertThat(updatedDataset.getResourceTags()).isEqualTo(updatedResourceTags); + + assertThat(updatedResourceTags.remove(PROJECT_ID + "updated-key2")).isNotNull(); + updatedDataset = + bigquery.update(dataset.toBuilder().setResourceTags(updatedResourceTags).build()); + assertThat(updatedDataset.getResourceTags()).isEqualTo(updatedResourceTags); + + assertThat(updatedDataset.delete()).isTrue(); + } + @Test public void testUpdateDatasetWithSelectedFields() { Dataset dataset = @@ -1767,6 +1803,23 @@ public void testCreateDatasetWithDefaultMaxTimeTravelHours() { RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset); } + @Test + public void testCreateDatasetWithSpecificResourceTags() { + String resourceTaggedDataset = RemoteBigQueryHelper.generateDatasetName(); + DatasetInfo info = + DatasetInfo.newBuilder(resourceTaggedDataset) + .setDescription(DESCRIPTION) + .setLabels(LABELS) + .setResourceTags(RESOURCE_TAGS) + .build(); + bigquery.create(info); + + Dataset dataset = bigquery.getDataset(DatasetId.of(resourceTaggedDataset)); + assertEquals(RESOURCE_TAGS, dataset.getResourceTags()); + + RemoteBigQueryHelper.forceDelete(bigquery, resourceTaggedDataset); + } + @Test public void testCreateDatasetWithDefaultCollation() { String collationDataset = RemoteBigQueryHelper.generateDatasetName(); From 10499a825e4e2349d268aadee2100a41e2633bc4 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Tue, 21 Jan 2025 14:56:41 -0800 Subject: [PATCH 2/6] add method to clirr-ignored-diff file --- google-cloud-bigquery/clirr-ignored-differences.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index b6546847f..9ef680c66 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -134,4 +134,9 @@ com/google/cloud/bigquery/DatasetInfo* *setMaxTimeTravelHours(*) + + 7013 + com/google/cloud/bigquery/DatasetInfo* + *setResourceTags(*) + \ No newline at end of file From 11f0f3e0501733e6fa5c7ae155f32a60f1dccb24 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Tue, 21 Jan 2025 16:22:02 -0800 Subject: [PATCH 3/6] Try Acl permissions to grant tag permissions --- .../cloud/bigquery/it/ITBigQueryTest.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 41450f877..e52349d4c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1295,25 +1295,35 @@ public void testUpdateDataset() { .setLabels(updateLabels) .setStorageBillingModel("LOGICAL") .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) - .setResourceTags(RESOURCE_TAGS) + // .setResourceTags(RESOURCE_TAGS) .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL"); assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(MAX_TIME_TRAVEL_HOURS); - assertThat(updatedDataset.getResourceTags()).isEqualTo(RESOURCE_TAGS); + // assertThat(updatedDataset.getResourceTags()).isEqualTo(RESOURCE_TAGS); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); assertThat(dataset.delete()).isTrue(); } + /* @Test - public void testUpdateDatasetResourceTags() { + public void testUpdateDatasetResourceTags() throws IOException { + String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); + + User user = new User(credentials.getClientEmail()); + Acl.Role role = Acl.Role.OWNER; + Acl acl = Acl.of(user, role); + Dataset dataset = bigquery.create( DatasetInfo.newBuilder(OTHER_DATASET) .setDescription("Some Description") + .setAcl(ImmutableList.of(acl)) .setLabels(Collections.singletonMap("a", "b")) .setResourceTags(RESOURCE_TAGS) .build()); @@ -1336,6 +1346,7 @@ public void testUpdateDatasetResourceTags() { assertThat(updatedDataset.delete()).isTrue(); } + */ @Test public void testUpdateDatasetWithSelectedFields() { @@ -1805,10 +1816,17 @@ public void testCreateDatasetWithDefaultMaxTimeTravelHours() { @Test public void testCreateDatasetWithSpecificResourceTags() { + ServiceAccountCredentials credentials = + (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); + User user = new User(credentials.getClientEmail()); + Acl.Role role = Acl.Role.OWNER; + Acl acl = Acl.of(user, role); + String resourceTaggedDataset = RemoteBigQueryHelper.generateDatasetName(); DatasetInfo info = DatasetInfo.newBuilder(resourceTaggedDataset) .setDescription(DESCRIPTION) + .setAcl(ImmutableList.of(acl)) .setLabels(LABELS) .setResourceTags(RESOURCE_TAGS) .build(); From 659cef7030b548dff9de79f73c259eff1255ec62 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Tue, 21 Jan 2025 16:27:50 -0800 Subject: [PATCH 4/6] Add exception to func signature --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index e52349d4c..d1da18c38 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1815,7 +1815,7 @@ public void testCreateDatasetWithDefaultMaxTimeTravelHours() { } @Test - public void testCreateDatasetWithSpecificResourceTags() { + public void testCreateDatasetWithSpecificResourceTags() throws IOException { ServiceAccountCredentials credentials = (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); User user = new User(credentials.getClientEmail()); From 1aca9ef40a2eb01efad76db4b88cc5e5640906a9 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Mon, 27 Jan 2025 13:51:20 -0800 Subject: [PATCH 5/6] Remove IT tests --- .../google/cloud/bigquery/DatasetInfo.java | 16 ++--- .../cloud/bigquery/it/ITBigQueryTest.java | 71 ------------------- 2 files changed, 6 insertions(+), 81 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java index 3c3bc616f..90d6c2cd5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java @@ -76,7 +76,7 @@ public Dataset apply(DatasetInfo datasetInfo) { private final ExternalDatasetReference externalDatasetReference; private final String storageBillingModel; private final Long maxTimeTravelHours; - private final Map resourceTags; + private final Annotations resourceTags; /** A builder for {@code DatasetInfo} objects. */ public abstract static class Builder { @@ -222,7 +222,7 @@ static final class BuilderImpl extends Builder { private ExternalDatasetReference externalDatasetReference; private String storageBillingModel; private Long maxTimeTravelHours; - private Map resourceTags; + private Annotations resourceTags = Annotations.ZERO; BuilderImpl() {} @@ -286,9 +286,7 @@ public Acl apply(Dataset.Access accessPb) { } this.storageBillingModel = datasetPb.getStorageBillingModel(); this.maxTimeTravelHours = datasetPb.getMaxTimeTravelHours(); - if (datasetPb.getResourceTags() != null) { - this.resourceTags = datasetPb.getResourceTags(); - } + this.resourceTags = Annotations.fromPb(datasetPb.getResourceTags()); } @Override @@ -409,7 +407,7 @@ public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) { @Override public Builder setResourceTags(Map resourceTags) { - this.resourceTags = resourceTags; + this.resourceTags = Annotations.fromUser(resourceTags); return this; } @@ -592,7 +590,7 @@ public Long getMaxTimeTravelHours() { * @return value or {@code null} for none */ public Map getResourceTags() { - return resourceTags; + return resourceTags.userMap(); } /** @@ -717,9 +715,7 @@ public Dataset.Access apply(Acl acl) { if (maxTimeTravelHours != null) { datasetPb.setMaxTimeTravelHours(maxTimeTravelHours); } - if (resourceTags != null) { - datasetPb.setResourceTags(resourceTags); - } + datasetPb.setResourceTags(resourceTags.toPb()); return datasetPb; } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index d1da18c38..0178ac10a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -223,10 +223,6 @@ public class ITBigQueryTest { ImmutableMap.of( "example-label1", "example-value1", "example-label2", "example-value2"); - private static final Map RESOURCE_TAGS = - ImmutableMap.of( - PROJECT_ID + "/example-key1", "example-value1", - PROJECT_ID + "/example-key2", "example-value2"); private static final Field TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) @@ -1282,7 +1278,6 @@ public void testUpdateDataset() { assertThat(dataset.getLabels()).containsExactly("a", "b"); assertThat(dataset.getStorageBillingModel()).isNull(); assertThat(dataset.getMaxTimeTravelHours()).isNull(); - assertThat(dataset.getResourceTags()).isNull(); Map updateLabels = new HashMap<>(); updateLabels.put("x", "y"); @@ -1295,59 +1290,17 @@ public void testUpdateDataset() { .setLabels(updateLabels) .setStorageBillingModel("LOGICAL") .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) - // .setResourceTags(RESOURCE_TAGS) .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL"); assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(MAX_TIME_TRAVEL_HOURS); - // assertThat(updatedDataset.getResourceTags()).isEqualTo(RESOURCE_TAGS); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); assertThat(dataset.delete()).isTrue(); } - /* - @Test - public void testUpdateDatasetResourceTags() throws IOException { - String accessPolicyDataset = RemoteBigQueryHelper.generateDatasetName(); - ServiceAccountCredentials credentials = - (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); - - User user = new User(credentials.getClientEmail()); - Acl.Role role = Acl.Role.OWNER; - Acl acl = Acl.of(user, role); - - Dataset dataset = - bigquery.create( - DatasetInfo.newBuilder(OTHER_DATASET) - .setDescription("Some Description") - .setAcl(ImmutableList.of(acl)) - .setLabels(Collections.singletonMap("a", "b")) - .setResourceTags(RESOURCE_TAGS) - .build()); - - Map updatedResourceTags = - ImmutableMap.of( - PROJECT_ID + "updated-key1", - "updated-value1", - PROJECT_ID + "updated-key2", - "updated-value2"); - - Dataset updatedDataset = - bigquery.update(dataset.toBuilder().setResourceTags(updatedResourceTags).build()); - assertThat(updatedDataset.getResourceTags()).isEqualTo(updatedResourceTags); - - assertThat(updatedResourceTags.remove(PROJECT_ID + "updated-key2")).isNotNull(); - updatedDataset = - bigquery.update(dataset.toBuilder().setResourceTags(updatedResourceTags).build()); - assertThat(updatedDataset.getResourceTags()).isEqualTo(updatedResourceTags); - - assertThat(updatedDataset.delete()).isTrue(); - } - */ - @Test public void testUpdateDatasetWithSelectedFields() { Dataset dataset = @@ -1814,30 +1767,6 @@ public void testCreateDatasetWithDefaultMaxTimeTravelHours() { RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset); } - @Test - public void testCreateDatasetWithSpecificResourceTags() throws IOException { - ServiceAccountCredentials credentials = - (ServiceAccountCredentials) GoogleCredentials.getApplicationDefault(); - User user = new User(credentials.getClientEmail()); - Acl.Role role = Acl.Role.OWNER; - Acl acl = Acl.of(user, role); - - String resourceTaggedDataset = RemoteBigQueryHelper.generateDatasetName(); - DatasetInfo info = - DatasetInfo.newBuilder(resourceTaggedDataset) - .setDescription(DESCRIPTION) - .setAcl(ImmutableList.of(acl)) - .setLabels(LABELS) - .setResourceTags(RESOURCE_TAGS) - .build(); - bigquery.create(info); - - Dataset dataset = bigquery.getDataset(DatasetId.of(resourceTaggedDataset)); - assertEquals(RESOURCE_TAGS, dataset.getResourceTags()); - - RemoteBigQueryHelper.forceDelete(bigquery, resourceTaggedDataset); - } - @Test public void testCreateDatasetWithDefaultCollation() { String collationDataset = RemoteBigQueryHelper.generateDatasetName(); From 8af57dd4317c4e535ce1e2781e8b56c21786f500 Mon Sep 17 00:00:00 2001 From: Liam Huffman Date: Mon, 27 Jan 2025 13:58:19 -0800 Subject: [PATCH 6/6] Write Integration tests for ds resource tags. Tests pass on my personal cloud test project but fail in the kokoro environment due to us not being able to add tags to the testing project. --- .../cloud/bigquery/it/ITBigQueryTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 0178ac10a..fdd1725f2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -223,6 +223,10 @@ public class ITBigQueryTest { ImmutableMap.of( "example-label1", "example-value1", "example-label2", "example-value2"); + private static final Map RESOURCE_TAGS = + ImmutableMap.of( + PROJECT_ID + "/example-key1", "example-value1", + PROJECT_ID + "/example-key2", "example-value2"); private static final Field TIMESTAMP_FIELD_SCHEMA = Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) .setMode(Field.Mode.NULLABLE) @@ -1278,6 +1282,7 @@ public void testUpdateDataset() { assertThat(dataset.getLabels()).containsExactly("a", "b"); assertThat(dataset.getStorageBillingModel()).isNull(); assertThat(dataset.getMaxTimeTravelHours()).isNull(); + assertThat(dataset.getResourceTags()).isNull(); Map updateLabels = new HashMap<>(); updateLabels.put("x", "y"); @@ -1290,17 +1295,49 @@ public void testUpdateDataset() { .setLabels(updateLabels) .setStorageBillingModel("LOGICAL") .setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS) + .setResourceTags(RESOURCE_TAGS) .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL"); assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(MAX_TIME_TRAVEL_HOURS); + assertThat(updatedDataset.getResourceTags()).isEqualTo(RESOURCE_TAGS); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); assertThat(dataset.delete()).isTrue(); } + @Test + public void testUpdateDatasetResourceTags() { + Dataset dataset = + bigquery.create( + DatasetInfo.newBuilder(OTHER_DATASET) + .setDescription("Some Description") + .setLabels(Collections.singletonMap("a", "b")) + .setResourceTags(RESOURCE_TAGS) + .build()); + + final String updatedKey1 = PROJECT_ID + "/updated-key1"; + final String updatedValue1 = "updated-value1"; + final String updatedKey2 = PROJECT_ID + "/updated-key2"; + final String updatedValue2 = "updated-value2"; + + Map updatedResourceTags = + new HashMap<>(ImmutableMap.of(updatedKey1, updatedValue1, updatedKey2, updatedValue2)); + updatedResourceTags.put(PROJECT_ID + "/example-key1", null); + updatedResourceTags.put(PROJECT_ID + "/example-key2", null); + + assertEquals(dataset.getResourceTags(), RESOURCE_TAGS); + + Dataset updatedDataset = + bigquery.update(dataset.toBuilder().setResourceTags(updatedResourceTags).build()); + assertThat(updatedDataset.getResourceTags()) + .containsExactly(updatedKey1, updatedValue1, updatedKey2, updatedValue2); + + assertTrue(dataset.delete()); + } + @Test public void testUpdateDatasetWithSelectedFields() { Dataset dataset = @@ -1767,6 +1804,23 @@ public void testCreateDatasetWithDefaultMaxTimeTravelHours() { RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset); } + @Test + public void testCreateDatasetWithSpecificResourceTags() { + String resourceTaggedDataset = RemoteBigQueryHelper.generateDatasetName(); + DatasetInfo info = + DatasetInfo.newBuilder(resourceTaggedDataset) + .setDescription(DESCRIPTION) + .setLabels(LABELS) + .setResourceTags(RESOURCE_TAGS) + .build(); + bigquery.create(info); + + Dataset dataset = bigquery.getDataset(DatasetId.of(resourceTaggedDataset)); + assertEquals(RESOURCE_TAGS, dataset.getResourceTags()); + + RemoteBigQueryHelper.forceDelete(bigquery, resourceTaggedDataset); + } + @Test public void testCreateDatasetWithDefaultCollation() { String collationDataset = RemoteBigQueryHelper.generateDatasetName();