From 8571ad18bc388d1f5c48a4eb74b37febd8f58602 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Tue, 13 Feb 2024 16:20:18 -0800 Subject: [PATCH 1/7] feat: support includeFolders list option --- .../com/google/cloud/storage/Storage.java | 8 ++++++++ .../com/google/cloud/storage/UnifiedOpts.java | 19 +++++++++++++++++++ .../cloud/storage/spi/v1/HttpStorageRpc.java | 1 + .../cloud/storage/spi/v1/StorageRpc.java | 3 ++- 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 5ca31c042d..4d0c56b533 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -1788,6 +1788,14 @@ public static BlobListOption matchGlob(@NonNull String glob) { return new BlobListOption(UnifiedOpts.matchGlob(glob)); } + /** + * Returns an option for whether to include all Folders (including empty Folders) in response. + */ + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + public static BlobListOption includeFolders(boolean includeFolders) { + return new BlobListOption(UnifiedOpts.includeFoldersAsPrefixes(includeFolders)); + } + /** * Returns an option to define the billing user project. This option is required by buckets with * `requester_pays` flag enabled to assign operation costs. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 18f79be572..17e7a31869 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -365,6 +365,11 @@ static Delimiter delimiter(@NonNull String delimiter) { return new Delimiter(delimiter); } + static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersAsPrefixes) { + return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes); + } + + @Deprecated static DetectContentType detectContentType() { return DetectContentType.INSTANCE; @@ -631,6 +636,20 @@ public Mapper rewriteObject() { } } + static final class IncludeFoldersAsPrefixes extends RpcOptVal implements ObjectListOpt { + + private static final long serialVersionUID = 321916692864878282L; + + private IncludeFoldersAsPrefixes(boolean val) { + super(StorageRpc.Option.INCLUDE_FOLDERS_AS_PREFIXES, val); + } + + @Override + public Mapper listObjects() { + return null;// return b -> b.setIncludeFoldersAsPrefixes(val); + } + } + static final class Delimiter extends RpcOptVal implements ObjectListOpt { private static final long serialVersionUID = -3789556789947615714L; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 2f1c3e2101..3ca2eabec3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -459,6 +459,7 @@ public Tuple> list(final String bucket, Map storageObjects = Iterables.concat( diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 7d671deed7..310666778d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -73,7 +73,8 @@ enum Option { DETECT_CONTENT_TYPE("detectContentType"), ENABLE_OBJECT_RETENTION("enableObjectRetention"), RETURN_RAW_INPUT_STREAM("returnRawInputStream"), - OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"); + OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"), + INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes");; private final String value; From bd65ab03212f7bc98ed71fb2756dd2798105dd6d Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Tue, 12 Mar 2024 16:06:09 -0700 Subject: [PATCH 2/7] Add grpc and HNS --- .../java/com/google/cloud/storage/Bucket.java | 6 ++ .../com/google/cloud/storage/BucketInfo.java | 86 +++++++++++++++++++ .../google/cloud/storage/GrpcConversions.java | 26 ++++++ .../google/cloud/storage/JsonConversions.java | 25 ++++++ .../com/google/cloud/storage/Storage.java | 3 + .../com/google/cloud/storage/UnifiedOpts.java | 2 +- .../google/cloud/storage/it/ITBucketTest.java | 72 ++++++++++++++++ 7 files changed, 219 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java index 3c1652bc96..6459e0a0a1 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java @@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { + infoBuilder.setHierarchicalNamespace(hierarchicalNamespace); + return this; + } + @Override public Bucket build() { return new Bucket(storage, infoBuilder); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 9fb46b5bdb..6b17c050cf 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -118,6 +118,7 @@ public class BucketInfo implements Serializable { private final Logging logging; private final CustomPlacementConfig customPlacementConfig; private final ObjectRetention objectRetention; + private final HierarchicalNamespace hierarchicalNamespace; private final transient ImmutableSet modifiedFields; @@ -713,6 +714,70 @@ public Logging build() { } } + public static final class HierarchicalNamespace implements Serializable { + + private static final long serialVersionUID = 5932926691444613101L; + private Boolean enabled; + + public Boolean getEnabled() { + return enabled; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof HierarchicalNamespace)) { + return false; + } + HierarchicalNamespace that = (HierarchicalNamespace) o; + return Objects.equals(enabled, that.enabled); + } + + @Override + public int hashCode() { + return Objects.hash(enabled); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("enabled", enabled).toString(); + } + + private HierarchicalNamespace() {} + + private HierarchicalNamespace(Builder builder) { + this.enabled = builder.enabled; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder toBuilder() { + return newBuilder().setEnabled(enabled); + } + + public static final class Builder { + private Boolean enabled; + + /** + * Sets whether Hierarchical Namespace (Folders) is enabled for this bucket. This can only be + * enabled at bucket create time. If this is enabled, Uniform Bucket-Level Access must also be + * enabled. + */ + public Builder setEnabled(Boolean enabled) { + this.enabled = enabled; + return this; + } + + public HierarchicalNamespace build() { + return new HierarchicalNamespace(this); + } + } + } + /** * Lifecycle rule for a bucket. Allows supported Actions, such as deleting and changing storage * class, to be executed when certain Conditions are met. @@ -1683,6 +1748,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) { public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig); + public abstract Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace); + abstract Builder setObjectRetention(ObjectRetention objectRetention); /** Creates a {@code BucketInfo} object. */ @@ -1783,6 +1850,7 @@ static final class BuilderImpl extends Builder { private Logging logging; private CustomPlacementConfig customPlacementConfig; private ObjectRetention objectRetention; + private HierarchicalNamespace hierarchicalNamespace; private final ImmutableSet.Builder modifiedFields = ImmutableSet.builder(); BuilderImpl(String name) { @@ -1822,6 +1890,7 @@ static final class BuilderImpl extends Builder { logging = bucketInfo.logging; customPlacementConfig = bucketInfo.customPlacementConfig; objectRetention = bucketInfo.objectRetention; + hierarchicalNamespace = bucketInfo.hierarchicalNamespace; } @Override @@ -2187,6 +2256,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) { return this; } + @Override + public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) { + if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) { + modifiedFields.add(BucketField.HIERARCHICAL_NAMESPACE); + } + this.hierarchicalNamespace = hierarchicalNamespace; + return this; + } + @Override Builder setLocationType(String locationType) { if (!Objects.equals(this.locationType, locationType)) { @@ -2428,6 +2506,7 @@ private Builder clearDeleteLifecycleRules() { logging = builder.logging; customPlacementConfig = builder.customPlacementConfig; objectRetention = builder.objectRetention; + hierarchicalNamespace = builder.hierarchicalNamespace; modifiedFields = builder.modifiedFields.build(); } @@ -2768,6 +2847,11 @@ public ObjectRetention getObjectRetention() { return objectRetention; } + /** Returns the Hierarchical Namespace (Folders) Configuration */ + public HierarchicalNamespace getHierarchicalNamespace() { + return hierarchicalNamespace; + } + /** Returns a builder for the current bucket. */ public Builder toBuilder() { return new BuilderImpl(this); @@ -2805,6 +2889,7 @@ public int hashCode() { autoclass, locationType, objectRetention, + hierarchicalNamespace, logging); } @@ -2846,6 +2931,7 @@ public boolean equals(Object o) { && Objects.equals(autoclass, that.autoclass) && Objects.equals(locationType, that.locationType) && Objects.equals(objectRetention, that.objectRetention) + && Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace) && Objects.equals(logging, that.logging); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java index 9ddf229c5c..291505f61f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java @@ -110,6 +110,10 @@ final class GrpcConversions { private final Codec iamConditionCodec = Codec.of(this::conditionEncode, this::conditionDecode); + private final Codec + hierarchicalNamespaceCodec = + Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); + @VisibleForTesting final Codec timestampCodec = Codec.of( @@ -297,6 +301,10 @@ private BucketInfo bucketInfoDecode(Bucket from) { .setDataLocations(customPlacementConfig.getDataLocationsList()) .build()); } + if (from.hasHierarchicalNamespace()) { + to.setHierarchicalNamespace( + hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace())); + } // TODO(frankyn): Add SelfLink when the field is available if (!from.getEtag().isEmpty()) { to.setEtag(from.getEtag()); @@ -382,6 +390,10 @@ private Bucket bucketInfoEncode(BucketInfo from) { .addAllDataLocations(customPlacementConfig.getDataLocations()) .build()); } + ifNonNull( + from.getHierarchicalNamespace(), + hierarchicalNamespaceCodec::encode, + to::setHierarchicalNamespace); // TODO(frankyn): Add SelfLink when the field is available ifNonNull(from.getEtag(), to::setEtag); return to.build(); @@ -589,6 +601,20 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) { return to.build(); } + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( + BucketInfo.HierarchicalNamespace from) { + Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder(); + ifNonNull(from.getEnabled(), to::setEnabled); + return to.build(); + } + + private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode( + Bucket.HierarchicalNamespace from) { + BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); + to.setEnabled(from.getEnabled()); + return to.build(); + } + private Bucket.IamConfig iamConfigEncode(BucketInfo.IamConfiguration from) { Bucket.IamConfig.Builder to = Bucket.IamConfig.newBuilder(); to.setUniformBucketLevelAccess(ublaEncode(from)); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index 4bbf08d4b9..1a1a644afe 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -137,6 +137,10 @@ final class JsonConversions { private final Codec blobInfoCodec = Codec.of(this::blobInfoEncode, this::blobInfoDecode); + private final Codec + hierarchicalNamespaceCodec = + Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); + private final Codec notificationInfoCodec = Codec.of(this::notificationEncode, this::notificationDecode); private final Codec @@ -437,6 +441,10 @@ private Bucket bucketInfoEncode(BucketInfo from) { this::customPlacementConfigEncode, to::setCustomPlacementConfig); ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention); + ifNonNull( + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceEncode, + to::setHierarchicalNamespace); return to; } @@ -487,6 +495,10 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket from.getCustomPlacementConfig(), this::customPlacementConfigDecode, to::setCustomPlacementConfig); + ifNonNull( + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceDecode, + to::setHierarchicalNamespace); ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention); return to.build(); } @@ -861,6 +873,19 @@ private com.google.api.services.storage.model.Notification notificationEncode( return to; } + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(BucketInfo.HierarchicalNamespace from) { + Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace(); + ifNonNull(from.getEnabled(), to::setEnabled); + return to; + } + + private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(Bucket.HierarchicalNamespace from) { + BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); + to.setEnabled(from.getEnabled()); + return to.build(); + } + + private NotificationInfo notificationDecode( com.google.api.services.storage.model.Notification from) { NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 4d0c56b533..1d0ae83471 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -159,6 +159,9 @@ enum BucketField implements FieldSelector, NamedField { CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"), @TransportCompatibility({Transport.HTTP, Transport.GRPC}) AUTOCLASS("autoclass"), + + @TransportCompatibility({Transport.HTTP, Transport.GRPC}) + HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"), @TransportCompatibility({Transport.HTTP}) OBJECT_RETENTION("objectRetention"); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 9b97ca6fb0..7e568b0f59 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -651,7 +651,7 @@ private IncludeFoldersAsPrefixes(boolean val) { @Override public Mapper listObjects() { - return null;// return b -> b.setIncludeFoldersAsPrefixes(val); + return b -> b.setIncludeFoldersAsPrefixes(val); } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java index e32be5c32c..f1c86b837a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java @@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.paging.Page; +import com.google.api.services.storage.model.Folder; import com.google.cloud.Policy; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; @@ -44,6 +45,7 @@ import com.google.cloud.storage.Storage.BucketListOption; import com.google.cloud.storage.Storage.BucketTargetOption; import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TestUtils; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.StorageITRunner; @@ -53,6 +55,7 @@ import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.Inject; import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.cloud.storage.spi.v1.HttpStorageRpc; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.time.Duration; @@ -550,6 +553,75 @@ public void testUpdateBucket_noModification() throws Exception { assertThat(gen2).isEqualTo(gen1); } } + @Test + public void createBucketWithHierarchicalNamespace() { + String bucketName = generator.randomBucketName(); + storage.create( + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); + try { + Bucket remoteBucket = storage.get(bucketName); + assertNotNull(remoteBucket.getHierarchicalNamespace()); + assertTrue(remoteBucket.getHierarchicalNamespace().getEnabled()); + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } + @Test + public void testListObjectsWithFolders() throws Exception { + String bucketName = generator.randomBucketName(); + storage.create( + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); + try { + com.google.api.services.storage.Storage apiaryStorage = + new HttpStorageRpc(StorageOptions.getDefaultInstance()).getStorage(); + apiaryStorage + .folders() + .insert(bucketName, new Folder().setName("F").setBucket(bucketName)) + .execute(); + + Page blobs = + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(false)); + + boolean found = false; + for (Blob blob : blobs.iterateAll()) { + if (blob.getName().equals("F/")) { + found = true; + } + } + assert (!found); + + blobs = + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(true)); + + for (Blob blob : blobs.iterateAll()) { + if (blob.getName().equals("F/")) { + found = true; + } + } + assert (found); + + } finally { + BucketCleaner.doCleanup(bucketName, storage); + } + } private void unsetRequesterPays() { Bucket remoteBucket = From 7629afb49f533b237c003c815b5bfa88f5451359 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 12 Mar 2024 23:11:50 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../google/cloud/storage/GrpcConversions.java | 14 ++--- .../google/cloud/storage/JsonConversions.java | 21 +++---- .../com/google/cloud/storage/UnifiedOpts.java | 1 - .../cloud/storage/spi/v1/StorageRpc.java | 3 +- .../google/cloud/storage/it/ITBucketTest.java | 56 ++++++++++--------- 5 files changed, 50 insertions(+), 45 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java index 291505f61f..d1084bc418 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java @@ -111,7 +111,7 @@ final class GrpcConversions { Codec.of(this::conditionEncode, this::conditionDecode); private final Codec - hierarchicalNamespaceCodec = + hierarchicalNamespaceCodec = Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); @VisibleForTesting @@ -303,7 +303,7 @@ private BucketInfo bucketInfoDecode(Bucket from) { } if (from.hasHierarchicalNamespace()) { to.setHierarchicalNamespace( - hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace())); + hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace())); } // TODO(frankyn): Add SelfLink when the field is available if (!from.getEtag().isEmpty()) { @@ -391,9 +391,9 @@ private Bucket bucketInfoEncode(BucketInfo from) { .build()); } ifNonNull( - from.getHierarchicalNamespace(), - hierarchicalNamespaceCodec::encode, - to::setHierarchicalNamespace); + from.getHierarchicalNamespace(), + hierarchicalNamespaceCodec::encode, + to::setHierarchicalNamespace); // TODO(frankyn): Add SelfLink when the field is available ifNonNull(from.getEtag(), to::setEtag); return to.build(); @@ -602,14 +602,14 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) { } private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( - BucketInfo.HierarchicalNamespace from) { + BucketInfo.HierarchicalNamespace from) { Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder(); ifNonNull(from.getEnabled(), to::setEnabled); return to.build(); } private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode( - Bucket.HierarchicalNamespace from) { + Bucket.HierarchicalNamespace from) { BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); to.setEnabled(from.getEnabled()); return to.build(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java index 1a1a644afe..c5add241dd 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java @@ -138,7 +138,7 @@ final class JsonConversions { Codec.of(this::blobInfoEncode, this::blobInfoDecode); private final Codec - hierarchicalNamespaceCodec = + hierarchicalNamespaceCodec = Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode); private final Codec @@ -442,9 +442,9 @@ private Bucket bucketInfoEncode(BucketInfo from) { to::setCustomPlacementConfig); ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention); ifNonNull( - from.getHierarchicalNamespace(), - this::hierarchicalNamespaceEncode, - to::setHierarchicalNamespace); + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceEncode, + to::setHierarchicalNamespace); return to; } @@ -496,9 +496,9 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket this::customPlacementConfigDecode, to::setCustomPlacementConfig); ifNonNull( - from.getHierarchicalNamespace(), - this::hierarchicalNamespaceDecode, - to::setHierarchicalNamespace); + from.getHierarchicalNamespace(), + this::hierarchicalNamespaceDecode, + to::setHierarchicalNamespace); ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention); return to.build(); } @@ -873,19 +873,20 @@ private com.google.api.services.storage.model.Notification notificationEncode( return to; } - private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(BucketInfo.HierarchicalNamespace from) { + private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode( + BucketInfo.HierarchicalNamespace from) { Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace(); ifNonNull(from.getEnabled(), to::setEnabled); return to; } - private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(Bucket.HierarchicalNamespace from) { + private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode( + Bucket.HierarchicalNamespace from) { BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder(); to.setEnabled(from.getEnabled()); return to.build(); } - private NotificationInfo notificationDecode( com.google.api.services.storage.model.Notification from) { NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic()); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 7e568b0f59..28d4c58a17 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -374,7 +374,6 @@ static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersA return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes); } - @Deprecated static DetectContentType detectContentType() { return DetectContentType.INSTANCE; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 310666778d..3b40f6a235 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -74,7 +74,8 @@ enum Option { ENABLE_OBJECT_RETENTION("enableObjectRetention"), RETURN_RAW_INPUT_STREAM("returnRawInputStream"), OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"), - INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes");; + INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes"); + ; private final String value; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java index f1c86b837a..5b759be6b2 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java @@ -553,17 +553,19 @@ public void testUpdateBucket_noModification() throws Exception { assertThat(gen2).isEqualTo(gen1); } } + @Test public void createBucketWithHierarchicalNamespace() { String bucketName = generator.randomBucketName(); storage.create( - BucketInfo.newBuilder(bucketName) - .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) - .setIamConfiguration( - BucketInfo.IamConfiguration.newBuilder() - .setIsUniformBucketLevelAccessEnabled(true) - .build()) - .build()); + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace( + BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); try { Bucket remoteBucket = storage.get(bucketName); assertNotNull(remoteBucket.getHierarchicalNamespace()); @@ -572,30 +574,32 @@ public void createBucketWithHierarchicalNamespace() { BucketCleaner.doCleanup(bucketName, storage); } } + @Test public void testListObjectsWithFolders() throws Exception { String bucketName = generator.randomBucketName(); storage.create( - BucketInfo.newBuilder(bucketName) - .setHierarchicalNamespace(BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) - .setIamConfiguration( - BucketInfo.IamConfiguration.newBuilder() - .setIsUniformBucketLevelAccessEnabled(true) - .build()) - .build()); + BucketInfo.newBuilder(bucketName) + .setHierarchicalNamespace( + BucketInfo.HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + BucketInfo.IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build()); try { com.google.api.services.storage.Storage apiaryStorage = - new HttpStorageRpc(StorageOptions.getDefaultInstance()).getStorage(); + new HttpStorageRpc(StorageOptions.getDefaultInstance()).getStorage(); apiaryStorage - .folders() - .insert(bucketName, new Folder().setName("F").setBucket(bucketName)) - .execute(); + .folders() + .insert(bucketName, new Folder().setName("F").setBucket(bucketName)) + .execute(); Page blobs = - storage.list( - bucketName, - Storage.BlobListOption.delimiter("/"), - Storage.BlobListOption.includeFolders(false)); + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(false)); boolean found = false; for (Blob blob : blobs.iterateAll()) { @@ -606,10 +610,10 @@ public void testListObjectsWithFolders() throws Exception { assert (!found); blobs = - storage.list( - bucketName, - Storage.BlobListOption.delimiter("/"), - Storage.BlobListOption.includeFolders(true)); + storage.list( + bucketName, + Storage.BlobListOption.delimiter("/"), + Storage.BlobListOption.includeFolders(true)); for (Blob blob : blobs.iterateAll()) { if (blob.getName().equals("F/")) { From da0ea73c10f8b6db22705c2bba5d3d7be9ed379c Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Tue, 12 Mar 2024 16:38:06 -0700 Subject: [PATCH 4/7] fix tests --- .../src/main/java/com/google/cloud/storage/BucketInfo.java | 3 +++ .../java/com/google/cloud/storage/it/ITBucketReadMaskTest.java | 1 + .../com/google/cloud/storage/it/ITOptionRegressionTest.java | 1 + 3 files changed, 5 insertions(+) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 6b17c050cf..0164e96220 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -714,6 +714,9 @@ public Logging build() { } } + /** + * The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. + */ public static final class HierarchicalNamespace implements Serializable { private static final long serialVersionUID = 5932926691444613101L; diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java index 93099d41e9..9c97760321 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java @@ -133,6 +133,7 @@ public ImmutableList parameters() { new Args<>(BucketField.TIME_CREATED, LazyAssertion.equal()), new Args<>(BucketField.UPDATED, LazyAssertion.equal()), new Args<>(BucketField.VERSIONING, LazyAssertion.equal()), + new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal()), new Args<>(BucketField.WEBSITE, LazyAssertion.equal())); List argsDefined = diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java index 4dbe55adc2..0b8e2b025c 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java @@ -816,6 +816,7 @@ public void storage_BucketListOption_fields_BucketField() { "items/timeCreated", "items/updated", "items/versioning", + "items/hierarchicalNamespace", "items/website"); s.list(BucketListOption.fields(TestUtils.filterOutHttpOnlyBucketFields(BucketField.values()))); requestAuditing.assertQueryParam("fields", expected, splitOnCommaToSet()); From c78991cf77523d99b6f781342583ca38dfb9f34b Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 12 Mar 2024 23:41:20 +0000 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../src/main/java/com/google/cloud/storage/BucketInfo.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java index 0164e96220..af3e4436ea 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java @@ -714,9 +714,7 @@ public Logging build() { } } - /** - * The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. - */ + /** The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. */ public static final class HierarchicalNamespace implements Serializable { private static final long serialVersionUID = 5932926691444613101L; From 384c6a128822021131ecd9107be590f90a8151a2 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Wed, 13 Mar 2024 16:20:09 -0700 Subject: [PATCH 6/7] fix tests again --- .../java/com/google/cloud/storage/it/ITOptionRegressionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java index 0b8e2b025c..7627bf3af8 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java @@ -326,6 +326,7 @@ public void storage_BucketGetOption_fields_BucketField() { "timeCreated", "updated", "versioning", + "hierarchicalNamespace", "website"); s.get( b.getName(), From afc383aa00c27556321794e2008726e7ed5f7165 Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Thu, 14 Mar 2024 10:11:55 -0700 Subject: [PATCH 7/7] fix clirr --- google-cloud-storage/clirr-ignored-differences.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ba32338734..635e882a69 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -15,6 +15,12 @@ * writeAndClose(*) + + 7013 + com/google/cloud/storage/BucketInfo$Builder + com.google.cloud.storage.BucketInfo$Builder setHierarchicalNamespace(com.google.cloud.storage.BucketInfo$HierarchicalNamespace) + + 7013 com/google/cloud/storage/BlobInfo$Builder