diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f1e26ed87e58..bcb08f0c3d42 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_SERVER_DEFAULTS_VALIDITY_PERIOD_MS_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConsts.ETAG; import static org.apache.hadoop.ozone.OzoneConsts.MAXIMUM_NUMBER_OF_PARTS_PER_UPLOAD; import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MAXIMUM_ACCESS_ID_LENGTH; @@ -1722,8 +1723,10 @@ public List listKeys(String volumeName, String bucketName, key.getCreationTime(), key.getModificationTime(), key.getReplicationConfig(), + Collections.singletonMap(ETAG, key.getETag()), key.isFile(), - key.getOwnerName())) + key.getOwnerName(), + Collections.emptyMap())) .collect(Collectors.toList()); } else { List keys = ozoneManagerClient.listKeys( @@ -1735,8 +1738,10 @@ public List listKeys(String volumeName, String bucketName, key.getCreationTime(), key.getModificationTime(), key.getReplicationConfig(), + key.getMetadata(), key.isFile(), - key.getOwnerName())) + key.getOwnerName(), + key.getTags())) .collect(Collectors.toList()); } } diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index dc8f098286b3..cee69f0f3603 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -98,7 +98,6 @@ import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -373,9 +372,8 @@ public void testPutDoubleSlashPrefixObject() throws IOException { final String bucketName = getBucketName(); final String keyName = "//dir1"; final String content = "bar"; - OzoneConfiguration conf = cluster.getConf(); // Create a FSO bucket for test - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() @@ -502,8 +500,7 @@ public void testGetObjectWithoutETag() throws Exception { String value = "sample value"; byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8); - OzoneConfiguration conf = cluster.getConf(); - try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + try (OzoneClient ozoneClient = cluster.newClient()) { ObjectStore store = ozoneClient.getObjectStore(); OzoneVolume volume = store.getS3Volume(); @@ -532,46 +529,16 @@ public void testGetObjectWithoutETag() throws Exception { } @Test - public void testListObjectsMany() { - final String bucketName = getBucketName(); - s3Client.createBucket(bucketName); - final List keyNames = Arrays.asList( - getKeyName("1"), - getKeyName("2"), - getKeyName("3") - ); - - for (String keyName: keyNames) { - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); - } - - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(2); - ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(0, 2)); - assertTrue(listObjectsResponse.isTruncated()); - - - listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withMaxKeys(2) - .withMarker(listObjectsResponse.getNextMarker()); - listObjectsResponse = s3Client.listObjects(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(2, keyNames.size())); - assertFalse(listObjectsResponse.isTruncated()); + public void testListObjectsMany() throws Exception { + testListObjectsMany(false); } @Test - public void testListObjectsManyV2() { + public void testListObjectsManyV2() throws Exception { + testListObjectsMany(true); + } + + private void testListObjectsMany(boolean isListV2) throws Exception { final String bucketName = getBucketName(); s3Client.createBucket(bucketName); final List keyNames = Arrays.asList( @@ -579,34 +546,91 @@ public void testListObjectsManyV2() { getKeyName("2"), getKeyName("3") ); + final List keyNamesWithoutETag = Arrays.asList( + getKeyName("4"), + getKeyName("5") + ); + final Map keyToEtag = new HashMap<>(); for (String keyName: keyNames) { - s3Client.putObject(bucketName, keyName, RandomStringUtils.secure().nextAlphanumeric(5)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, + RandomStringUtils.secure().nextAlphanumeric(5)); + keyToEtag.put(keyName, putObjectResult.getETag()); } + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); - ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(2); - ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(2); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + + for (String keyNameWithoutETag : keyNamesWithoutETag) { + byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, + valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + Collections.emptyMap())) { + out.write(valueBytes); + } + } + } + + List objectSummaries; + String continuationToken; + if (isListV2) { + ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() + .withBucketName(bucketName) + .withMaxKeys(2); + ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.getNextContinuationToken(); + } else { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withMaxKeys(2); + ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.getNextMarker(); + } + assertThat(objectSummaries).hasSize(2); + assertEquals(objectSummaries.stream() .map(S3ObjectSummary::getKey).collect(Collectors.toList()), keyNames.subList(0, 2)); - assertTrue(listObjectsResponse.isTruncated()); + for (S3ObjectSummary objectSummary : objectSummaries) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } + // Include both keys with and without ETag + if (isListV2) { + ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() + .withBucketName(bucketName) + .withMaxKeys(5) + .withContinuationToken(continuationToken); + ListObjectsV2Result listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertFalse(listObjectsResponse.isTruncated()); + } else { + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withMaxKeys(5) + .withMarker(continuationToken); + ObjectListing listObjectsResponse = s3Client.listObjects(listObjectsRequest); + objectSummaries = listObjectsResponse.getObjectSummaries(); + assertEquals(bucketName, listObjectsResponse.getBucketName()); + assertFalse(listObjectsResponse.isTruncated()); + } - listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withMaxKeys(2) - .withContinuationToken(listObjectsResponse.getNextContinuationToken()); - listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); - assertThat(listObjectsResponse.getObjectSummaries()).hasSize(1); - assertEquals(bucketName, listObjectsResponse.getBucketName()); - assertEquals(listObjectsResponse.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey).collect(Collectors.toList()), - keyNames.subList(2, keyNames.size())); - assertFalse(listObjectsResponse.isTruncated()); + assertThat(objectSummaries).hasSize(3); + assertEquals(keyNames.get(2), objectSummaries.get(0).getKey()); + assertEquals(keyNamesWithoutETag.get(0), objectSummaries.get(1).getKey()); + assertEquals(keyNamesWithoutETag.get(1), objectSummaries.get(2).getKey()); + for (S3ObjectSummary objectSummary : objectSummaries) { + assertEquals(keyToEtag.get(objectSummary.getKey()), objectSummary.getETag()); + } } @Test @@ -961,7 +985,7 @@ private boolean isBucketEmpty(Bucket bucket) { } private String getBucketName() { - return getBucketName(null); + return getBucketName(""); } private String getBucketName(String suffix) { @@ -969,7 +993,7 @@ private String getBucketName(String suffix) { } private String getKeyName() { - return getKeyName(null); + return getKeyName(""); } private String getKeyName(String suffix) { diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 834580b6e8b7..7ef1342886c6 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -20,7 +20,9 @@ import static org.apache.hadoop.ozone.OzoneConsts.MB; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.calculateDigest; import static org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils.createFile; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -28,17 +30,29 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import javax.xml.bind.DatatypeConverter; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.hadoop.ozone.s3.S3GatewayService; import org.apache.ozone.test.OzoneTestBase; @@ -57,7 +71,12 @@ import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.Tag; import software.amazon.awssdk.services.s3.model.Tagging; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -133,6 +152,116 @@ public void testPutObject() { assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag()); } + @Test + public void testListObjectsMany() throws Exception { + testListObjectsMany(false); + } + + @Test + public void testListObjectsManyV2() throws Exception { + testListObjectsMany(true); + } + + private void testListObjectsMany(boolean isListV2) throws Exception { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + final List keyNames = Arrays.asList( + getKeyName("1"), + getKeyName("2"), + getKeyName("3") + ); + final List keyNamesWithoutETag = Arrays.asList( + getKeyName("4"), + getKeyName("5") + ); + final Map keyToEtag = new HashMap<>(); + for (String keyName: keyNames) { + PutObjectResponse putObjectResponse = s3Client.putObject(b -> b + .bucket(bucketName) + .key(keyName), + RequestBody.fromString(RandomStringUtils.secure().nextAlphanumeric(5))); + keyToEtag.put(keyName, putObjectResponse.eTag()); + } + try (OzoneClient ozoneClient = cluster.newClient()) { + ObjectStore store = ozoneClient.getObjectStore(); + + OzoneVolume volume = store.getS3Volume(); + OzoneBucket bucket = volume.getBucket(bucketName); + + for (String keyNameWithoutETag : keyNamesWithoutETag) { + byte[] valueBytes = RandomStringUtils.secure().nextAlphanumeric(5).getBytes(StandardCharsets.UTF_8); + try (OzoneOutputStream out = bucket.createKey(keyNameWithoutETag, + valueBytes.length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, ReplicationFactor.ONE), + Collections.emptyMap())) { + out.write(valueBytes); + } + } + } + + List s3Objects; + String continuationToken; + if (isListV2) { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(2) + .build(); + ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.nextContinuationToken(); + } else { + ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(bucketName) + .maxKeys(2) + .build(); + ListObjectsResponse listObjectsResponse = s3Client.listObjects(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertTrue(listObjectsResponse.isTruncated()); + continuationToken = listObjectsResponse.nextMarker(); + } + assertThat(s3Objects).hasSize(2); + assertEquals(s3Objects.stream() + .map(S3Object::key).collect(Collectors.toList()), + keyNames.subList(0, 2)); + for (S3Object s3Object : s3Objects) { + assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag()); + } + + // Include both keys with and without ETag + if (isListV2) { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(5) + .continuationToken(continuationToken) + .build(); + ListObjectsV2Response listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertFalse(listObjectsResponse.isTruncated()); + } else { + ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(bucketName) + .maxKeys(5) + .marker(continuationToken) + .build(); + ListObjectsResponse listObjectsResponse = s3Client.listObjects(listObjectsRequest); + s3Objects = listObjectsResponse.contents(); + assertEquals(bucketName, listObjectsResponse.name()); + assertFalse(listObjectsResponse.isTruncated()); + } + + assertThat(s3Objects).hasSize(3); + assertEquals(keyNames.get(2), s3Objects.get(0).key()); + assertEquals(keyNamesWithoutETag.get(0), s3Objects.get(1).key()); + assertEquals(keyNamesWithoutETag.get(1), s3Objects.get(2).key()); + for (S3Object s3Object : s3Objects) { + assertEquals(keyToEtag.get(s3Object.key()), s3Object.eTag()); + } + } + @Test public void testCopyObject() { final String sourceBucketName = getBucketName("source"); @@ -196,7 +325,7 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception } private String getBucketName() { - return getBucketName(null); + return getBucketName(""); } private String getBucketName(String suffix) { @@ -204,7 +333,7 @@ private String getBucketName(String suffix) { } private String getKeyName() { - return getKeyName(null); + return getKeyName(""); } private String getKeyName(String suffix) {