diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java index bb5ba27a28e5..ec42a0d7b4f1 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/S3SDKTestUtils.java @@ -20,10 +20,16 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; import java.security.MessageDigest; +import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.ozone.test.InputSubstream; @@ -94,4 +100,32 @@ public static String extractUploadId(String xml) { } return null; } + + /** + * Open an HttpURLConnection with the given parameters. + * + * @param url The URL to connect to. + * @param httpMethod The HTTP method to use (e.g., "GET", "PUT", "POST", etc.). + * @param headers A map of request headers to set. Can be null. + * @param body The request body as a byte array. Can be null. + * @return An open HttpURLConnection. + * @throws IOException If an I/O error occurs. + */ + public static HttpURLConnection openHttpURLConnection(URL url, String httpMethod, Map> headers, + byte[] body) throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod(httpMethod); + if (headers != null) { + headers.forEach((key, values) -> values.forEach(value -> connection.addRequestProperty(key, value))); + } + + if (body != null) { + connection.setDoOutput(true); + try (OutputStream os = connection.getOutputStream()) { + IOUtils.write(body, os); + os.flush(); + } + } + return connection; + } } diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 54bdddf6e1c4..3fe4f6ab5a11 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -77,7 +77,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; @@ -122,8 +121,11 @@ import org.apache.hadoop.ozone.s3.util.S3Consts; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.OzoneTestBase; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.io.TempDir; @@ -1059,196 +1061,159 @@ public void testQuotaExceeded() throws IOException { assertEquals("QuotaExceeded", ase.getErrorCode()); } - @Test - public void testPresignedUrlGet() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class PresignedUrlTests { + private static final String BUCKET_NAME = "presigned-url-bucket"; + private static final String CONTENT = "bar"; + // Set the presigned URL to expire after one hour. + private final Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + @BeforeAll + public void setup() { + s3Client.createBucket(BUCKET_NAME); + } - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + @Test + public void testPresignedUrlGet() throws IOException { + final String keyName = getKeyName(); - // Set the presigned URL to expire after one hour. - Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - // Generate the presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.GET) - .withExpiration(expiration); - generatePresignedUrlRequest.addRequestParameter("x-custom-parameter", "custom-value"); - URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Download the object using HttpUrlConnection (since v1.1) - // Capture the response body to a byte array. - URL presignedUrl = new URL(url.toExternalForm()); - HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("GET"); - // Download the result of executing the request. - try (InputStream s3is = connection.getInputStream(); - ByteArrayOutputStream bos = new ByteArrayOutputStream( - content.getBytes(StandardCharsets.UTF_8).length)) { - IOUtils.copy(s3is, bos); - assertEquals(content, bos.toString("UTF-8")); - } - } + InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); - @Test - public void testPresignedUrlHead() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(bucketName); + s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); - InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + // Generate the presigned URL + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.GET).withExpiration(expiration); + generatePresignedUrlRequest.addRequestParameter("x-custom-parameter", "custom-value"); + URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - // Set the presigned URL to expire after one hour. - Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - // Test HeadObject presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.HEAD) - .withExpiration(expiration); - URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - URL presignedUrl = new URL(url.toExternalForm()); - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("HEAD"); - - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "HeadObject presigned URL should return 200 OK"); - } finally { - if (connection != null) { - connection.disconnect(); + // Download the object using HttpUrlConnection (since v1.1) + // Capture the response body to a byte array. + HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); + // Download the result of executing the request. + try (InputStream s3is = connection.getInputStream(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) { + IOUtils.copy(s3is, bos); + assertEquals(CONTENT, bos.toString("UTF-8")); } } - // Test HeadBucket presigned URL - GeneratePresignedUrlRequest generateBucketPresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, null) - .withMethod(HttpMethod.HEAD) - .withExpiration(expiration); - URL bucketUrl = s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest); - - URL presignedBucketUrl = new URL(bucketUrl.toExternalForm()); - HttpURLConnection bucketConnection = null; - try { - bucketConnection = (HttpURLConnection) presignedBucketUrl.openConnection(); - bucketConnection.setRequestMethod("HEAD"); - - int bucketResponseCode = bucketConnection.getResponseCode(); - assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should return 200 OK"); - } finally { - if (bucketConnection != null) { - bucketConnection.disconnect(); + @Test + public void testPresignedUrlHead() throws IOException { + final String keyName = getKeyName(); + + InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8)); + s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); + + // Test HeadObject presigned URL + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.HEAD).withExpiration(expiration); + URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); + + HttpURLConnection connection = null; + try { + connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "HEAD", null, null); + int responseCode = connection.getResponseCode(); + assertEquals(200, responseCode, "HeadObject presigned URL should return 200 OK"); + } finally { + if (connection != null) { + connection.disconnect(); + } } - } - } - - @Test - public void testPresignedUrlPutObject() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String expectedContent = "bar"; - s3Client.createBucket(bucketName); - // Set the presigned URL to expire after one hour. - Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - // Test PutObjectRequest presigned URL - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.PUT) - .withExpiration(expiration); - URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("PUT"); - connection.setDoOutput(true); - try (OutputStream os = connection.getOutputStream()) { - os.write(expectedContent.getBytes(StandardCharsets.UTF_8)); - os.flush(); + // Test HeadBucket presigned URL + GeneratePresignedUrlRequest generateBucketPresignedUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, null).withMethod(HttpMethod.HEAD).withExpiration(expiration); + URL presignedBucketUrl = s3Client.generatePresignedUrl(generateBucketPresignedUrlRequest); + + HttpURLConnection bucketConnection = null; + try { + bucketConnection = S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null); + int bucketResponseCode = bucketConnection.getResponseCode(); + assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should return 200 OK"); + } finally { + if (bucketConnection != null) { + bucketConnection.disconnect(); + } } + } - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, "PutObject presigned URL should return 200 OK"); - String actualContent; - S3Object s3Object = s3Client.getObject(bucketName, keyName); - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - } - assertEquals(expectedContent, actualContent, "Downloaded content should match uploaded content"); - } finally { - if (connection != null) { - connection.disconnect(); + @Test + public void testPresignedUrlPutObject() throws Exception { + final String keyName = getKeyName(); + + // Test PutObjectRequest presigned URL + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration); + URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest); + + HttpURLConnection connection = null; + try { + connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", + null, CONTENT.getBytes(StandardCharsets.UTF_8)); + int responseCode = connection.getResponseCode(); + assertEquals(200, responseCode, "PutObject presigned URL should return 200 OK"); + String actualContent; + S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); + try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { + actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } + assertEquals(CONTENT, actualContent, "Downloaded content should match uploaded content"); + } finally { + if (connection != null) { + connection.disconnect(); + } } } - } - @Test - public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); - final Map tags = new HashMap<>(); - tags.put("tag1", "value1"); - tags.put("tag2", "value2"); - - s3Client.createBucket(bucketName); + @Test + public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { + final String keyName = getKeyName(); + final Map userMetadata = new HashMap<>(); + userMetadata.put("key1", "value1"); + userMetadata.put("key2", "value2"); + final Map tags = new HashMap<>(); + tags.put("tag1", "value1"); + tags.put("tag2", "value2"); - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - createFile(multipartUploadFile, (int) (10 * MB)); + File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (10 * MB)); - // create MPU using presigned URL - GeneratePresignedUrlRequest initMPUPresignedUrlRequest = - createInitMPUPresignedUrlRequest(bucketName, keyName, userMetadata, tags); + // create MPU using presigned URL + GeneratePresignedUrlRequest initMPUPresignedUrlRequest = + createInitMPUPresignedUrlRequest(keyName, userMetadata, tags); - String uploadId = initMultipartUpload(initMPUPresignedUrlRequest); + String uploadId = initMultipartUpload(initMPUPresignedUrlRequest); - // upload parts using presigned URL - List completedParts = uploadParts(multipartUploadFile, bucketName, keyName, uploadId); + // upload parts using presigned URL + List completedParts = uploadParts(multipartUploadFile, keyName, uploadId); - // Complete multipart upload using presigned URL - completeMPU(bucketName, keyName, uploadId, completedParts); + // Complete multipart upload using presigned URL + completeMPU(keyName, uploadId, completedParts); - // Verify upload result - ObjectMetadata objectMeta = s3Client.getObjectMetadata(bucketName, keyName); - assertEquals(userMetadata, objectMeta.getUserMetadata()); + // Verify upload result + ObjectMetadata objectMeta = s3Client.getObjectMetadata(BUCKET_NAME, keyName); + assertEquals(userMetadata, objectMeta.getUserMetadata()); - // Verify content - S3Object s3Object = s3Client.getObject(bucketName, keyName); - assertEquals(tags.size(), s3Object.getTaggingCount()); - String actualContent; - try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { - actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + // Verify content + S3Object s3Object = s3Client.getObject(BUCKET_NAME, keyName); + assertEquals(tags.size(), s3Object.getTaggingCount()); + String actualContent; + try (S3ObjectInputStream inputStream = s3Object.getObjectContent()) { + actualContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } + String expectedContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); + assertEquals(expectedContent, actualContent, "Downloaded content should match uploaded content"); } - String expectedContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); - assertEquals(expectedContent, actualContent, "Downloaded content should match uploaded content"); - } - private static void completeMPU(String bucketName, String keyName, String uploadId, List completedParts) - throws IOException { - GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.POST) - .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60)); - request.addRequestParameter("uploadId", uploadId); + private void completeMPU(String keyName, String uploadId, List completedParts) throws IOException { + GeneratePresignedUrlRequest request = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); + request.addRequestParameter("uploadId", uploadId); - HttpURLConnection httpConnection = null; - try { - httpConnection = (HttpURLConnection) s3Client.generatePresignedUrl(request).openConnection(); - httpConnection.setRequestMethod("POST"); - httpConnection.setDoOutput(true); + URL presignedUrl = s3Client.generatePresignedUrl(request); // Generate completion XML payload StringBuilder completionXml = new StringBuilder(); @@ -1262,154 +1227,150 @@ private static void completeMPU(String bucketName, String keyName, String upload completionXml.append(""); byte[] completionPayloadBytes = completionXml.toString().getBytes(StandardCharsets.UTF_8); - try (OutputStream os = httpConnection.getOutputStream()) { - IOUtils.write(completionPayloadBytes, os); - } - int responseCode = httpConnection.getResponseCode(); - assertEquals(200, responseCode, "Complete multipart upload should return 200 OK"); - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); + HttpURLConnection httpConnection = null; + try { + httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", null, completionPayloadBytes); + int responseCode = httpConnection.getResponseCode(); + assertEquals(200, responseCode, "Complete multipart upload should return 200 OK"); + } finally { + if (httpConnection != null) { + httpConnection.disconnect(); + } } } - } - private static List uploadParts(File multipartUploadFile, String bucketName, String keyName, - String uploadId) throws IOException { - List completedParts = new ArrayList<>(); - ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB)); - long filePosition = 0; - long fileLength = multipartUploadFile.length(); - int partNumber = 1; - - try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { - while (filePosition < fileLength) { - file.seek(filePosition); - long bytesRead = file.getChannel().read(byteBuffer); - byteBuffer.flip(); - - // generate presigned URL for each part - GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.PUT) - .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60)); - request.addRequestParameter("partNumber", String.valueOf(partNumber)); - request.addRequestParameter("uploadId", uploadId); - - URL presignedUrl = s3Client.generatePresignedUrl(request); - - // upload each part using presigned URL - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setDoOutput(true); - connection.setRequestMethod("PUT"); - connection.setRequestProperty("Content-Length", String.valueOf(byteBuffer.remaining())); - - try (OutputStream os = connection.getOutputStream()) { - os.write(byteBuffer.array(), 0, byteBuffer.remaining()); - os.flush(); + private List uploadParts(File multipartUploadFile, String keyName, String uploadId) throws IOException { + List completedParts = new ArrayList<>(); + ByteBuffer byteBuffer = ByteBuffer.allocate((int) (5 * MB)); + long filePosition = 0; + long fileLength = multipartUploadFile.length(); + int partNumber = 1; + + try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { + while (filePosition < fileLength) { + file.seek(filePosition); + long bytesRead = file.getChannel().read(byteBuffer); + byteBuffer.flip(); + + // generate presigned URL for each part + GeneratePresignedUrlRequest request = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT) + .withExpiration(expiration); + request.addRequestParameter("partNumber", String.valueOf(partNumber)); + request.addRequestParameter("uploadId", uploadId); + + URL presignedUrl = s3Client.generatePresignedUrl(request); + + // upload each part using presigned URL + HttpURLConnection connection = null; + try { + Map> headers = new HashMap<>(); + List header = Collections.singletonList(String.valueOf(byteBuffer.remaining())); + headers.put("Content-Length", header); + byte[] body = new byte[byteBuffer.remaining()]; + byteBuffer.get(body); + + connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT", headers, body); + int responseCode = connection.getResponseCode(); + assertEquals(200, responseCode, String.format("Upload part %d should return 200 OK", partNumber)); + + String etag = connection.getHeaderField("ETag"); + PartETag partETag = new PartETag(partNumber, etag); + completedParts.add(partETag); + } finally { + if (connection != null) { + connection.disconnect(); + } } - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, String.format("Upload part %d should return 200 OK", partNumber)); + byteBuffer.clear(); + filePosition += bytesRead; + partNumber++; + } + } + return completedParts; + } - String etag = connection.getHeaderField("ETag"); - PartETag partETag = new PartETag(partNumber, etag); - completedParts.add(partETag); - } finally { - if (connection != null) { - connection.disconnect(); + private String initMultipartUpload(GeneratePresignedUrlRequest request) throws IOException { + URL presignedUrl = s3Client.generatePresignedUrl(request); + String uploadId; + HttpURLConnection httpConnection = null; + try { + Map customRequestHeaders = request.getCustomRequestHeaders(); + Map> headers = new HashMap<>(); + if (customRequestHeaders != null) { + for (Map.Entry entry : customRequestHeaders.entrySet()) { + headers.put(entry.getKey(), Collections.singletonList(entry.getValue())); } } + httpConnection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "POST", headers, null); + int initMPUConnectionResponseCode = httpConnection.getResponseCode(); + assertEquals(200, initMPUConnectionResponseCode); - byteBuffer.clear(); - filePosition += bytesRead; - partNumber++; + try (InputStream is = httpConnection.getInputStream()) { + String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); + uploadId = S3SDKTestUtils.extractUploadId(responseXml); + } + } finally { + if (httpConnection != null) { + httpConnection.disconnect(); + } } + return uploadId; } - return completedParts; - } - private String initMultipartUpload(GeneratePresignedUrlRequest request) throws IOException { - URL presignedUrl = s3Client.generatePresignedUrl(request); - String uploadId; - HttpURLConnection httpConnection = null; - try { - httpConnection = (HttpURLConnection) presignedUrl.openConnection(); - httpConnection.setRequestMethod("POST"); - httpConnection.setDoOutput(true); - request.getCustomRequestHeaders().forEach(httpConnection::setRequestProperty); - - httpConnection.connect(); - int initMPUConnectionResponseCode = httpConnection.getResponseCode(); - assertEquals(200, initMPUConnectionResponseCode); - try (InputStream is = httpConnection.getInputStream()) { - String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); - uploadId = S3SDKTestUtils.extractUploadId(responseXml); - } - } finally { - if (httpConnection != null) { - httpConnection.disconnect(); + private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String keyName, + Map userMetadata, + Map tags) throws Exception { + GeneratePresignedUrlRequest initMPUPresignUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.POST).withExpiration(expiration); + + userMetadata.forEach((k, v) -> { + initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX + k, v); + }); + + StringBuilder tagValueBuilder = new StringBuilder(); + for (Map.Entry entry : tags.entrySet()) { + if (tagValueBuilder.length() > 0) { + tagValueBuilder.append('&'); + } + tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(), "UTF-8")); } + initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER, tagValueBuilder.toString()); + return initMPUPresignUrlRequest; } - return uploadId; - } - private GeneratePresignedUrlRequest createInitMPUPresignedUrlRequest(String bucketName, String keyName, - Map userMetadata, - Map tags) throws Exception { - GeneratePresignedUrlRequest initMPUPresignUrlRequest = new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.POST) - .withExpiration(new Date(System.currentTimeMillis() + 1000 * 60 * 60)); - - userMetadata.forEach((k, v) -> { - initMPUPresignUrlRequest.putCustomRequestHeader(CUSTOM_METADATA_HEADER_PREFIX + k, v); - }); - - StringBuilder tagValueBuilder = new StringBuilder(); - for (Map.Entry entry : tags.entrySet()) { - if (tagValueBuilder.length() > 0) { - tagValueBuilder.append('&'); + @Test + public void testPresignedUrlDelete() throws IOException { + final String keyName = getKeyName(); + + try (InputStream is = new ByteArrayInputStream(CONTENT.getBytes(StandardCharsets.UTF_8))) { + s3Client.putObject(BUCKET_NAME, keyName, is, new ObjectMetadata()); } - tagValueBuilder.append(entry.getKey()).append('=').append(URLEncoder.encode(entry.getValue(), "UTF-8")); - } - initMPUPresignUrlRequest.putCustomRequestHeader(S3Consts.TAG_HEADER, tagValueBuilder.toString()); - return initMPUPresignUrlRequest; - } - @Test - public void testPresignedUrlDelete() throws IOException { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; + // Generate the presigned URL for DELETE + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.DELETE) + .withExpiration(expiration); + URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); + + // Execute the DELETE request using HttpUrlConnection + HttpURLConnection connection = null; + try { + connection = S3SDKTestUtils.openHttpURLConnection(url, "DELETE", null, null); + int responseCode = connection.getResponseCode(); + // Verify the response code is 204 (No Content) + assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); + } finally { + if (connection != null) { + connection.disconnect(); + } + } - s3Client.createBucket(bucketName); - try (InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + // Verify the object is deleted + assertFalse(s3Client.doesObjectExist(BUCKET_NAME, keyName)); } - - // Set the presigned URL to expire after one hour. - Date expiration = Date.from(Instant.now().plusMillis(1000 * 60 * 60)); - - // Generate the presigned URL for DELETE - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, keyName) - .withMethod(HttpMethod.DELETE) - .withExpiration(expiration); - URL url = s3Client.generatePresignedUrl(generatePresignedUrlRequest); - - // Execute the DELETE request using HttpUrlConnection - URL presignedUrl = new URL(url.toExternalForm()); - HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("DELETE"); - int responseCode = connection.getResponseCode(); - - // Verify the response code is 204 (No Content) - assertEquals(HttpURLConnection.HTTP_NO_CONTENT, responseCode); - - // Verify the object is deleted - assertFalse(s3Client.doesObjectExist(bucketName, keyName)); } /** diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 31f58c6d9a29..79928c3264f8 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -33,7 +33,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; @@ -72,6 +71,7 @@ import org.apache.hadoop.ozone.s3.endpoint.S3Owner; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.OzoneTestBase; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Nested; @@ -456,26 +456,46 @@ public void testResumableDownloadWithEtagMismatch() throws Exception { } } - @Test - public void testPresignedUrlGet() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(b -> b.bucket(bucketName)); + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class PresignedUrlTests { + private static final String CONTENT = "bar"; + private static final String BUCKET_NAME = "presigned-url-bucket"; + private final SdkHttpClient sdkHttpClient = ApacheHttpClient.create(); + // The URL will expire in 10 minutes. + private final Duration duration = Duration.ofMinutes(10); + private S3Presigner presigner; - s3Client.putObject(b -> b - .bucket(bucketName) - .key(keyName), - RequestBody.fromString(content)); + @BeforeAll + public void setup() { + s3Client.createBucket(b -> b.bucket(BUCKET_NAME)); + presigner = S3Presigner.builder() + // TODO: Find a way to retrieve the path style configuration from S3Client instead + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) + .region(s3Client.serviceClientConfiguration().region()) + .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build(); + } - try (S3Presigner presigner = createS3Presigner()) { - GetObjectRequest objectRequest = GetObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); + @AfterAll + public void tearDown() { + presigner.close(); + sdkHttpClient.close(); + } + + @Test + public void testPresignedUrlGet() throws Exception { + final String keyName = getKeyName(); + + s3Client.putObject(b -> b + .bucket(BUCKET_NAME) + .key(keyName), + RequestBody.fromString(CONTENT)); + + GetObjectRequest objectRequest = GetObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build(); GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) // The URL will expire in 10 minutes. + .signatureDuration(duration) .getObjectRequest(objectRequest) .build(); @@ -484,66 +504,46 @@ public void testPresignedUrlGet() throws Exception { // Download the object using HttpUrlConnection (since v1.1) // Capture the response body to a byte array. URL presignedUrl = presignedRequest.url(); - HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("GET"); + HttpURLConnection connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "GET", null, null); // Download the result of executing the request. try (InputStream s3is = connection.getInputStream(); - ByteArrayOutputStream bos = new ByteArrayOutputStream( - content.getBytes(StandardCharsets.UTF_8).length)) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) { IoUtils.copy(s3is, bos); - assertEquals(content, bos.toString("UTF-8")); + assertEquals(CONTENT, bos.toString("UTF-8")); + } finally { + connection.disconnect(); } // Use the AWS SDK for Java SdkHttpClient class to do the download - SdkHttpRequest request = SdkHttpRequest.builder() - .method(SdkHttpMethod.GET) - .uri(presignedUrl.toURI()) - .build(); + SdkHttpRequest request = SdkHttpRequest.builder().method(SdkHttpMethod.GET).uri(presignedUrl.toURI()).build(); - HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() - .request(request) - .build(); + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder().request(request).build(); - try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create(); - ByteArrayOutputStream bos = new ByteArrayOutputStream( - content.getBytes(StandardCharsets.UTF_8).length)) { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(CONTENT.getBytes(StandardCharsets.UTF_8).length)) { HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); - assertTrue(response.responseBody().isPresent(), () -> "The presigned url download request " + - "should have a response body"); - response.responseBody().ifPresent( - abortableInputStream -> { - try { - IoUtils.copy(abortableInputStream, bos); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - assertEquals(content, bos.toString("UTF-8")); + assertTrue(response.responseBody().isPresent(), + () -> "The presigned url download request " + "should have a response body"); + response.responseBody().ifPresent(abortableInputStream -> { + try { + IoUtils.copy(abortableInputStream, bos); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + assertEquals(CONTENT, bos.toString("UTF-8")); } } - } - @Test - public void testPresignedUrlHead() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(b -> b.bucket(bucketName)); - - s3Client.putObject(b -> b - .bucket(bucketName) - .key(keyName), - RequestBody.fromString(content)); + @Test + public void testPresignedUrlHead() throws Exception { + final String keyName = getKeyName(); - try (S3Presigner presigner = createS3Presigner()) { + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); - HeadObjectRequest objectRequest = HeadObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); + HeadObjectRequest objectRequest = HeadObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build(); HeadObjectPresignRequest presignRequest = HeadObjectPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) + .signatureDuration(duration) .headObjectRequest(objectRequest) .build(); @@ -552,40 +552,29 @@ public void testPresignedUrlHead() throws Exception { URL presignedUrl = presignedRequest.url(); HttpURLConnection connection = null; try { - connection = (HttpURLConnection) presignedUrl.openConnection(); - connection.setRequestMethod("HEAD"); - + connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "HEAD", null, null); int responseCode = connection.getResponseCode(); assertEquals(200, responseCode, "HeadObject presigned URL should return 200 OK"); - - // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request - SdkHttpRequest request = SdkHttpRequest.builder() - .method(SdkHttpMethod.HEAD) - .uri(presignedUrl.toURI()) - .build(); - - HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() - .request(request) - .build(); - - try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) { - HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); - assertEquals(200, response.httpResponse().statusCode(), - "HeadObject presigned URL should return 200 OK via SdkHttpClient"); - } } finally { if (connection != null) { connection.disconnect(); } } + // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request + SdkHttpRequest request = SdkHttpRequest.builder().method(SdkHttpMethod.HEAD).uri(presignedUrl.toURI()).build(); + + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder().request(request).build(); + + HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); + assertEquals(200, response.httpResponse().statusCode(), + "HeadObject presigned URL should return 200 OK via SdkHttpClient"); + // Test HeadBucket presigned URL - HeadBucketRequest bucketRequest = HeadBucketRequest.builder() - .bucket(bucketName) - .build(); + HeadBucketRequest bucketRequest = HeadBucketRequest.builder().bucket(BUCKET_NAME).build(); HeadBucketPresignRequest headBucketPresignRequest = HeadBucketPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) + .signatureDuration(duration) .headBucketRequest(bucketRequest) .build(); @@ -594,50 +583,35 @@ public void testPresignedUrlHead() throws Exception { URL presignedBucketUrl = presignedBucketRequest.url(); HttpURLConnection bucketConnection = null; try { - bucketConnection = (HttpURLConnection) presignedBucketUrl.openConnection(); - bucketConnection.setRequestMethod("HEAD"); - + bucketConnection = S3SDKTestUtils.openHttpURLConnection(presignedBucketUrl, "HEAD", null, null); int bucketResponseCode = bucketConnection.getResponseCode(); assertEquals(200, bucketResponseCode, "HeadBucket presigned URL should return 200 OK"); - - // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request for bucket - SdkHttpRequest bucketSdkRequest = SdkHttpRequest.builder() - .method(SdkHttpMethod.HEAD) - .uri(presignedBucketUrl.toURI()) - .build(); - - HttpExecuteRequest bucketExecuteRequest = HttpExecuteRequest.builder() - .request(bucketSdkRequest) - .build(); - - try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) { - HttpExecuteResponse response = sdkHttpClient.prepareRequest(bucketExecuteRequest).call(); - assertEquals(200, response.httpResponse().statusCode(), - "HeadBucket presigned URL should return 200 OK via SdkHttpClient"); - } } finally { if (bucketConnection != null) { bucketConnection.disconnect(); } } - } - } - @Test - public void testPresignedUrlPut() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - s3Client.createBucket(b -> b.bucket(bucketName)); + // Use the AWS SDK for Java SdkHttpClient class to test the HEAD request for bucket + SdkHttpRequest bucketSdkRequest = SdkHttpRequest.builder() + .method(SdkHttpMethod.HEAD) + .uri(presignedBucketUrl.toURI()) + .build(); + HttpExecuteRequest bucketExecuteRequest = HttpExecuteRequest.builder().request(bucketSdkRequest).build(); - try (S3Presigner presigner = createS3Presigner()) { + HttpExecuteResponse bucketResponse = sdkHttpClient.prepareRequest(bucketExecuteRequest).call(); + assertEquals(200, bucketResponse.httpResponse().statusCode(), + "HeadBucket presigned URL should return 200 OK via SdkHttpClient"); + } - PutObjectRequest objectRequest = PutObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); + @Test + public void testPresignedUrlPut() throws Exception { + final String keyName = getKeyName(); + + PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build(); PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) + .signatureDuration(duration) .putObjectRequest(objectRequest) .build(); @@ -645,460 +619,371 @@ public void testPresignedUrlPut() throws Exception { // use http url connection HttpURLConnection connection = null; - String expectedContent; String actualContent; try { - expectedContent = "This is a test content for presigned PUT URL."; - connection = (HttpURLConnection) presignedRequest.url().openConnection(); - connection.setRequestMethod("PUT"); - connection.setDoOutput(true); - try (OutputStream os = connection.getOutputStream()) { - os.write(expectedContent.getBytes(StandardCharsets.UTF_8)); - os.flush(); - } - + connection = S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "PUT", + presignedRequest.signedHeaders(), CONTENT.getBytes(StandardCharsets.UTF_8)); int responseCode = connection.getResponseCode(); assertEquals(200, responseCode, "PutObject presigned URL should return 200 OK"); - - //verify the object was uploaded - ResponseInputStream object1 = s3Client.getObject(b1 -> b1.bucket(bucketName).key(keyName)); - actualContent = IoUtils.toUtf8String(object1); - assertEquals(expectedContent, actualContent); - - // Use the AWS SDK for Java SdkHttpClient class to test the PUT request - expectedContent = "This content is for testing the SdkHttpClient PUT request."; - SdkHttpRequest request = SdkHttpRequest.builder() - .method(SdkHttpMethod.PUT) - .uri(presignedRequest.url().toURI()) - .build(); - - byte[] bytes = expectedContent.getBytes(StandardCharsets.UTF_8); - HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() - .request(request) - .contentStreamProvider(() -> new ByteArrayInputStream(bytes)) - .build(); - - try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) { - HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); - assertEquals(200, response.httpResponse().statusCode(), - "PutObject presigned URL should return 200 OK via SdkHttpClient"); - } - - //verify the object was uploaded - ResponseInputStream object = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)); - actualContent = IoUtils.toUtf8String(object); - assertEquals(expectedContent, actualContent); } finally { if (connection != null) { connection.disconnect(); } } - } - } + //verify the object was uploaded + ResponseInputStream object1 = s3Client.getObject(b1 -> b1.bucket(BUCKET_NAME).key(keyName)); + actualContent = IoUtils.toUtf8String(object1); + assertEquals(CONTENT, actualContent); - @Test - public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final Map userMetadata = new HashMap<>(); - userMetadata.put("key1", "value1"); - userMetadata.put("key2", "value2"); + // Use the AWS SDK for Java SdkHttpClient class to test the PUT request + SdkHttpRequest request = SdkHttpRequest.builder() + .method(SdkHttpMethod.PUT) + .uri(presignedRequest.url().toURI()) + .build(); - List tags = Arrays.asList( - Tag.builder().key("tag1").value("value1").build(), - Tag.builder().key("tag2").value("value2").build() - ); + byte[] bytes = CONTENT.getBytes(StandardCharsets.UTF_8); + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() + .request(request) + .contentStreamProvider(() -> new ByteArrayInputStream(bytes)) + .build(); - s3Client.createBucket(b -> b.bucket(bucketName)); + HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); + assertEquals(200, response.httpResponse().statusCode(), + "PutObject presigned URL should return 200 OK via SdkHttpClient"); - File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); - createFile(multipartUploadFile, (int) (10 * MB)); + //verify the object was uploaded + ResponseInputStream object = s3Client.getObject(b -> b.bucket(BUCKET_NAME).key(keyName)); + actualContent = IoUtils.toUtf8String(object); + assertEquals(CONTENT, actualContent); + } + + @Test + public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception { + final String keyName = getKeyName(); + final Map userMetadata = new HashMap<>(); + userMetadata.put("key1", "value1"); + userMetadata.put("key2", "value2"); - try (S3Presigner presigner = createS3Presigner()) { + List tags = Arrays.asList(Tag.builder().key("tag1").value("value1").build(), + Tag.builder().key("tag2").value("value2").build()); + + File multipartUploadFile = Files.createFile(tempDir.resolve("multipartupload.txt")).toFile(); + createFile(multipartUploadFile, (int) (10 * MB)); // generate create MPU presigned URL CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder() - .bucket(bucketName) + .bucket(BUCKET_NAME) .key(keyName) .metadata(userMetadata) .tagging(Tagging.builder().tagSet(tags).build()) .build(); CreateMultipartUploadPresignRequest createMPUPresignRequest = CreateMultipartUploadPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) + .signatureDuration(duration) .createMultipartUploadRequest(createRequest) .build(); PresignedCreateMultipartUploadRequest presignCreateMultipartUpload = presigner.presignCreateMultipartUpload(createMPUPresignRequest); - mpuWithHttpURLConnection(presignCreateMultipartUpload, multipartUploadFile, bucketName, keyName, presigner, - userMetadata, tags); - mpuWithSdkHttpClient(presignCreateMultipartUpload, multipartUploadFile, bucketName, keyName, presigner, - userMetadata, tags); + mpuWithHttpURLConnection(presignCreateMultipartUpload, multipartUploadFile, keyName, userMetadata, tags); + mpuWithSdkHttpClient(presignCreateMultipartUpload, multipartUploadFile, keyName, userMetadata, tags); } - } - private void mpuWithHttpURLConnection(PresignedCreateMultipartUploadRequest presignCreateMultipartUpload, - File multipartUploadFile, String bucketName, String keyName, - S3Presigner presigner, Map userMetadata, - List tags) throws IOException { - // create MPU using presigned URL - String uploadId; - HttpURLConnection createMultiPartUploadConnection = null; - try { - createMultiPartUploadConnection = openHttpUrlConnection(presignCreateMultipartUpload); - int createMultiPartUploadConnectionResponseCode = createMultiPartUploadConnection.getResponseCode(); - assertEquals(200, createMultiPartUploadConnectionResponseCode, - "CreateMultipartUploadPresignRequest should return 200 OK"); - - try (InputStream is = createMultiPartUploadConnection.getInputStream()) { - String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); - uploadId = S3SDKTestUtils.extractUploadId(responseXml); - } - } finally { - if (createMultiPartUploadConnection != null) { - createMultiPartUploadConnection.disconnect(); + private void mpuWithHttpURLConnection(PresignedCreateMultipartUploadRequest presignCreateMultipartUpload, + File multipartUploadFile, String keyName, Map userMetadata, + List tags) throws IOException { + // create MPU using presigned URL + String uploadId; + HttpURLConnection createMPUConnection = null; + try { + createMPUConnection = S3SDKTestUtils.openHttpURLConnection(presignCreateMultipartUpload.url(), "POST", + presignCreateMultipartUpload.signedHeaders(), null); + int createMultiPartUploadConnectionResponseCode = createMPUConnection.getResponseCode(); + assertEquals(200, createMultiPartUploadConnectionResponseCode, + "CreateMultipartUploadPresignRequest should return 200 OK"); + + try (InputStream is = createMPUConnection.getInputStream()) { + String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); + uploadId = S3SDKTestUtils.extractUploadId(responseXml); + } + } finally { + if (createMPUConnection != null) { + createMPUConnection.disconnect(); + } } - } - - // Upload parts using presigned URL - List completedParts = - uploadPartWithHttpURLConnection(multipartUploadFile, bucketName, keyName, presigner, uploadId); - // complete MPU using presigned URL - CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) - .build(); - CompleteMultipartUploadPresignRequest completeMultipartUploadPresignRequest = - CompleteMultipartUploadPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) - .completeMultipartUploadRequest(completeRequest) - .build(); + // Upload parts using presigned URL + List completedParts = uploadPartWithHttpURLConnection(multipartUploadFile, keyName, uploadId); - PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = - presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest); + // complete MPU using presigned URL + CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() + .bucket(BUCKET_NAME) + .key(keyName) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) + .build(); + CompleteMultipartUploadPresignRequest completeMPUPresignRequest = CompleteMultipartUploadPresignRequest.builder() + .signatureDuration(duration) + .completeMultipartUploadRequest(completeRequest) + .build(); - completeMPUWithHttpUrlConnection(presignedCompleteMultipartUploadRequest, completedParts); + PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = + presigner.presignCompleteMultipartUpload(completeMPUPresignRequest); - // verify upload result - HeadObjectResponse headObjectResponse = s3Client.headObject(b -> b.bucket(bucketName).key(keyName)); - assertTrue(headObjectResponse.hasMetadata()); - assertEquals(userMetadata, headObjectResponse.metadata()); - - ResponseInputStream object = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)); - assertEquals(tags.size(), object.response().tagCount()); - String actualContent = IoUtils.toUtf8String(object); - String originalContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); - assertEquals(originalContent, actualContent, "Uploaded file content should match original file content"); - } + completeMPUWithHttpUrlConnection(presignedCompleteMultipartUploadRequest, completedParts); - private List uploadPartWithHttpURLConnection(File multipartUploadFile, String bucketName, - String keyName, S3Presigner presigner, String uploadId) - throws IOException { - List completedParts = new ArrayList<>(); - int partNumber = 1; - ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB)); + // verify upload result + HeadObjectResponse headObjectResponse = s3Client.headObject(b -> b.bucket(BUCKET_NAME).key(keyName)); + assertTrue(headObjectResponse.hasMetadata()); + assertEquals(userMetadata, headObjectResponse.metadata()); - try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { - long fileSize = file.length(); - long position = 0; + ResponseInputStream object = s3Client.getObject(b -> b.bucket(BUCKET_NAME).key(keyName)); + assertEquals(tags.size(), object.response().tagCount()); + String actualContent = IoUtils.toUtf8String(object); + String originalContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); + assertEquals(originalContent, actualContent, "Uploaded file content should match original file content"); + } - while (position < fileSize) { - file.seek(position); - long read = file.getChannel().read(bb); + private List uploadPartWithHttpURLConnection(File multipartUploadFile, String keyName, + String uploadId) throws IOException { + List completedParts = new ArrayList<>(); + int partNumber = 1; + ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB)); - bb.flip(); + try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { + long fileSize = file.length(); + long position = 0; - // First create an UploadPartRequest - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .partNumber(partNumber) - .contentLength((long) bb.remaining()) - .build(); + while (position < fileSize) { + file.seek(position); + long read = file.getChannel().read(bb); - // Generate presigned URL for each part - UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) - .uploadPartRequest(uploadPartRequest) - .build(); + bb.flip(); - PresignedUploadPartRequest presignedRequest = presigner.presignUploadPart(presignRequest); + // First create an UploadPartRequest + UploadPartRequest request = UploadPartRequest.builder() + .bucket(BUCKET_NAME) + .key(keyName) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength((long) bb.remaining()) + .build(); - // use presigned URL to upload the part - HttpURLConnection connection = null; - try { - connection = (HttpURLConnection) presignedRequest.url().openConnection(); - connection.setDoOutput(true); - connection.setRequestMethod("PUT"); + // Generate presigned URL for each part + UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder() + .signatureDuration(duration) + .uploadPartRequest(request) + .build(); - try (OutputStream os = connection.getOutputStream()) { - os.write(bb.array(), 0, bb.remaining()); - os.flush(); + PresignedUploadPartRequest presignedRequest = presigner.presignUploadPart(presignRequest); + + // use presigned URL to upload the part + HttpURLConnection connection = null; + try { + byte[] body = new byte[bb.remaining()]; + bb.get(body); + connection = S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "PUT", + presignedRequest.signedHeaders(), body); + + int responseCode = connection.getResponseCode(); + assertEquals(200, responseCode, String.format("Upload part %d should return 200 OK", partNumber)); + + String etag = connection.getHeaderField("ETag"); + CompletedPart part = CompletedPart.builder().partNumber(partNumber).eTag(etag).build(); + completedParts.add(part); + } finally { + if (connection != null) { + connection.disconnect(); + } } - int responseCode = connection.getResponseCode(); - assertEquals(200, responseCode, - String.format("Upload part %d should return 200 OK", partNumber)); - - String etag = connection.getHeaderField("ETag"); - CompletedPart part = CompletedPart.builder() - .partNumber(partNumber) - .eTag(etag) - .build(); - completedParts.add(part); - } finally { - if (connection != null) { - connection.disconnect(); - } + bb.clear(); + position += read; + partNumber++; } - - bb.clear(); - position += read; - partNumber++; } + return completedParts; } - return completedParts; - } - private HttpURLConnection openHttpUrlConnection(PresignedCreateMultipartUploadRequest request) throws IOException { - HttpURLConnection connection = (HttpURLConnection) request.url().openConnection(); - connection.setDoOutput(true); - connection.setRequestMethod("POST"); - for (Map.Entry> entry : request.signedHeaders().entrySet()) { - String key = entry.getKey(); - for (String value : entry.getValue()) { - connection.setRequestProperty(key, value); - } - } - return connection; - } + private void completeMPUWithHttpUrlConnection(PresignedCompleteMultipartUploadRequest request, + List completedParts) throws IOException { + HttpURLConnection connection = null; + try { + String xmlPayload = buildCompleteMultipartUploadXml(completedParts); + byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8); + connection = S3SDKTestUtils.openHttpURLConnection(request.url(), "POST", request.signedHeaders(), payloadBytes); - private void completeMPUWithHttpUrlConnection(PresignedCompleteMultipartUploadRequest request, - List completedParts) throws IOException { - HttpURLConnection completeMultipartUploadConnection = null; - try { - completeMultipartUploadConnection = (HttpURLConnection) request.url().openConnection(); - completeMultipartUploadConnection.setDoOutput(true); - completeMultipartUploadConnection.setRequestMethod("POST"); - - // copy headers - for (Map.Entry> entry : request.signedHeaders().entrySet()) { - String key = entry.getKey(); - for (String value : entry.getValue()) { - completeMultipartUploadConnection.setRequestProperty(key, value); + int responseCode = connection.getResponseCode(); + assertEquals(200, responseCode, "CompleteMultipartUploadPresignRequest should return 200 OK"); + } finally { + if (connection != null) { + connection.disconnect(); } } - - String xmlPayload = buildCompleteMultipartUploadXml(completedParts); - byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8); - try (OutputStream os = completeMultipartUploadConnection.getOutputStream()) { - IOUtils.write(payloadBytes, os); - os.flush(); - } - - int completeMultipartUploadConnectionResponseCode = completeMultipartUploadConnection.getResponseCode(); - assertEquals(200, completeMultipartUploadConnectionResponseCode, - "CompleteMultipartUploadPresignRequest should return 200 OK"); - } finally { - if (completeMultipartUploadConnection != null) { - completeMultipartUploadConnection.disconnect(); - } } - } - private void mpuWithSdkHttpClient(PresignedCreateMultipartUploadRequest presignCreateMultipartUpload, - File multipartUploadFile, String bucketName, String keyName, - S3Presigner presigner, Map userMetadata, List tags) - throws Exception { - // create MPU using presigned URL - String uploadId; - try (SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) { - uploadId = createMPUWithSdkHttpClient(presignCreateMultipartUpload, sdkHttpClient); + private void mpuWithSdkHttpClient(PresignedCreateMultipartUploadRequest presignCreateMultipartUpload, + File multipartUploadFile, String keyName, Map userMetadata, + List tags) throws Exception { + // create MPU using presigned URL + String uploadId = createMPUWithSdkHttpClient(presignCreateMultipartUpload); // Upload parts using presigned URL - List completedParts = - uploadPartWithSdkHttpClient(multipartUploadFile, bucketName, keyName, presigner, uploadId, sdkHttpClient); + List completedParts = uploadPartWithSdkHttpClient(multipartUploadFile, keyName, uploadId); // complete MPU using presigned URL - completeMPUWithSdkHttpClient(bucketName, keyName, presigner, uploadId, completedParts, sdkHttpClient); + completeMPUWithSdkHttpClient(keyName, uploadId, completedParts); // verify upload result - HeadObjectResponse headObjectResponse = s3Client.headObject(b -> b.bucket(bucketName).key(keyName)); + HeadObjectResponse headObjectResponse = s3Client.headObject(b -> b.bucket(BUCKET_NAME).key(keyName)); assertTrue(headObjectResponse.hasMetadata()); assertEquals(userMetadata, headObjectResponse.metadata()); - ResponseInputStream object = s3Client.getObject(b -> b.bucket(bucketName).key(keyName)); + ResponseInputStream object = s3Client.getObject(b -> b.bucket(BUCKET_NAME).key(keyName)); assertEquals(tags.size(), object.response().tagCount()); String actualContent = IoUtils.toUtf8String(object); String originalContent = new String(Files.readAllBytes(multipartUploadFile.toPath()), StandardCharsets.UTF_8); assertEquals(originalContent, actualContent, "Uploaded file content should match original file content"); } - } - private void completeMPUWithSdkHttpClient(String bucketName, String keyName, S3Presigner presigner, String uploadId, - List completedParts, SdkHttpClient sdkHttpClient) throws Exception { - CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) - .build(); + private void completeMPUWithSdkHttpClient(String keyName, String uploadId, List completedParts) + throws Exception { + CompleteMultipartUploadRequest request = CompleteMultipartUploadRequest.builder() + .bucket(BUCKET_NAME) + .key(keyName) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) + .build(); - CompleteMultipartUploadPresignRequest completeMultipartUploadPresignRequest = - CompleteMultipartUploadPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) - .completeMultipartUploadRequest(completeRequest) - .build(); + CompleteMultipartUploadPresignRequest presignRequest = CompleteMultipartUploadPresignRequest.builder() + .signatureDuration(duration) + .completeMultipartUploadRequest(request) + .build(); - PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = - presigner.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest); + PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = + presigner.presignCompleteMultipartUpload(presignRequest); - String xmlPayload = buildCompleteMultipartUploadXml(completedParts); - byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8); + String xmlPayload = buildCompleteMultipartUploadXml(completedParts); + byte[] payloadBytes = xmlPayload.getBytes(StandardCharsets.UTF_8); - SdkHttpRequest completeMultipartUploadRequest = SdkHttpRequest.builder() - .method(SdkHttpMethod.POST) - .uri(presignedCompleteMultipartUploadRequest.url().toURI()) - .build(); + SdkHttpRequest sdkHttpRequest = SdkHttpRequest.builder() + .method(SdkHttpMethod.POST) + .uri(presignedCompleteMultipartUploadRequest.url().toURI()) + .build(); - HttpExecuteRequest completeMultipartUploadExecuteRequest = HttpExecuteRequest.builder() - .request(completeMultipartUploadRequest) - .contentStreamProvider(() -> new ByteArrayInputStream(payloadBytes)) - .build(); + HttpExecuteRequest httpExecuteRequest = HttpExecuteRequest.builder() + .request(sdkHttpRequest) + .contentStreamProvider(() -> new ByteArrayInputStream(payloadBytes)) + .build(); - HttpExecuteResponse completeMultipartUploadResponse = - sdkHttpClient.prepareRequest(completeMultipartUploadExecuteRequest).call(); - assertEquals(200, completeMultipartUploadResponse.httpResponse().statusCode(), - "CompleteMultipartUploadPresignRequest should return 200 OK"); - } + HttpExecuteResponse response = sdkHttpClient.prepareRequest(httpExecuteRequest).call(); + assertEquals(200, response.httpResponse().statusCode(), + "CompleteMultipartUploadPresignRequest should return 200 OK"); + } - private List uploadPartWithSdkHttpClient(File multipartUploadFile, String bucketName, String keyName, - S3Presigner presigner, String uploadId, - SdkHttpClient httpClient) throws Exception { - List completedParts = new ArrayList<>(); - int partNumber = 1; - ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB)); + private List uploadPartWithSdkHttpClient(File multipartUploadFile, String keyName, String uploadId) + throws Exception { + List completedParts = new ArrayList<>(); + int partNumber = 1; + ByteBuffer bb = ByteBuffer.allocate((int) (5 * MB)); - try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { - long fileSize = file.length(); - long position = 0; + try (RandomAccessFile file = new RandomAccessFile(multipartUploadFile, "r")) { + long fileSize = file.length(); + long position = 0; - while (position < fileSize) { - file.seek(position); - long read = file.getChannel().read(bb); + while (position < fileSize) { + file.seek(position); + long read = file.getChannel().read(bb); - bb.flip(); + bb.flip(); - // Generate presigned URL for each part - UploadPartRequest uploadPartRequest = UploadPartRequest.builder() - .bucket(bucketName) - .key(keyName) - .uploadId(uploadId) - .partNumber(partNumber) - .contentLength((long) bb.remaining()) - .build(); + // Generate presigned URL for each part + UploadPartRequest request = UploadPartRequest.builder() + .bucket(BUCKET_NAME) + .key(keyName) + .uploadId(uploadId) + .partNumber(partNumber) + .contentLength((long) bb.remaining()) + .build(); - UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) - .uploadPartRequest(uploadPartRequest) - .build(); + UploadPartPresignRequest presignRequest = UploadPartPresignRequest.builder() + .signatureDuration(duration) + .uploadPartRequest(request) + .build(); - PresignedUploadPartRequest presignedRequest = presigner.presignUploadPart(presignRequest); + PresignedUploadPartRequest presignedRequest = presigner.presignUploadPart(presignRequest); - // upload each part using presigned URL - SdkHttpRequest uploadPartSdkRequest = SdkHttpRequest.builder() - .method(SdkHttpMethod.PUT) - .uri(presignedRequest.url().toURI()) - .build(); + // upload each part using presigned URL + SdkHttpRequest uploadPartSdkRequest = SdkHttpRequest.builder() + .method(SdkHttpMethod.PUT) + .uri(presignedRequest.url().toURI()) + .build(); - byte[] bytes = new byte[bb.remaining()]; - bb.get(bytes); + byte[] bytes = new byte[bb.remaining()]; + bb.get(bytes); - HttpExecuteRequest uploadPartExecuteRequest = HttpExecuteRequest.builder() - .request(uploadPartSdkRequest) - .contentStreamProvider(() -> new ByteArrayInputStream(bytes)) - .build(); + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() + .request(uploadPartSdkRequest) + .contentStreamProvider(() -> new ByteArrayInputStream(bytes)) + .build(); - HttpExecuteResponse uploadPartResponse = httpClient.prepareRequest(uploadPartExecuteRequest).call(); + HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); - String etag = uploadPartResponse.httpResponse() - .firstMatchingHeader("ETag") - .orElseThrow(() -> new RuntimeException("ETag missing in response")); + String etag = response.httpResponse().firstMatchingHeader("ETag") + .orElseThrow(() -> new RuntimeException("ETag missing in response")); - CompletedPart part = CompletedPart.builder() - .partNumber(partNumber) - .eTag(etag) - .build(); - completedParts.add(part); + CompletedPart part = CompletedPart.builder().partNumber(partNumber).eTag(etag).build(); + completedParts.add(part); - bb.clear(); - position += read; - partNumber++; + bb.clear(); + position += read; + partNumber++; + } } + return completedParts; } - return completedParts; - } - - private String createMPUWithSdkHttpClient(PresignedCreateMultipartUploadRequest request, - SdkHttpClient httpClient) throws Exception { - String uploadId; - SdkHttpRequest createMultipartUploadRequest = SdkHttpRequest.builder() - .method(SdkHttpMethod.POST) - .uri(request.url().toURI()) - .headers(request.signedHeaders()) - .build(); - HttpExecuteRequest createMultipartUploadExecuteRequest = HttpExecuteRequest.builder() - .request(createMultipartUploadRequest) - .build(); + private String createMPUWithSdkHttpClient(PresignedCreateMultipartUploadRequest request) throws Exception { + String uploadId; + SdkHttpRequest sdkHttpRequest = SdkHttpRequest.builder() + .method(SdkHttpMethod.POST) + .uri(request.url().toURI()) + .headers(request.signedHeaders()) + .build(); - HttpExecuteResponse createMultipartUploadResponse = - httpClient.prepareRequest(createMultipartUploadExecuteRequest).call(); + HttpExecuteRequest httpExecuteRequest = HttpExecuteRequest.builder() + .request(sdkHttpRequest) + .build(); - try (InputStream is = createMultipartUploadResponse.responseBody().get()) { - String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); - uploadId = S3SDKTestUtils.extractUploadId(responseXml); + HttpExecuteResponse response = sdkHttpClient.prepareRequest(httpExecuteRequest).call(); + try (InputStream is = response.responseBody().get()) { + String responseXml = IOUtils.toString(is, StandardCharsets.UTF_8); + uploadId = S3SDKTestUtils.extractUploadId(responseXml); + } + return uploadId; } - return uploadId; - } - private String buildCompleteMultipartUploadXml(List parts) { - StringBuilder xml = new StringBuilder(); - xml.append("\n"); - for (CompletedPart part : parts) { - xml.append(" \n"); - xml.append(" ").append(part.partNumber()).append("\n"); - xml.append(" ").append(part.eTag()).append("\n"); - xml.append(" \n"); + private String buildCompleteMultipartUploadXml(List parts) { + StringBuilder xml = new StringBuilder(); + xml.append("\n"); + for (CompletedPart part : parts) { + xml.append(" \n"); + xml.append(" ").append(part.partNumber()).append("\n"); + xml.append(" ").append(part.eTag()).append("\n"); + xml.append(" \n"); + } + xml.append(""); + return xml.toString(); } - xml.append(""); - return xml.toString(); - } - @Test - public void testPresignedUrlDelete() throws Exception { - final String bucketName = getBucketName(); - final String keyName = getKeyName(); - final String content = "bar"; - s3Client.createBucket(b -> b.bucket(bucketName)); - - s3Client.putObject(b -> b - .bucket(bucketName) - .key(keyName), - RequestBody.fromString(content)); + @Test + public void testPresignedUrlDelete() throws Exception { + final String keyName = getKeyName(); - try (S3Presigner presigner = createS3Presigner()) { + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); - DeleteObjectRequest objectRequest = DeleteObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); + DeleteObjectRequest objectRequest = DeleteObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build(); DeleteObjectPresignRequest presignRequest = DeleteObjectPresignRequest.builder() .signatureDuration(Duration.ofMinutes(10)) @@ -1110,69 +995,37 @@ public void testPresignedUrlDelete() throws Exception { // use http url connection HttpURLConnection connection = null; try { - connection = (HttpURLConnection) presignedRequest.url().openConnection(); - connection.setRequestMethod("DELETE"); - + connection = S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "DELETE", null, null); int responseCode = connection.getResponseCode(); assertEquals(204, responseCode, "DeleteObject presigned URL should return 204 No Content"); //verify the object was deleted - assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b -> b.bucket(bucketName).key(keyName))); + assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b -> b.bucket(BUCKET_NAME).key(keyName))); } finally { if (connection != null) { connection.disconnect(); } } - } - // use SdkHttpClient - s3Client.putObject(b -> b - .bucket(bucketName) - .key(keyName), - RequestBody.fromString(content)); - - try (S3Presigner presigner = createS3Presigner(); - SdkHttpClient sdkHttpClient = ApacheHttpClient.create()) { - - DeleteObjectRequest objectRequest = DeleteObjectRequest.builder() - .bucket(bucketName) - .key(keyName) - .build(); - - DeleteObjectPresignRequest presignRequest = DeleteObjectPresignRequest.builder() - .signatureDuration(Duration.ofMinutes(10)) - .deleteObjectRequest(objectRequest) - .build(); - - PresignedDeleteObjectRequest presignedRequest = presigner.presignDeleteObject(presignRequest); + // use SdkHttpClient + s3Client.putObject(b -> b.bucket(BUCKET_NAME).key(keyName), RequestBody.fromString(CONTENT)); SdkHttpRequest request = SdkHttpRequest.builder() .method(SdkHttpMethod.DELETE) .uri(presignedRequest.url().toURI()) .build(); - HttpExecuteRequest executeRequest = HttpExecuteRequest.builder() - .request(request) - .build(); + HttpExecuteRequest executeRequest = HttpExecuteRequest.builder().request(request).build(); HttpExecuteResponse response = sdkHttpClient.prepareRequest(executeRequest).call(); assertEquals(204, response.httpResponse().statusCode(), "DeleteObject presigned URL should return 204 No Content via SdkHttpClient"); //verify the object was deleted - assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b -> b.bucket(bucketName).key(keyName))); + assertThrows(NoSuchKeyException.class, () -> s3Client.getObject(b -> b.bucket(BUCKET_NAME).key(keyName))); } } - private S3Presigner createS3Presigner() { - return S3Presigner.builder() - // TODO: Find a way to retrieve the path style configuration from S3Client instead - .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) - .endpointOverride(s3Client.serviceClientConfiguration().endpointOverride().get()) - .region(s3Client.serviceClientConfiguration().region()) - .credentialsProvider(s3Client.serviceClientConfiguration().credentialsProvider()).build(); - } - /** * Tests the functionality to create a snapshot of an Ozone bucket and then read files * from the snapshot directory using the S3 SDK.