From 3826f97eaa8bb94dcf1204ad13755f067085aef3 Mon Sep 17 00:00:00 2001 From: Ravi Singh Date: Sat, 20 Aug 2022 09:34:03 +0530 Subject: [PATCH] Updating default value of max list items per call (#856) * Updating default value of max list items per call * fixed integration tests * fixed integration tests * addressed review comments * fixed formatting * addressed review comments * Update CHANGES.md Co-authored-by: Igor Dvorzhak --- gcs/CHANGES.md | 2 + gcs/CONFIGURATION.md | 2 +- ...ogleHadoopFileSystemConfigurationTest.java | 2 +- .../gcsio/GoogleCloudStorageOptions.java | 2 +- ...udStorageFileSystemNewIntegrationTest.java | 45 +++---- .../GoogleCloudStorageNewIntegrationTest.java | 110 ++++++------------ .../hadoop/gcsio/GoogleCloudStorageTest.java | 91 ++++++--------- .../gcsio/TrackingHttpRequestInitializer.java | 40 ++++++- 8 files changed, 128 insertions(+), 166 deletions(-) diff --git a/gcs/CHANGES.md b/gcs/CHANGES.md index 95a1aa625a..e882c7b3fd 100644 --- a/gcs/CHANGES.md +++ b/gcs/CHANGES.md @@ -72,6 +72,8 @@ fs.gs.outputstream.type ``` +1. Set default value for `fs.gs.list.max.items.per.call` property to `5000`. + ### 2.2.2 - 2021-06-25 1. Support footer prefetch in gRPC read channel. diff --git a/gcs/CONFIGURATION.md b/gcs/CONFIGURATION.md index 656cb5f3b8..04ae6f05a7 100644 --- a/gcs/CONFIGURATION.md +++ b/gcs/CONFIGURATION.md @@ -102,7 +102,7 @@ Maximum number of threads used to execute batch requests in parallel. -* `fs.gs.list.max.items.per.call` (default: `1024`) +* `fs.gs.list.max.items.per.call` (default: `5000`) Maximum number of items to return in response for list Cloud Storage requests. diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemConfigurationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemConfigurationTest.java index 323aad90e6..233078d2fe 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemConfigurationTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemConfigurationTest.java @@ -102,7 +102,7 @@ public class GoogleHadoopFileSystemConfigurationTest { put("fs.gs.inputstream.support.gzip.encoding.enable", false); put("fs.gs.io.buffersize.write", 64 * 1024 * 1024); put("fs.gs.lazy.init.enable", false); - put("fs.gs.list.max.items.per.call", 1024L); + put("fs.gs.list.max.items.per.call", 5000L); put("fs.gs.marker.file.pattern", null); put("fs.gs.max.requests.per.batch", 15L); put("fs.gs.max.wait.for.empty.object.creation.ms", 3_000); diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageOptions.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageOptions.java index 9a1b5c38b2..1be4734d08 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageOptions.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageOptions.java @@ -66,7 +66,7 @@ public abstract class GoogleCloudStorageOptions { public static final int MAX_WAIT_MILLIS_FOR_EMPTY_OBJECT_CREATION = 3_000; /** Default number of items to return per call to the list* GCS RPCs. */ - public static final long MAX_LIST_ITEMS_PER_CALL_DEFAULT = 1024; + public static final long MAX_LIST_ITEMS_PER_CALL_DEFAULT = 5000; /** Default setting for maximum number of requests per GCS batch. */ public static final long MAX_REQUESTS_PER_BATCH_DEFAULT = 15; diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemNewIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemNewIntegrationTest.java index 8466002a91..ef08cabefb 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemNewIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemNewIntegrationTest.java @@ -590,8 +590,7 @@ public void listFileInfo_file() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( getRequestString(bucketName, fileObject), - listRequestWithTrailingDelimiter( - bucketName, fileObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, fileObject + "/", /* pageToken= */ null)); assertThat(fileInfos).hasSize(1); FileInfo fileInfo = fileInfos.get(0); @@ -616,8 +615,7 @@ public void listFileInfo_file_directoryPath() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - bucketName, fileObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, fileObject + "/", /* pageToken= */ null)); } @Test @@ -637,8 +635,7 @@ public void listFileInfo_directory() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( getRequestString(bucketName, dirObject), - listRequestWithTrailingDelimiter( - bucketName, dirObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, dirObject + "/", /* pageToken= */ null)); assertThat(fileInfos.stream().map(FileInfo::exists).collect(toList())) .containsExactly(true, true); @@ -686,8 +683,7 @@ public void listFileInfo_directory_directoryPath() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - bucketName, dirObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, dirObject + "/", /* pageToken= */ null)); assertThat(fileInfos.stream().map(FileInfo::exists).collect(toList())) .containsExactly(true, true); @@ -713,8 +709,7 @@ public void listFileInfo_implicitDirectory() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( getRequestString(bucketName, dirObject), - listRequestWithTrailingDelimiter( - bucketName, dirObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, dirObject + "/", /* pageToken= */ null)); assertThat(fileInfos.stream().map(FileInfo::exists).collect(toList())).containsExactly(true); assertThat(fileInfos.stream().map(FileInfo::getPath).collect(toList())) @@ -738,8 +733,7 @@ public void listFileInfo_emptyDirectoryObject() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( getRequestString(bucketName, dirObject), - listRequestWithTrailingDelimiter( - bucketName, dirObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, dirObject + "/", /* pageToken= */ null)); assertThat(fileInfos).isEmpty(); } @@ -760,8 +754,7 @@ public void listFileInfo_notFound() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( getRequestString(bucketName, dirObject), - listRequestWithTrailingDelimiter( - bucketName, dirObject + "/", /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucketName, dirObject + "/", /* pageToken= */ null)); } @Test @@ -805,7 +798,7 @@ public void listFileInfo_customFields_required() throws Exception { bucketName, dirObject + "/", /* objectFields= */ "bucket,name", - /* maxResults= */ 1024, + /* pageToken= */ null)); } @@ -850,7 +843,7 @@ public void listFileInfo_customFields_some() throws Exception { bucketName, dirObject + "/", /* objectFields= */ "bucket,name,contentType", - /* maxResults= */ 1024, + /* pageToken= */ null)); } @@ -880,11 +873,7 @@ public void listFileInfo_customFields_fails() throws Exception { .containsExactly( getRequestString(bucketName, dirObject), listRequestWithTrailingDelimiter( - bucketName, - dirObject + "/", - /* objectFields= */ "bucket", - /* maxResults= */ 1024, - /* pageToken= */ null)); + bucketName, dirObject + "/", /* objectFields= */ "bucket", /* pageToken= */ null)); } @Test @@ -975,13 +964,7 @@ private void delete_directory(boolean parallelStatus) throws Exception { listRequestWithTrailingDelimiter( bucketName, dirObject + "/d1/", /* maxResults= */ 1, /* pageToken= */ null), listRequestString( - bucketName, - /* flatList= */ true, - /* includeTrailingDelimiter= */ null, - dirObject + "/d1/", - "bucket,name,generation", - /* maxResults= */ 1024, - /* pageToken= */ null), + bucketName, true, null, dirObject + "/d1/", "bucket,name,generation", null), deleteRequestString(bucketName, dirObject + "/d1/", /* generationId= */ 1)); assertThat(gcsFs.exists(bucketUri.resolve(dirObject + "/d1"))).isFalse(); @@ -1015,7 +998,7 @@ public void delete_inferredDirectory() throws Exception { /* includeTrailingDelimiter= */ null, dirObject + "/d1/", "bucket,name,generation", - /* maxResults= */ 1024, + /* pageToken= */ null), deleteRequestString(bucketName, dirObject + "/d1/f1", /* generationId= */ 1), getRequestString(bucketName, dirObject + "/")); @@ -1190,7 +1173,7 @@ public void rename_directory_sequential() throws Exception { /* includeTrailingDelimiter= */ null, dirObject + "/srcParent/srcDir/", "bucket,name,generation", - /* maxResults= */ 1024, + /* pageToken= */ null), // Copy file copyRequestString( @@ -1259,7 +1242,7 @@ public void rename_directory_parallel() throws Exception { /* includeTrailingDelimiter= */ null, dirObject + "/srcParent/srcDir/", "bucket,name,generation", - /* maxResults= */ 1024, + /* pageToken= */ null), // Copy file copyRequestString( diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageNewIntegrationTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageNewIntegrationTest.java index ef6ca2208a..52c3eb9456 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageNewIntegrationTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageNewIntegrationTest.java @@ -112,8 +112,7 @@ public void listObjectInfo_nonExistentBucket_nullPrefix() throws Exception { list_nonExistentBucket_nullPrefix( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -123,8 +122,7 @@ public void listObjectInfoPage_nonExistentBucket_nullPrefix() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_nonExistentBucket_nullPrefix( @@ -149,8 +147,7 @@ public void listObjectInfo_emptyBucket_nullPrefix() throws Exception { list_emptyBucket_nullPrefix( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -160,8 +157,7 @@ public void listObjectInfoPage_emptyBucket_nullPrefix() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_emptyBucket_nullPrefix( @@ -186,8 +182,7 @@ public void listObjectInfo_nonEmptyBucket_nullPrefix_object() throws Exception { list_nonEmptyBucket_nullPrefix_object( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -197,8 +192,7 @@ public void listObjectInfoPage_nonEmptyBucket_nullPrefix_object() throws Excepti getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_nonEmptyBucket_nullPrefix_object( @@ -224,8 +218,7 @@ public void listObjectInfo_nonEmptyBucket_nullPrefix_emptyDir() throws Exception list_nonEmptyBucket_nullPrefix_emptyDir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -235,8 +228,7 @@ public void listObjectInfoPage_nonEmptyBucket_nullPrefix_emptyDir() throws Excep getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_nonEmptyBucket_nullPrefix_emptyDir( @@ -262,8 +254,7 @@ public void listObjectInfo_nonEmptyBucket_nullPrefix_subDir() throws Exception { list_nonEmptyBucket_nullPrefix_subDir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -273,8 +264,7 @@ public void listObjectInfoPage_nonEmptyBucket_nullPrefix_subDir() throws Excepti getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_nonEmptyBucket_nullPrefix_subDir( @@ -300,8 +290,7 @@ public void listObjectInfo_nonEmptyBucket_nullPrefix_implicitSubDir() throws Exc list_nonEmptyBucket_nullPrefix_implicitSubDir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -311,8 +300,7 @@ public void listObjectInfoPage_nonEmptyBucket_nullPrefix_implicitSubDir() throws getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_nonEmptyBucket_nullPrefix_implicitSubDir( @@ -338,8 +326,7 @@ public void listObjectInfo_prefix_doesNotExist() throws Exception { list_prefix_doesNotExist( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -349,8 +336,7 @@ public void listObjectInfoPage_prefix_doesNotExist() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_prefix_doesNotExist( @@ -376,8 +362,7 @@ public void listObjectInfo_prefixObject_empty() throws Exception { list_prefixObject_empty( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -387,8 +372,7 @@ public void listObjectInfoPage_prefixObject_empty() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_prefixObject_empty( @@ -415,8 +399,7 @@ public void listObjectInfo_prefixObject_withObject() throws Exception { list_prefixObject_withObject( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -426,8 +409,7 @@ public void listObjectInfoPage_prefixObject_withObject() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_prefixObject_withObject( @@ -453,8 +435,7 @@ public void listObjectInfo_prefixObject_withSubdir() throws Exception { list_prefixObject_withSubdir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -464,8 +445,7 @@ public void listObjectInfoPage_prefixObject_withSubdir() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_prefixObject_withSubdir( @@ -491,8 +471,7 @@ public void listObjectInfo_prefixObject_withImplicitSubdir() throws Exception { list_prefixObject_withImplicitSubdir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -502,8 +481,7 @@ public void listObjectInfoPage_prefixObject_withImplicitSubdir() throws Exceptio getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_prefixObject_withImplicitSubdir( @@ -530,8 +508,7 @@ public void listObjectInfo_implicitPrefix_withObject() throws Exception { list_implicitPrefix_withObject( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -541,8 +518,7 @@ public void listObjectInfoPage_implicitPrefix_withObject() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_implicitPrefix_withObject( @@ -568,8 +544,7 @@ public void listObjectInfo_implicitPrefix_withSubdir() throws Exception { list_implicitPrefix_withSubdir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -579,8 +554,7 @@ public void listObjectInfoPage_implicitPrefix_withSubdir() throws Exception { getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_implicitPrefix_withSubdir( @@ -606,8 +580,7 @@ public void listObjectInfo_implicitPrefix_withImplicitSubdir() throws Exception list_implicitPrefix_withImplicitSubdir( (gcs, bucket, prefix) -> getObjectNames(gcs.listObjectInfo(bucket, prefix)), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } @Test @@ -617,8 +590,7 @@ public void listObjectInfoPage_implicitPrefix_withImplicitSubdir() throws Except getObjectNames( gcs.listObjectInfoPage(bucket, prefix, /* pageToken= */ null).getItems()), (bucket, prefix) -> - listRequestWithTrailingDelimiter( - bucket, prefix, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(bucket, prefix, /* pageToken= */ null)); } private void list_implicitPrefix_withImplicitSubdir( @@ -724,7 +696,7 @@ public void listObjectInfo_includePrefix_emptyBucket() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - testBucket, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)); + testBucket, /* prefix= */ null, /* pageToken= */ null)); } @Test @@ -743,7 +715,7 @@ public void listObjectInfo_includePrefix_objectInBucket() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - testBucket, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)); + testBucket, /* prefix= */ null, /* pageToken= */ null)); } @Test @@ -762,7 +734,7 @@ public void listObjectInfo_includePrefix_implicitDirInBucket() throws Exception assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - testBucket, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)); + testBucket, /* prefix= */ null, /* pageToken= */ null)); } @Test @@ -780,8 +752,7 @@ public void listObjectInfo_includePrefix_onlyPrefixObject() throws Exception { assertThat(getObjectNames(listedObjects)).containsExactly(testDir); assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - testBucket, testDir, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(testBucket, testDir, /* pageToken= */ null)); } @Test @@ -799,8 +770,7 @@ public void listObjectInfo_includePrefix_object() throws Exception { assertThat(getObjectNames(listedObjects)).containsExactly(testDir, testDir + "obj"); assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - testBucket, testDir, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(testBucket, testDir, /* pageToken= */ null)); } @Test @@ -818,8 +788,7 @@ public void listObjectInfo_includePrefix_subdir() throws Exception { assertThat(getObjectNames(listedObjects)).containsExactly(testDir, testDir + "dir/"); assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - testBucket, testDir, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(testBucket, testDir, /* pageToken= */ null)); } @Test @@ -837,8 +806,7 @@ public void listObjectInfo_includePrefix_implicitPrefixObject() throws Exception assertThat(getObjectNames(listedObjects)).containsExactly(testDir, testDir + "obj"); assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - testBucket, testDir, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(testBucket, testDir, /* pageToken= */ null)); } @Test @@ -856,8 +824,7 @@ public void listObjectInfo_includePrefix_implicitSubdir() throws Exception { assertThat(getObjectNames(listedObjects)).containsExactly(testDir, testDir + "dir/"); assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - testBucket, testDir, /* maxResults= */ 1024, /* pageToken= */ null)); + listRequestWithTrailingDelimiter(testBucket, testDir, /* pageToken= */ null)); } @Test @@ -890,10 +857,7 @@ public void listObjectInfo_allMetadataFieldsCorrect() throws Exception { assertThat(gcsRequestsTracker.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - objectId.getBucketName(), - testDirName, - /* maxResults= */ 1024, - /* pageToken= */ null)); + objectId.getBucketName(), testDirName, /* pageToken= */ null)); } @Test diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTest.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTest.java index fb4f5734f6..98ba664cb9 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTest.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTest.java @@ -2139,8 +2139,7 @@ public void listObjectInfo_objects() throws IOException { .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2173,8 +2172,7 @@ public void listObjectInfo_prefixesAndPrefixObjects() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, objectPrefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, objectPrefix, /* pageToken= */ null)) .inOrder(); } @@ -2211,10 +2209,8 @@ public void listObjectInfo_prefixesAndObjects() throws IOException { .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null), - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null), + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, pageToken)) .inOrder(); } @@ -2231,7 +2227,7 @@ public void listObjectInfo_includePrefix_emptyBucket() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2251,7 +2247,7 @@ public void listObjectInfo_includePrefix_objectInBucket() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2272,7 +2268,7 @@ public void listObjectInfo_includePrefix_implicitDirInBucket() throws IOExceptio assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2297,7 +2293,7 @@ public void listObjectInfo_includePrefix_dirInBucket() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2330,10 +2326,8 @@ public void listObjectInfo_includePrefix_prefixObjectDoesNotExist() throws IOExc .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null), - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null), + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, pageToken)) .inOrder(); } @@ -2365,10 +2359,8 @@ public void listObjectInfo_includePrefix_prefixObjectDoesNotExist_objects() thro .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null), - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null), + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, pageToken)) .inOrder(); } @@ -2401,10 +2393,8 @@ public void listObjectInfo_includePrefix_prefixObjectDoesNotExist_prefixes() thr .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null), - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null), + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, pageToken)) .inOrder(); } @@ -2438,10 +2428,8 @@ public void listObjectInfo_includePrefix_prefixObjectExists() throws IOException .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null), - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null), + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, pageToken)) .inOrder(); } @@ -2464,8 +2452,7 @@ public void listObjectInfo_includePrefix_onlyPrefixObjectExists() throws IOExcep assertThat(objects).containsExactly(createItemInfoForStorageObject(prefixObj)).inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2483,8 +2470,7 @@ public void listObjectInfo_includePrefix_prefixDoesNotExist() throws IOException assertThat(objects).isEmpty(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2518,8 +2504,7 @@ public void listObjectInfo_inferImplicit() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, objectPrefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, objectPrefix, /* pageToken= */ null)) .inOrder(); } @@ -2552,8 +2537,7 @@ public void listObjectInfoPage_objects() throws IOException { .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2588,8 +2572,7 @@ public void listObjectInfoPage_prefixesAndPrefixObjects() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, objectPrefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, objectPrefix, /* pageToken= */ null)) .inOrder(); } @@ -2624,8 +2607,7 @@ public void listObjectInfoPage_prefixesAndObjects() throws IOException { .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2647,7 +2629,7 @@ public void listObjectInfoPage_includePrefix_emptyBucket() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2672,7 +2654,7 @@ public void listObjectInfoPage_includePrefix_objectInBucket() throws IOException assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2698,7 +2680,7 @@ public void listObjectInfoPage_includePrefix_implicitDirInBucket() throws IOExce assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2728,7 +2710,7 @@ public void listObjectInfoPage_includePrefix_dirInBucket() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( listRequestWithTrailingDelimiter( - BUCKET_NAME, /* prefix= */ null, /* maxResults= */ 1024, /* pageToken= */ null)) + BUCKET_NAME, /* prefix= */ null, /* pageToken= */ null)) .inOrder(); } @@ -2761,8 +2743,7 @@ public void listObjectInfoPage_includePrefix_prefixObjectDoesNotExist() throws I .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2794,8 +2775,7 @@ public void listObjectInfoPage_includePrefix_prefixObjectDoesNotExist_objects() .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2827,8 +2807,7 @@ public void listObjectInfoPage_includePrefix_prefixObjectDoesNotExist_prefixes() .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2863,8 +2842,7 @@ public void listObjectInfoPage_includePrefix_prefixObjectExists() throws IOExcep .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2891,8 +2869,7 @@ public void listObjectInfoPage_includePrefix_onlyPrefixObjectExists() throws IOE .inOrder(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2912,8 +2889,7 @@ public void listObjectInfoPage_includePrefix_prefixDoesNotExist() throws IOExcep assertThat(objectsPage.getItems()).isEmpty(); assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, prefix, /* pageToken= */ null)) .inOrder(); } @@ -2949,8 +2925,7 @@ public void listObjectInfoPage_inferImplicit() throws IOException { assertThat(trackingRequestInitializerWithRetries.getAllRequestStrings()) .containsExactly( - listRequestWithTrailingDelimiter( - BUCKET_NAME, objectPrefix, /* maxResults= */ 1024, /* pageToken= */ null)) + listRequestWithTrailingDelimiter(BUCKET_NAME, objectPrefix, /* pageToken= */ null)) .inOrder(); } diff --git a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/TrackingHttpRequestInitializer.java b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/TrackingHttpRequestInitializer.java index 5d6e36b03b..5788ac6d18 100644 --- a/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/TrackingHttpRequestInitializer.java +++ b/gcsio/src/test/java/com/google/cloud/hadoop/gcsio/TrackingHttpRequestInitializer.java @@ -88,7 +88,7 @@ public class TrackingHttpRequestInitializer implements HttpRequestInitializer { "DELETE:" + GOOGLEAPIS_ENDPOINT + "/storage/v1/b/%s/o/%s?ifGenerationMatch=%s"; private static final String LIST_BUCKETS_REQUEST_FORMAT = - "GET:" + GOOGLEAPIS_ENDPOINT + "/storage/v1/b?maxResults=1024&project=%s"; + "GET:" + GOOGLEAPIS_ENDPOINT + "/storage/v1/b?maxResults=5000&project=%s"; private static final String LIST_REQUEST_FORMAT = "GET:" @@ -437,12 +437,33 @@ public static String listRequestString( pageToken); } + public static String listRequestWithTrailingDelimiter( + String bucket, String prefix, String pageToken) { + return listRequestWithTrailingDelimiter( + bucket, + prefix, + /* objectFields= */ OBJECT_FIELDS, + (int) GoogleCloudStorageOptions.MAX_LIST_ITEMS_PER_CALL_DEFAULT, + pageToken); + } + public static String listRequestWithTrailingDelimiter( String bucket, String prefix, int maxResults, String pageToken) { return listRequestWithTrailingDelimiter( bucket, prefix, /* objectFields= */ OBJECT_FIELDS, maxResults, pageToken); } + public static String listRequestWithTrailingDelimiter( + String bucket, String prefix, String objectFields, String pageToken) { + return listRequestString( + bucket, + /* includeTrailingDelimiter= */ true, + prefix, + objectFields, + (int) GoogleCloudStorageOptions.MAX_LIST_ITEMS_PER_CALL_DEFAULT, + pageToken); + } + public static String listRequestWithTrailingDelimiter( String bucket, String prefix, String objectFields, int maxResults, String pageToken) { return listRequestString( @@ -481,6 +502,23 @@ public static String listRequestString( pageToken); } + public static String listRequestString( + String bucket, + boolean flatList, + Boolean includeTrailingDelimiter, + String prefix, + String objectFields, + String pageToken) { + return listRequestString( + bucket, + flatList, + includeTrailingDelimiter, + prefix, + objectFields, + (int) GoogleCloudStorageOptions.MAX_LIST_ITEMS_PER_CALL_DEFAULT, + pageToken); + } + public static String listRequestString( String bucket, boolean flatList,