HDDS-12870. Fix listObjects corner cases

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -29,6 +29,7 @@
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -766,6 +767,16 @@ private void addKey(ListObjectResponse response, OzoneKey next) {
     response.addKey(keyMetadata);
   }
 
+  @VisibleForTesting
+  public void setOzoneConfiguration(OzoneConfiguration config) {
+    this.ozoneConfiguration = config;
+  }
+
+  @VisibleForTesting
+  public OzoneConfiguration getOzoneConfiguration() {
+    return this.ozoneConfiguration;
+  }
+
   @Override
   @PostConstruct
   public void init() {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpointBuilder.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+/**
+ * Builder for BucketEndpoint in tests.
+ */
+public class BucketEndpointBuilder extends
+    EndpointBuilder<BucketEndpoint> {
+
+  public BucketEndpointBuilder() {
+    super(BucketEndpoint::new);
+  }
+
+  @Override
+  public BucketEndpoint build() {
+    BucketEndpoint endpoint = super.build();
+    endpoint.setOzoneConfiguration(getConfig());
+
+    return endpoint;
+  }
+}

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java

@@ -111,7 +111,7 @@ public static EndpointBuilder<RootEndpoint> newRootEndpointBuilder() {
   }
 
   public static EndpointBuilder<BucketEndpoint> newBucketEndpointBuilder() {
-    return new EndpointBuilder<>(BucketEndpoint::new);
+    return new BucketEndpointBuilder();
   }
 
   public static EndpointBuilder<ObjectEndpoint> newObjectEndpointBuilder() {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.s3.endpoint;
 
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -26,6 +27,8 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
+import java.util.stream.IntStream;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -541,6 +544,39 @@ public void testListObjectsWithInvalidMaxKeys() throws Exception {
     assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e2.getCode());
   }
 
+  @Test
+  public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception {
+    // Arrange: Create a bucket with 1001 keys
+    String[] keys = IntStream.range(0, 1001).mapToObj(i -> "file" + i).toArray(String[]::new);
+    OzoneClient client = createClientWithKeys(keys);
+
+    // Arrange: Set the max-keys limit in the configuration
+    OzoneConfiguration config = new OzoneConfiguration();
+    final String configuredMaxKeysLimit = "900";
+    config.set(OZONE_S3G_LIST_MAX_KEYS_LIMIT, configuredMaxKeysLimit);
+
+    // Arrange: Build and initialize the BucketEndpoint with the config
+    BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+        .setClient(client)
+        .setConfig(config)
+        .build();
+    bucketEndpoint.init();
+
+    // Assert: Ensure the config value is correctly set in the endpoint
+    assertEquals(configuredMaxKeysLimit,
+        bucketEndpoint.getOzoneConfiguration().get(OZONE_S3G_LIST_MAX_KEYS_LIMIT));
+
+    // Act: Request more keys than the configured max-keys limit
+    final int requestedMaxKeys = Integer.parseInt(configuredMaxKeysLimit) + 1;
+    ListObjectResponse response = (ListObjectResponse)
+        bucketEndpoint.get("b1", null, null, null, requestedMaxKeys,
+            null, null, null, null, null, null, null,
+            1000, null).getEntity();
+
+    // Assert: The number of returned keys should be capped at the configured limit
+    assertEquals(Integer.parseInt(configuredMaxKeysLimit), response.getContents().size());
+  }
+
   private void assertEncodingTypeObject(
       String exceptName, String exceptEncodingType, EncodingTypeObject object) {
     assertEquals(exceptName, object.getName());