HDDS-12870. Fix listObjects corner cases

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1962,6 +1962,15 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.s3g.list.max.keys.limit</name>
+    <value>1000</value>
+    <description>
+      Maximum number of keys returned by S3 ListObjects/ListObjectsV2 API.
+      AWS default is 1000. Can be overridden per deployment in ozone-site.xml.
+    </description>
+  </property>
+
   <property>
     <name>ozone.s3g.secret.http.enabled</name>
     <value>false</value>

hadoop-ozone/dist/src/main/smoketest/s3/objectlist.robot

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation     S3 max-keys validation test for negative and zero values
+Library           OperatingSystem
+Library           String
+Resource          ../commonlib.robot
+Resource          commonawslib.robot
+Test Timeout      3 minutes
+Suite Setup       Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL}   http://s3g:9878
+${BUCKET}         generated
+
+*** Keywords ***
+Prepare Many Objects In Bucket
+    [Arguments]    ${count}=1100
+    Execute    mkdir -p /tmp/manyfiles
+    FOR    ${i}    IN RANGE    ${count}
+        Execute    echo "test-${i}" > /tmp/manyfiles/obj-${i}
+    END
+    Execute    aws s3 cp /tmp/manyfiles s3://${BUCKET}/ --recursive --endpoint-url=${ENDPOINT_URL}
+
+*** Test Cases ***
+
+List objects with negative max-keys should fail
+    ${result} =    Execute AWSS3APICli and checkrc    list-objects-v2 --bucket ${BUCKET} --max-keys -1    255
+    Should Contain    ${result}    InvalidArgument
+
+List objects with zero max-keys should fail
+    ${result} =    Execute AWSS3APICli and checkrc    list-objects-v2 --bucket ${BUCKET} --max-keys 0    255
+    Should Contain    ${result}    InvalidArgument
+
+List objects with max-keys exceeding config limit should not return more than limit
+    Prepare Many Objects In Bucket    1100
+    ${result}=    Execute AWSS3APICli and checkrc    list-objects-v2 --bucket ${BUCKET} --max-keys 9999 --endpoint-url=${ENDPOINT_URL} --output json    0
+    ${tmpfile}=   Generate Random String    8
+    ${tmpfile}=   Set Variable    /tmp/result_${tmpfile}.json
+    Create File   ${tmpfile}    ${result}
+    ${count}=     Execute and checkrc    jq -r '.Contents | length' ${tmpfile}    0
+    Should Be True    ${count} <= 1000
+    Remove File   ${tmpfile}
+
+List objects with max-keys less than config limit should return correct count
+    Prepare Many Objects In Bucket    1100
+    ${result}=    Execute AWSS3APICli and checkrc    list-objects-v2 --bucket ${BUCKET} --max-keys 500 --endpoint-url=${ENDPOINT_URL} --output json    0
+    ${tmpfile}=   Generate Random String    8
+    ${tmpfile}=   Set Variable    /tmp/result_${tmpfile}.json
+    Create File   ${tmpfile}    ${result}
+    ${count}=     Execute and checkrc    jq -r '.Contents | length' ${tmpfile}    0
+    Should Be True    ${count} == 500
+    Remove File   ${tmpfile}

hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh

@@ -85,5 +85,6 @@ run_robot_test objectmultidelete
 run_robot_test objecthead
 run_robot_test MultipartUpload
 run_robot_test objecttagging
+run_robot_test objectlist
 
 rebot --outputdir results/ results/*.xml

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java

@@ -98,6 +98,10 @@ public final class S3GatewayConfigKeys {
   public static final String OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY
       = "ozone.s3g.metrics.percentiles.intervals.seconds";
 
+  // S3 ListObjects max-keys limit (default: 1000, AWS compatible)
+  public static final String OZONE_S3G_LIST_MAX_KEYS_LIMIT = "ozone.s3g.list.max.keys.limit";
+  public static final int OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT = 1000;
+
   /**
    * Never constructed.
    */

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -23,10 +23,13 @@
 import static org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED;
 import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT;
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NOT_IMPLEMENTED;
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -37,6 +40,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
@@ -91,6 +95,7 @@ public class BucketEndpoint extends EndpointBase {
       LoggerFactory.getLogger(BucketEndpoint.class);
 
   private boolean listKeysShallowEnabled;
+  private int maxKeysLimit = 1000;
 
   @Inject
   private OzoneConfiguration ozoneConfiguration;
@@ -142,6 +147,8 @@ public Response get(
         return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads);
       }
 
+      maxKeys = validateMaxKeys(maxKeys);
+
       if (prefix == null) {
         prefix = "";
       }
@@ -292,6 +299,14 @@ public Response get(
     return Response.ok(response).build();
   }
 
+  private int validateMaxKeys(int maxKeys) throws OS3Exception {
+    if (maxKeys <= 0) {
+      throw newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0");
+    }
+
+    return Math.min(maxKeys, maxKeysLimit);
+  }
+
   @PUT
   public Response put(@PathParam("bucket") String bucketName,
                       @QueryParam("acl") String aclMarker,
@@ -752,10 +767,24 @@ private void addKey(ListObjectResponse response, OzoneKey next) {
     response.addKey(keyMetadata);
   }
 
+  @VisibleForTesting
+  public void setOzoneConfiguration(OzoneConfiguration config) {
+    this.ozoneConfiguration = config;
+  }
+
+  @VisibleForTesting
+  public OzoneConfiguration getOzoneConfiguration() {
+    return this.ozoneConfiguration;
+  }
+
   @Override
+  @PostConstruct
   public void init() {
     listKeysShallowEnabled = ozoneConfiguration.getBoolean(
         OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED,
         OZONE_S3G_LIST_KEYS_SHALLOW_ENABLED_DEFAULT);
+    maxKeysLimit = ozoneConfiguration.getInt(
+        OZONE_S3G_LIST_MAX_KEYS_LIMIT,
+        OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT);
   }
 }

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpointBuilder.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+/**
+ * Builder for BucketEndpoint in tests.
+ */
+public class BucketEndpointBuilder extends
+    EndpointBuilder<BucketEndpoint> {
+
+  public BucketEndpointBuilder() {
+    super(BucketEndpoint::new);
+  }
+
+  @Override
+  public BucketEndpoint build() {
+    BucketEndpoint endpoint = super.build();
+    endpoint.setOzoneConfiguration(getConfig());
+
+    return endpoint;
+  }
+}

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBuilder.java

@@ -111,7 +111,7 @@ public static EndpointBuilder<RootEndpoint> newRootEndpointBuilder() {
   }
 
   public static EndpointBuilder<BucketEndpoint> newBucketEndpointBuilder() {
-    return new EndpointBuilder<>(BucketEndpoint::new);
+    return new BucketEndpointBuilder();
   }
 
   public static EndpointBuilder<ObjectEndpoint> newObjectEndpointBuilder() {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.s3.endpoint;
 
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_LIST_MAX_KEYS_LIMIT;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.ENCODING_TYPE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -26,6 +27,8 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
+import java.util.stream.IntStream;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientStub;
@@ -519,6 +522,61 @@ public void testEncodingTypeException() throws IOException {
     assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode());
   }
 
+  @Test
+  public void testListObjectsWithInvalidMaxKeys() throws Exception {
+    OzoneClient client = createClientWithKeys("file1");
+    BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+        .setClient(client)
+        .build();
+
+    // maxKeys < 0
+    OS3Exception e1 = assertThrows(OS3Exception.class, () ->
+        bucketEndpoint.get("bucket", null, null, null, -1, null,
+            null, null, null, null, null, null, 1000, null)
+    );
+    assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e1.getCode());
+
+    // maxKeys == 0
+    OS3Exception e2 = assertThrows(OS3Exception.class, () ->
+        bucketEndpoint.get("bucket", null, null, null, 0, null,
+            null, null, null, null, null, null, 1000, null)
+    );
+    assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e2.getCode());
+  }
+
+  @Test
+  public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception {
+    // Arrange: Create a bucket with 1001 keys
+    String[] keys = IntStream.range(0, 1001).mapToObj(i -> "file" + i).toArray(String[]::new);
+    OzoneClient client = createClientWithKeys(keys);
+
+    // Arrange: Set the max-keys limit in the configuration
+    OzoneConfiguration config = new OzoneConfiguration();
+    final String configuredMaxKeysLimit = "900";
+    config.set(OZONE_S3G_LIST_MAX_KEYS_LIMIT, configuredMaxKeysLimit);
+
+    // Arrange: Build and initialize the BucketEndpoint with the config
+    BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+        .setClient(client)
+        .setConfig(config)
+        .build();
+    bucketEndpoint.init();
+
+    // Assert: Ensure the config value is correctly set in the endpoint
+    assertEquals(configuredMaxKeysLimit,
+        bucketEndpoint.getOzoneConfiguration().get(OZONE_S3G_LIST_MAX_KEYS_LIMIT));
+
+    // Act: Request more keys than the configured max-keys limit
+    final int requestedMaxKeys = Integer.parseInt(configuredMaxKeysLimit) + 1;
+    ListObjectResponse response = (ListObjectResponse)
+        bucketEndpoint.get("b1", null, null, null, requestedMaxKeys,
+            null, null, null, null, null, null, null,
+            1000, null).getEntity();
+
+    // Assert: The number of returned keys should be capped at the configured limit
+    assertEquals(Integer.parseInt(configuredMaxKeysLimit), response.getContents().size());
+  }
+
   private void assertEncodingTypeObject(
       String exceptName, String exceptEncodingType, EncodingTypeObject object) {
     assertEquals(exceptName, object.getName());