HDDS-12870. Fix listObjects corner cases

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/rest/S3V4Signer.java

@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.rest;
+
+import java.net.HttpURLConnection;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+/**
+ * Utility for generating AWS S3 Signature V4 headers for HTTP requests.
+ * Only supports unsigned payloads (useful for GET/HEAD).
+ */
+public final class S3V4Signer {
+  private S3V4Signer() {
+    throw new AssertionError("Utility class");
+  }
+
+  /**
+   * Signs the given HttpURLConnection with AWS S3 Signature V4 headers.
+   *
+   * @param conn        The HttpURLConnection to sign
+   * @param accessKey   AWS access key
+   * @param secretKey   AWS secret key
+   * @param region      AWS region (e.g., "us-east-1")
+   * @param service     AWS service (e.g., "s3")
+   * @param bucket      S3 bucket name
+   * @param queryString The query string (e.g., "max-keys=-1")
+   */
+  public static void signRequest(HttpURLConnection conn, String accessKey, String secretKey, String region,
+                                 String service, String bucket, String queryString) {
+    try {
+      String method = conn.getRequestMethod();
+      String host = conn.getURL().getHost();
+      int port = conn.getURL().getPort();
+      if (port != -1 && port != 80 && port != 443) {
+        host = host + ":" + port;
+      }
+      String amzDate = getAmzDate();
+      String dateStamp = getDateStamp();
+
+      String payloadHash = hash(""); // SHA256("") for empty payload
+      String canonicalUri = "/" + bucket + "/";
+      String canonicalHeaders = "host:" + host + "\n" +
+          "x-amz-content-sha256:" + payloadHash + "\n" +
+          "x-amz-date:" + amzDate + "\n";
+      String signedHeaders = "host;x-amz-content-sha256;x-amz-date";
+
+      String canonicalRequest = method + "\n" +
+          canonicalUri + "\n" +
+          queryString + "\n" +
+          canonicalHeaders + "\n" +
+          signedHeaders + "\n" +
+          payloadHash;
+
+      String algorithm = "AWS4-HMAC-SHA256";
+      String credentialScope = dateStamp + "/" + region + "/" + service + "/aws4_request";
+      String stringToSign = algorithm + "\n" +
+          amzDate + "\n" +
+          credentialScope + "\n" +
+          hash(canonicalRequest);
+
+      byte[] signingKey = getSignatureKey(secretKey, dateStamp, region, service);
+      String signature = bytesToHex(hmacSHA256(stringToSign, signingKey));
+
+      String authorizationHeader = algorithm + " " +
+          "Credential=" + accessKey + "/" + credentialScope + ", " +
+          "SignedHeaders=" + signedHeaders + ", " +
+          "Signature=" + signature;
+
+      conn.setRequestProperty("x-amz-date", amzDate);
+      conn.setRequestProperty("x-amz-content-sha256", payloadHash);
+      conn.setRequestProperty("Authorization", authorizationHeader);
+    } catch (Exception e) {
+      throw new RuntimeException("S3V4Signer failed to sign request: " + e.getMessage(), e);
+    }
+  }
+
+  /**
+   * Returns the current date in the format "yyyyMMdd'T'HHmmss'Z'".
+   *
+   * @return The current date in the format "yyyyMMdd'T'HHmmss'Z'"
+   */
+  private static String getAmzDate() {
+    SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
+    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
+    return fmt.format(new Date());
+  }
+
+  /**
+   * Returns the current date in the format "yyyyMMdd".
+   *
+   * @return The current date in the format "yyyyMMdd"
+   */
+  private static String getDateStamp() {
+    SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
+    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
+    return fmt.format(new Date());
+  }
+
+  /**
+   * Returns the SHA-256 hash of the given string.
+   *
+   * @param text The string to hash
+   * @return The SHA-256 hash of the given string
+   * @throws Exception If hashing fails
+   */
+  private static String hash(String text) throws Exception {
+    MessageDigest md = MessageDigest.getInstance("SHA-256");
+    md.update(text.getBytes(StandardCharsets.UTF_8));
+    byte[] digest = md.digest();
+    return bytesToHex(digest);
+  }
+
+  /**
+   * Returns the HMAC-SHA256 of the given data using the given key.
+   *
+   * @param data The data to sign
+   * @param key  The key to use for signing
+   * @return The HMAC-SHA256 of the given data using the given key
+   * @throws Exception If signing fails
+   */
+  private static byte[] hmacSHA256(String data, byte[] key) throws Exception {
+    String algorithm = "HmacSHA256";
+    Mac mac = Mac.getInstance(algorithm);
+    mac.init(new SecretKeySpec(key, algorithm));
+    return mac.doFinal(data.getBytes(StandardCharsets.UTF_8));
+  }
+
+  /**
+   * Returns the signature key for the given secret key, date stamp, region, and service.
+   *
+   * @param key        The secret key
+   * @param dateStamp  The date stamp
+   * @param regionName The region name
+   * @param serviceName The service name
+   * @return The signature key for the given secret key, date stamp, region, and service
+   * @throws Exception If key derivation fails
+   */
+  private static byte[] getSignatureKey(String key, String dateStamp, String regionName, String serviceName)
+      throws Exception {
+    byte[] kSecret = ("AWS4" + key).getBytes(StandardCharsets.UTF_8);
+    byte[] kDate = hmacSHA256(dateStamp, kSecret);
+    byte[] kRegion = hmacSHA256(regionName, kDate);
+    byte[] kService = hmacSHA256(serviceName, kRegion);
+    return hmacSHA256("aws4_request", kService);
+  }
+
+  /**
+   * Returns the hexadecimal representation of the given bytes.
+   *
+   * @param bytes The bytes to convert
+   * @return The hexadecimal representation of the given bytes
+   */
+  private static String bytesToHex(byte[] bytes) {
+    StringBuilder sb = new StringBuilder();
+    for (byte b : bytes) {
+      sb.append(String.format("%02x", b));
+    }
+    return sb.toString();
+  }
+}

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/rest/TestS3RestNonSdkCases.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.rest;
+
+import static org.apache.hadoop.ozone.OzoneConsts.LOCALHOST;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.s3.S3GatewayService;
+import org.apache.ozone.test.OzoneTestBase;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Integration tests for S3 REST edge cases that cannot be triggered by AWS SDK clients.
+ * For example: negative/zero max-keys, or other invalid parameters that SDK would block client-side.
+ */
+public class TestS3RestNonSdkCases extends OzoneTestBase {
+
+  private static MiniOzoneCluster cluster = null;
+  private static S3GatewayService s3g = null;
+  private static final String ACCESS_KEY = "testuser";
+  private static final String SECRET_KEY = "testpass";
+
+  @BeforeAll
+  public static void startCluster() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
+    s3g = new S3GatewayService();
+
+    cluster = MiniOzoneCluster.newBuilder(conf)
+        .addService(s3g)
+        .setNumDatanodes(3)
+        .build();
+    cluster.waitForClusterToBeReady();
+    cluster.newClient().getObjectStore().createS3Bucket(getTestBucketName());
+  }
+
+  @AfterAll
+  public static void shutdownCluster() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test
+  public void testListObjectsWithNegativeMaxKeys() throws Exception {
+    final String bucketName = getTestBucketName();
+    String s3Endpoint = getS3EndpointURL();
+    String queryString = "max-keys=-1";
+    String url = s3Endpoint + "/" + bucketName + "/?" + queryString;
+
+    HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+    conn.setRequestMethod("GET");
+    S3V4Signer.signRequest(conn, ACCESS_KEY, SECRET_KEY, "us-east-1", "s3", bucketName, queryString);
+
+    int code = conn.getResponseCode();
+    assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, code,
+        "Should return 400 Bad Request (InvalidArgument) for max-keys=-1");
+  }
+
+  private static String getS3EndpointURL() {
+    String addr = s3g.getConf().get(OZONE_S3G_HTTP_ADDRESS_KEY);
+    String hostPort = addr.replace("0.0.0.0", LOCALHOST);
+    return "http://" + hostPort;
+  }
+
+  private static String getTestBucketName() {
+    return ("testrestnegmaxkeys" + System.currentTimeMillis()).toLowerCase();
+  }
+}

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java

@@ -142,6 +142,8 @@ public Response get(
         return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads);
       }
 
+      maxKeys = validateMaxKeys(maxKeys);
+
       if (prefix == null) {
         prefix = "";
       }
@@ -292,6 +294,14 @@ public Response get(
     return Response.ok(response).build();
   }
 
+  private int validateMaxKeys(int maxKeys) throws OS3Exception {
+    if (maxKeys <= 0) {
+      throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, "maxKeys must be > 0");
+    }
+
+    return Math.min(maxKeys, 1000);
+  }
+
   @PUT
   public Response put(@PathParam("bucket") String bucketName,
                       @QueryParam("acl") String aclMarker,

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java

@@ -521,6 +521,28 @@ public void testEncodingTypeException() throws IOException {
     assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode());
   }
 
+  @Test
+  public void testListObjectsWithInvalidMaxKeys() throws Exception {
+    OzoneClient client = createClientWithKeys("file1");
+    BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder()
+        .setClient(client)
+        .build();
+
+    // maxKeys < 0
+    OS3Exception e1 = assertThrows(OS3Exception.class, () ->
+        bucketEndpoint.get("bucket", null, null, null, -1, null,
+            null, null, null, null, null, null, 1000, null)
+    );
+    assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e1.getCode());
+
+    // maxKeys == 0
+    OS3Exception e2 = assertThrows(OS3Exception.class, () ->
+        bucketEndpoint.get("bucket", null, null, null, 0, null,
+            null, null, null, null, null, null, 1000, null)
+    );
+    assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e2.getCode());
+  }
+
   private void assertEncodingTypeObject(
       String exceptName, String exceptEncodingType, EncodingTypeObject object) {
     assertEquals(exceptName, object.getName());