HDDS-7922. [FSO] S3G folder support fso layout filestatus s3A compatibility

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -1367,7 +1367,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
             keyInfo.getBucketName(), keyName,
             keyInfo.getDataSize(), keyInfo.getCreationTime(),
             keyInfo.getModificationTime(),
-            keyInfo.getReplicationConfig());
+            keyInfo.getReplicationConfig(),
+            keyInfo.isFile());
 
         keysResultList.add(ozoneKey);
 
@@ -1468,7 +1469,8 @@ private void addKeyPrefixInfoToResultList(String keyPrefix,
             keyInfo.getBucketName(), keyName,
             keyInfo.getDataSize(), keyInfo.getCreationTime(),
             keyInfo.getModificationTime(),
-            keyInfo.getReplicationConfig());
+            keyInfo.getReplicationConfig(),
+            keyInfo.isFile());
         keysResultList.add(ozoneKey);
       }
     }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -20,7 +20,6 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 
@@ -61,45 +60,38 @@ public class OzoneKey {
   private ReplicationConfig replicationConfig;
 
   private Map<String, String> metadata = new HashMap<>();
+
   /**
-   * Constructs OzoneKey from OmKeyInfo.
-   *
+   * Indicator if key is a file.
    */
-  @SuppressWarnings("parameternumber")
-  @Deprecated
-  public OzoneKey(String volumeName, String bucketName,
-                  String keyName, long size, long creationTime,
-                  long modificationTime, ReplicationType type,
-                  int replicationFactor) {
-    this(volumeName, bucketName, keyName, size, creationTime, modificationTime,
-            ReplicationConfig.fromTypeAndFactor(type,
-                    ReplicationFactor.valueOf(replicationFactor)));
-  }
+  private final boolean isFile;
 
   /**
    * Constructs OzoneKey from OmKeyInfo.
    *
    */
   @SuppressWarnings("parameternumber")
   public OzoneKey(String volumeName, String bucketName,
-                  String keyName, long size, long creationTime,
-                  long modificationTime, ReplicationConfig replicationConfig) {
+      String keyName, long size, long creationTime,
+      long modificationTime, ReplicationConfig replicationConfig,
+      boolean isFile) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.name = keyName;
     this.dataSize = size;
     this.creationTime = Instant.ofEpochMilli(creationTime);
     this.modificationTime = Instant.ofEpochMilli(modificationTime);
     this.replicationConfig = replicationConfig;
+    this.isFile = isFile;
   }
 
   @SuppressWarnings("parameternumber")
   public OzoneKey(String volumeName, String bucketName,
                   String keyName, long size, long creationTime,
                   long modificationTime, ReplicationConfig replicationConfig,
-                  Map<String, String> metadata) {
+                  Map<String, String> metadata, boolean isFile) {
     this(volumeName, bucketName, keyName, size, creationTime,
-        modificationTime, replicationConfig);
+        modificationTime, replicationConfig, isFile);
     this.metadata.putAll(metadata);
   }
 
@@ -187,11 +179,19 @@ public ReplicationConfig getReplicationConfig() {
     return replicationConfig;
   }
 
+  /**
+   * Returns indicator if key is a file.
+   * @return file
+   */
+  public boolean isFile() {
+    return isFile;
+  }
+
   public static OzoneKey fromKeyInfo(OmKeyInfo keyInfo) {
     return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
         keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
         keyInfo.getModificationTime(), keyInfo.getReplicationConfig(),
-        keyInfo.getMetadata());
+        keyInfo.getMetadata(), keyInfo.isFile());
   }
 
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -21,7 +21,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
-import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.ratis.util.function.CheckedSupplier;
 
@@ -43,25 +42,6 @@ public class OzoneKeyDetails extends OzoneKey {
 
   private final CheckedSupplier<OzoneInputStream, IOException> contentSupplier;
 
-  /**
-   * Constructs OzoneKeyDetails from OmKeyInfo.
-   */
-  @SuppressWarnings("parameternumber")
-  @Deprecated
-  public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
-                         long size, long creationTime, long modificationTime,
-                         List<OzoneKeyLocation> ozoneKeyLocations,
-                         ReplicationType type, Map<String, String> metadata,
-                         FileEncryptionInfo feInfo, int replicationFactor) {
-    super(volumeName, bucketName, keyName, size, creationTime,
-        modificationTime, type, replicationFactor);
-    this.ozoneKeyLocations = ozoneKeyLocations;
-    this.feInfo = feInfo;
-    contentSupplier = null;
-    this.setMetadata(metadata);
-  }
-
-
   /**
    * Constructs OzoneKeyDetails from OmKeyInfo.
    */
@@ -72,9 +52,10 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
       ReplicationConfig replicationConfig,
       Map<String, String> metadata,
       FileEncryptionInfo feInfo,
-      CheckedSupplier<OzoneInputStream, IOException> contentSupplier) {
+      CheckedSupplier<OzoneInputStream, IOException> contentSupplier,
+      boolean isFile) {
     super(volumeName, bucketName, keyName, size, creationTime,
-            modificationTime, replicationConfig, metadata);
+        modificationTime, replicationConfig, metadata, isFile);
     this.ozoneKeyLocations = ozoneKeyLocations;
     this.feInfo = feInfo;
     this.contentSupplier = contentSupplier;
@@ -93,6 +74,8 @@ public FileEncryptionInfo getFileEncryptionInfo() {
 
   /**
    * Get OzoneInputStream to read the content of the key.
+   * @return OzoneInputStream
+   * @throws IOException
    */
   @JsonIgnore
   public OzoneInputStream getContent() throws IOException {

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1543,7 +1543,8 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
         key.getDataSize(),
         key.getCreationTime(),
         key.getModificationTime(),
-        key.getReplicationConfig()))
+        key.getReplicationConfig(),
+        key.isFile()))
         .collect(Collectors.toList());
   }
 
@@ -1594,7 +1595,7 @@ private OzoneKeyDetails getOzoneKeyDetails(OmKeyInfo keyInfo) {
         keyInfo.getModificationTime(), ozoneKeyLocations,
         keyInfo.getReplicationConfig(), keyInfo.getMetadata(),
         keyInfo.getFileEncryptionInfo(),
-        () -> getInputStreamWithRetryFunction(keyInfo));
+        () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile());
   }
 
   @Override

hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot

@@ -151,14 +151,34 @@ Incorrect values for end and start offset
                                 Should Be Equal            ${expectedData}            ${actualData}
 
 Zero byte file
-    ${result} =                 Execute AWSS3APICli and checkrc        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result   255
-                                Should contain             ${result}        InvalidRange
-
-    ${result} =                 Execute AWSS3APICli and checkrc        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result   255
-                                Should contain             ${result}        InvalidRange
-
-    ${result} =                 Execute AWSS3APICli and checkrc        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result   255
-                                Should contain             ${result}        InvalidRange
+    ${result} =			        Execute                    ozone sh bucket info /s3v/${BUCKET}
+    ${linked} = 		        Execute				       echo '${result}' | jq -j '.sourceVolume,"/",.sourceBucket'
+    ${eval} = 			        Evaluate			       "source" in """${linked}"""
+    	      			        IF	${eval} == ${True}
+    ${result} =                 Execute				       ozone sh bucket info ${linked}
+				                END
+    ${fsolayout} =    		    Evaluate    	   	       "OPTIMIZED" in """${result}"""
+
+    ${result} =                 Execute AWSS3APICli and checkrc     get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result   255
+    	      			        IF 	${fsolayout} == ${True}
+                                Should contain              ${result}       NoSuchKey
+				                ELSE
+                                Should contain              ${result}       InvalidRange
+				                END
+
+    ${result} =                 Execute AWSS3APICli and checkrc     get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result   255
+                                IF 	${fsolayout} == ${True}
+                                Should contain              ${result}       NoSuchKey
+                                ELSE
+                                Should contain              ${result}       InvalidRange
+				                END
+
+    ${result} =                 Execute AWSS3APICli and checkrc     get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result   255
+    	      			        IF 	${fsolayout} == ${True}
+                                Should contain              ${result}       NoSuchKey
+				                ELSE
+                                Should contain              ${result}       InvalidRange
+				                END
 
 Create file with user defined metadata
                                 Execute                   echo "Randomtext" > /tmp/testfile2

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -355,6 +355,8 @@ public Response get(
       OzoneKeyDetails keyDetails = getClientProtocol()
           .getS3KeyDetails(bucketName, keyPath);
 
+      isFile(keyPath, keyDetails);
+
       long length = keyDetails.getDataSize();
 
       LOG.debug("Data length of the key {} is {}", keyPath, length);
@@ -503,6 +505,8 @@ public Response head(
     OzoneKey key;
     try {
       key = getClientProtocol().headS3Object(bucketName, keyPath);
+
+      isFile(keyPath, key);
       // TODO: return the specified range bytes of this object.
     } catch (OMException ex) {
       AUDIT.logReadFailure(
@@ -536,6 +540,22 @@ public Response head(
     return response.build();
   }
 
+  private void isFile(String keyPath, OzoneKey key) throws OMException {
+    /*
+      Necessary for directories in buckets with FSO layout.
+      Intended for apps which use Hadoop S3A.
+      Example of such app is Trino (through Hive connector).
+     */
+    boolean isFsoDirCreationEnabled = ozoneConfiguration
+        .getBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED,
+            OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT);
+    if (isFsoDirCreationEnabled &&
+        !key.isFile() &&
+        !keyPath.endsWith("/")) {
+      throw new OMException(ResultCodes.KEY_NOT_FOUND);
+    }
+  }
+
   /**
    * Abort multipart upload request.
    * @param bucket

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java

@@ -151,7 +151,7 @@ public void close() throws IOException {
                 System.currentTimeMillis(),
                 System.currentTimeMillis(),
                 new ArrayList<>(), replicationConfig, metadata, null,
-                () -> readKey(key)
+                () -> readKey(key), true
             ));
             super.close();
           }
@@ -183,7 +183,7 @@ public void close() throws IOException {
                 System.currentTimeMillis(),
                 System.currentTimeMillis(),
                 new ArrayList<>(), finalReplicationCon, metadata, null,
-                () -> readKey(key)
+                () -> readKey(key), true
             ));
             super.close();
           }
@@ -215,7 +215,8 @@ public OzoneKey headObject(String key) throws IOException {
           ozoneKeyDetails.getDataSize(),
           ozoneKeyDetails.getCreationTime().toEpochMilli(),
           ozoneKeyDetails.getModificationTime().toEpochMilli(),
-          ozoneKeyDetails.getReplicationConfig());
+          ozoneKeyDetails.getReplicationConfig(),
+          ozoneKeyDetails.isFile());
     } else {
       throw new OMException(ResultCodes.KEY_NOT_FOUND);
     }
@@ -446,4 +447,17 @@ public void setReplicationConfig(ReplicationConfig replicationConfig) {
   public ReplicationConfig getReplicationConfig() {
     return this.replicationConfig;
   }
+
+  @Override
+  public void createDirectory(String keyName) throws IOException {
+    keyDetails.put(keyName, new OzoneKeyDetails(
+        getVolumeName(),
+        getName(),
+        keyName,
+        0,
+        System.currentTimeMillis(),
+        System.currentTimeMillis(),
+        new ArrayList<>(), replicationConfig, new HashMap<>(), null,
+        () -> readKey(keyName), false));
+  }
 }

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java

@@ -40,9 +40,12 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
 import org.mockito.Mockito;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
+import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
 import static org.mockito.Mockito.doReturn;
 
@@ -218,4 +221,26 @@ private void setDefaultHeader() {
     doReturn(CONTENT_ENCODING1)
         .when(headers).getHeaderString("Content-Encoding");
   }
+
+  @Test
+  public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash()
+      throws IOException {
+    // GIVEN
+    final String bucketName = "b1";
+    final String keyPath = "keyDir";
+    OzoneConfiguration config = new OzoneConfiguration();
+    config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true");
+    rest.setOzoneConfiguration(config);
+    OzoneBucket bucket = client.getObjectStore().getS3Bucket(bucketName);
+    bucket.createDirectory(keyPath);
+
+    // WHEN
+    final OS3Exception ex =
+        Assertions.assertThrows(OS3Exception.class,
+            () -> rest.get(bucketName, keyPath, null, 0, null));
+
+    // THEN
+    Assertions.assertEquals(NO_SUCH_KEY.getCode(), ex.getCode());
+    bucket.deleteKey(keyPath);
+  }
 }