diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 300ae817c420..4210876f987b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1367,7 +1367,8 @@ private boolean getChildrenKeys(String keyPrefix, String startKey, keyInfo.getBucketName(), keyName, keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), - keyInfo.getReplicationConfig()); + keyInfo.getReplicationConfig(), + keyInfo.isFile()); keysResultList.add(ozoneKey); @@ -1468,7 +1469,8 @@ private void addKeyPrefixInfoToResultList(String keyPrefix, keyInfo.getBucketName(), keyName, keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), - keyInfo.getReplicationConfig()); + keyInfo.getReplicationConfig(), + keyInfo.isFile()); keysResultList.add(ozoneKey); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 44a400230c0d..fba9826df1a1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -61,20 +60,11 @@ public class OzoneKey { private ReplicationConfig replicationConfig; private Map metadata = new HashMap<>(); + /** - * Constructs OzoneKey from OmKeyInfo. - * + * Indicator if key is a file. */ - @SuppressWarnings("parameternumber") - @Deprecated - public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationType type, - int replicationFactor) { - this(volumeName, bucketName, keyName, size, creationTime, modificationTime, - ReplicationConfig.fromTypeAndFactor(type, - ReplicationFactor.valueOf(replicationFactor))); - } + private final boolean isFile; /** * Constructs OzoneKey from OmKeyInfo. @@ -82,8 +72,9 @@ public OzoneKey(String volumeName, String bucketName, */ @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationConfig replicationConfig) { + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, + boolean isFile) { this.volumeName = volumeName; this.bucketName = bucketName; this.name = keyName; @@ -91,15 +82,16 @@ public OzoneKey(String volumeName, String bucketName, this.creationTime = Instant.ofEpochMilli(creationTime); this.modificationTime = Instant.ofEpochMilli(modificationTime); this.replicationConfig = replicationConfig; + this.isFile = isFile; } @SuppressWarnings("parameternumber") public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, long modificationTime, ReplicationConfig replicationConfig, - Map metadata) { + Map metadata, boolean isFile) { this(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); + modificationTime, replicationConfig, isFile); this.metadata.putAll(metadata); } @@ -187,11 +179,19 @@ public ReplicationConfig getReplicationConfig() { return replicationConfig; } + /** + * Returns indicator if key is a file. + * @return file + */ + public boolean isFile() { + return isFile; + } + public static OzoneKey fromKeyInfo(OmKeyInfo keyInfo) { return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), - keyInfo.getMetadata()); + keyInfo.getMetadata(), keyInfo.isFile()); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index ff8e7a80d59f..8c29b66fd34a 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.ratis.util.function.CheckedSupplier; @@ -43,25 +42,6 @@ public class OzoneKeyDetails extends OzoneKey { private final CheckedSupplier contentSupplier; - /** - * Constructs OzoneKeyDetails from OmKeyInfo. - */ - @SuppressWarnings("parameternumber") - @Deprecated - public OzoneKeyDetails(String volumeName, String bucketName, String keyName, - long size, long creationTime, long modificationTime, - List ozoneKeyLocations, - ReplicationType type, Map metadata, - FileEncryptionInfo feInfo, int replicationFactor) { - super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, type, replicationFactor); - this.ozoneKeyLocations = ozoneKeyLocations; - this.feInfo = feInfo; - contentSupplier = null; - this.setMetadata(metadata); - } - - /** * Constructs OzoneKeyDetails from OmKeyInfo. */ @@ -72,9 +52,10 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig, Map metadata, FileEncryptionInfo feInfo, - CheckedSupplier contentSupplier) { + CheckedSupplier contentSupplier, + boolean isFile) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig, metadata); + modificationTime, replicationConfig, metadata, isFile); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; this.contentSupplier = contentSupplier; @@ -93,6 +74,8 @@ public FileEncryptionInfo getFileEncryptionInfo() { /** * Get OzoneInputStream to read the content of the key. + * @return OzoneInputStream + * @throws IOException */ @JsonIgnore public OzoneInputStream getContent() throws IOException { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index b22ca6670897..569901b667f8 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1543,7 +1543,8 @@ public List listKeys(String volumeName, String bucketName, key.getDataSize(), key.getCreationTime(), key.getModificationTime(), - key.getReplicationConfig())) + key.getReplicationConfig(), + key.isFile())) .collect(Collectors.toList()); } @@ -1594,7 +1595,7 @@ private OzoneKeyDetails getOzoneKeyDetails(OmKeyInfo keyInfo) { keyInfo.getModificationTime(), ozoneKeyLocations, keyInfo.getReplicationConfig(), keyInfo.getMetadata(), keyInfo.getFileEncryptionInfo(), - () -> getInputStreamWithRetryFunction(keyInfo)); + () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile()); } @Override diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 2f74decb617a..2a81d2caeea6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -151,14 +151,34 @@ Incorrect values for end and start offset Should Be Equal ${expectedData} ${actualData} Zero byte file - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 - Should contain ${result} InvalidRange - - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 - Should contain ${result} InvalidRange - - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 - Should contain ${result} InvalidRange + ${result} = Execute ozone sh bucket info /s3v/${BUCKET} + ${linked} = Execute echo '${result}' | jq -j '.sourceVolume,"/",.sourceBucket' + ${eval} = Evaluate "source" in """${linked}""" + IF ${eval} == ${True} + ${result} = Execute ozone sh bucket info ${linked} + END + ${fsolayout} = Evaluate "OPTIMIZED" in """${result}""" + + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 + IF ${fsolayout} == ${True} + Should contain ${result} NoSuchKey + ELSE + Should contain ${result} InvalidRange + END + + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 + IF ${fsolayout} == ${True} + Should contain ${result} NoSuchKey + ELSE + Should contain ${result} InvalidRange + END + + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 + IF ${fsolayout} == ${True} + Should contain ${result} NoSuchKey + ELSE + Should contain ${result} InvalidRange + END Create file with user defined metadata Execute echo "Randomtext" > /tmp/testfile2 diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index fe51cf77f2e1..be0eaacffa0f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -355,6 +355,8 @@ public Response get( OzoneKeyDetails keyDetails = getClientProtocol() .getS3KeyDetails(bucketName, keyPath); + isFile(keyPath, keyDetails); + long length = keyDetails.getDataSize(); LOG.debug("Data length of the key {} is {}", keyPath, length); @@ -503,6 +505,8 @@ public Response head( OzoneKey key; try { key = getClientProtocol().headS3Object(bucketName, keyPath); + + isFile(keyPath, key); // TODO: return the specified range bytes of this object. } catch (OMException ex) { AUDIT.logReadFailure( @@ -536,6 +540,22 @@ public Response head( return response.build(); } + private void isFile(String keyPath, OzoneKey key) throws OMException { + /* + Necessary for directories in buckets with FSO layout. + Intended for apps which use Hadoop S3A. + Example of such app is Trino (through Hive connector). + */ + boolean isFsoDirCreationEnabled = ozoneConfiguration + .getBoolean(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, + OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT); + if (isFsoDirCreationEnabled && + !key.isFile() && + !keyPath.endsWith("/")) { + throw new OMException(ResultCodes.KEY_NOT_FOUND); + } + } + /** * Abort multipart upload request. * @param bucket diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 1972fb7646c1..34c9a46096e9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -151,7 +151,7 @@ public void close() throws IOException { System.currentTimeMillis(), System.currentTimeMillis(), new ArrayList<>(), replicationConfig, metadata, null, - () -> readKey(key) + () -> readKey(key), true )); super.close(); } @@ -183,7 +183,7 @@ public void close() throws IOException { System.currentTimeMillis(), System.currentTimeMillis(), new ArrayList<>(), finalReplicationCon, metadata, null, - () -> readKey(key) + () -> readKey(key), true )); super.close(); } @@ -215,7 +215,8 @@ public OzoneKey headObject(String key) throws IOException { ozoneKeyDetails.getDataSize(), ozoneKeyDetails.getCreationTime().toEpochMilli(), ozoneKeyDetails.getModificationTime().toEpochMilli(), - ozoneKeyDetails.getReplicationConfig()); + ozoneKeyDetails.getReplicationConfig(), + ozoneKeyDetails.isFile()); } else { throw new OMException(ResultCodes.KEY_NOT_FOUND); } @@ -446,4 +447,17 @@ public void setReplicationConfig(ReplicationConfig replicationConfig) { public ReplicationConfig getReplicationConfig() { return this.replicationConfig; } + + @Override + public void createDirectory(String keyName) throws IOException { + keyDetails.put(keyName, new OzoneKeyDetails( + getVolumeName(), + getName(), + keyName, + 0, + System.currentTimeMillis(), + System.currentTimeMillis(), + new ArrayList<>(), replicationConfig, new HashMap<>(), null, + () -> readKey(keyName), false)); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 03c0751dde73..150b44436b24 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -40,9 +40,12 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import org.mockito.Mockito; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.mockito.Mockito.doReturn; @@ -218,4 +221,26 @@ private void setDefaultHeader() { doReturn(CONTENT_ENCODING1) .when(headers).getHeaderString("Content-Encoding"); } + + @Test + public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash() + throws IOException { + // GIVEN + final String bucketName = "b1"; + final String keyPath = "keyDir"; + OzoneConfiguration config = new OzoneConfiguration(); + config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); + rest.setOzoneConfiguration(config); + OzoneBucket bucket = client.getObjectStore().getS3Bucket(bucketName); + bucket.createDirectory(keyPath); + + // WHEN + final OS3Exception ex = + Assertions.assertThrows(OS3Exception.class, + () -> rest.get(bucketName, keyPath, null, 0, null)); + + // THEN + Assertions.assertEquals(NO_SUCH_KEY.getCode(), ex.getCode()); + bucket.deleteKey(keyPath); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index 9080f06310b4..2e463e3d4a5b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -36,10 +36,14 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; + import org.apache.commons.lang3.RandomStringUtils; +import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.jupiter.api.Assertions; /** * Test head object. @@ -101,4 +105,88 @@ public void testHeadFailByBadName() throws Exception { Assert.assertEquals(HTTP_NOT_FOUND, ex.getHttpCode()); } } + + @Test + public void testHeadWhenKeyIsAFileAndKeyPathDoesNotEndWithASlash() + throws IOException, OS3Exception { + // GIVEN + final String keyPath = "keyDir"; + OzoneConfiguration config = new OzoneConfiguration(); + config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); + keyEndpoint.setOzoneConfiguration(config); + String keyContent = "content"; + OzoneOutputStream out = bucket.createKey(keyPath, + keyContent.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>()); + out.write(keyContent.getBytes(UTF_8)); + out.close(); + + // WHEN + final Response response = keyEndpoint.head(bucketName, keyPath); + + // THEN + Assertions.assertEquals(HttpStatus.SC_OK, response.getStatus()); + bucket.deleteKey(keyPath); + } + + @Test + public void testHeadWhenKeyIsDirectoryAndKeyPathDoesNotEndWithASlash() + throws IOException, OS3Exception { + // GIVEN + final String keyPath = "keyDir"; + OzoneConfiguration config = new OzoneConfiguration(); + config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); + keyEndpoint.setOzoneConfiguration(config); + bucket.createDirectory(keyPath); + + // WHEN + final Response response = keyEndpoint.head(bucketName, keyPath); + + // THEN + Assertions.assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatus()); + bucket.deleteKey(keyPath); + } + + @Test + public void testHeadWhenKeyIsDirectoryAndKeyPathEndsWithASlash() + throws IOException, OS3Exception { + // GIVEN + final String keyPath = "keyDir/"; + OzoneConfiguration config = new OzoneConfiguration(); + config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); + keyEndpoint.setOzoneConfiguration(config); + bucket.createDirectory(keyPath); + + // WHEN + final Response response = keyEndpoint.head(bucketName, keyPath); + + // THEN + Assertions.assertEquals(HttpStatus.SC_OK, response.getStatus()); + bucket.deleteKey(keyPath); + } + + @Test + public void testHeadWhenKeyIsAFileAndKeyPathEndsWithASlash() + throws IOException, OS3Exception { + // GIVEN + final String keyPath = "keyFile"; + OzoneConfiguration config = new OzoneConfiguration(); + config.set(OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED, "true"); + keyEndpoint.setOzoneConfiguration(config); + String keyContent = "content"; + OzoneOutputStream out = bucket.createKey(keyPath, + keyContent.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>()); + out.write(keyContent.getBytes(UTF_8)); + out.close(); + + // WHEN + final Response response = keyEndpoint.head(bucketName, keyPath + "/"); + + // THEN + Assertions.assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatus()); + bucket.deleteKey(keyPath); + } }