From f0267c58b843144bcd52de2353e543524d188710 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Mahadevuni Date: Mon, 26 Apr 2021 12:30:11 +0530 Subject: [PATCH] Fix S3 directory detection based on ContentType header Cherry pick of https://github.com/trinodb/trino/pull/6992 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Ślizak --- .../presto/hive/s3/PrestoS3FileSystem.java | 7 ++++--- .../presto/hive/s3/TestPrestoS3FileSystem.java | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java index 64ee3b24e9121..102531e2a42f1 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java @@ -56,6 +56,7 @@ import com.google.common.collect.AbstractSequentialIterator; import com.google.common.collect.Iterators; import com.google.common.io.Closer; +import com.google.common.net.MediaType; import io.airlift.units.DataSize; import io.airlift.units.Duration; import org.apache.hadoop.conf.Configurable; @@ -155,8 +156,6 @@ public class PrestoS3FileSystem extends ExtendedFileSystem { - static final String S3_DIRECTORY_OBJECT_CONTENT_TYPE = "application/x-directory"; - private static final Logger log = Logger.get(PrestoS3FileSystem.class); private static final PrestoS3FileSystemStats STATS = new PrestoS3FileSystemStats(); private static final PrestoS3FileSystemMetricCollector METRIC_COLLECTOR = new PrestoS3FileSystemMetricCollector(STATS); @@ -166,6 +165,7 @@ public class PrestoS3FileSystem private static final String PATH_SEPARATOR = "/"; private static final Duration BACKOFF_MIN_SLEEP = new Duration(1, SECONDS); private static final int HTTP_RANGE_NOT_SATISFIABLE = 416; + private static final MediaType DIRECTORY_MEDIA_TYPE = MediaType.create("application", "x-directory"); private URI uri; private Path workingDirectory; @@ -359,7 +359,8 @@ public FileStatus getFileStatus(Path path) return new FileStatus( getObjectSize(path, metadata), - S3_DIRECTORY_OBJECT_CONTENT_TYPE.equals(metadata.getContentType()), + // Some directories (e.g. uploaded through S3 GUI) return a charset in the Content-Type header + MediaType.parse(metadata.getContentType()).is(DIRECTORY_MEDIA_TYPE), 1, BLOCK_SIZE.toBytes(), lastModifiedTime(metadata), diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java index 90aeccd15b05a..6e8e9eba71aaf 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java @@ -66,7 +66,6 @@ import java.util.concurrent.atomic.AtomicInteger; import static com.facebook.airlift.testing.Assertions.assertInstanceOf; -import static com.facebook.presto.hive.s3.PrestoS3FileSystem.S3_DIRECTORY_OBJECT_CONTENT_TYPE; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ACCESS_KEY; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ACL_TYPE; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_CREDENTIALS_PROVIDER; @@ -635,8 +634,7 @@ public void testFullBucketOwnerControlAcl() } } - @Test - public void testEmptyDirectory() + private void testEmptyDirectoryWithContentType(String s3ObjectContentType) throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { @@ -647,7 +645,7 @@ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetada { if (getObjectMetadataRequest.getKey().equals("empty-dir/")) { ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentType(S3_DIRECTORY_OBJECT_CONTENT_TYPE); + objectMetadata.setContentType(s3ObjectContentType); return objectMetadata; } return super.getObjectMetadata(getObjectMetadataRequest); @@ -661,6 +659,14 @@ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetada } } + @Test + public void testEmptyDirectory() + throws Exception + { + testEmptyDirectoryWithContentType("application/x-directory"); + testEmptyDirectoryWithContentType("application/x-directory; charset=UTF-8"); + } + @Test public void testPrestoS3InputStreamEOS() throws Exception {