diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursor.java index 6aa1613c11351..c051efd885b39 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursor.java @@ -84,7 +84,7 @@ import static java.lang.Math.min; import static java.util.Objects.requireNonNull; -class GenericHiveRecordCursor +public class GenericHiveRecordCursor implements RecordCursor { private static final DateTimeZone JVM_TIME_ZONE = DateTimeZone.getDefault(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index 68d6a52342d92..b962526104608 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -40,6 +40,7 @@ import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory; import com.facebook.presto.hive.rule.HivePlanOptimizerProvider; import com.facebook.presto.hive.s3.PrestoS3ClientFactory; +import com.facebook.presto.hive.s3select.S3SelectRecordCursorProvider; import com.facebook.presto.orc.CachingStripeMetadataSource; import com.facebook.presto.orc.DwrfAwareStripeMetadataSourceFactory; import com.facebook.presto.orc.EncryptionLibrary; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 0ca6afb9c30d0..e347f33b7ef20 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -375,7 +375,7 @@ public static Optional getCompressionCodec(TextInputFormat inp return (Class>) clazz.asSubclass(InputFormat.class); } - static String getInputFormatName(Properties schema) + public static String getInputFormatName(Properties schema) { String name = schema.getProperty(FILE_INPUT_FORMAT); checkCondition(name != null, HIVE_INVALID_METADATA, "Table or partition is missing Hive input format property: %s", FILE_INPUT_FORMAT); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java index ee58cdfecb09e..ee1497a63f1ad 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/StoragePartitionLoader.java @@ -75,9 +75,9 @@ import static com.facebook.presto.hive.NestedDirectoryPolicy.FAIL; import static com.facebook.presto.hive.NestedDirectoryPolicy.IGNORED; import static com.facebook.presto.hive.NestedDirectoryPolicy.RECURSE; -import static com.facebook.presto.hive.S3SelectPushdown.shouldEnablePushdownForTable; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.MetastoreUtil.getPartitionLocation; +import static com.facebook.presto.hive.s3select.S3SelectPushdown.shouldEnablePushdownForTable; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectCsvRecordReader.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectCsvRecordReader.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/S3SelectCsvRecordReader.java rename to presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectCsvRecordReader.java index bce0117a073f7..366027c592e04 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectCsvRecordReader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectCsvRecordReader.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.amazonaws.services.s3.model.CSVInput; import com.amazonaws.services.s3.model.CSVOutput; @@ -20,6 +20,7 @@ import com.amazonaws.services.s3.model.InputSerialization; import com.amazonaws.services.s3.model.OutputSerialization; import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.s3.PrestoS3ClientFactory; import com.facebook.presto.hive.s3.PrestoS3FileSystem; import com.facebook.presto.spi.PrestoException; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectLineRecordReader.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java rename to presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectLineRecordReader.java index cea81f6df5dda..1c193f3b9abc7 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectLineRecordReader.java @@ -11,10 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.s3.HiveS3Config; import com.facebook.presto.hive.s3.PrestoS3ClientFactory; import com.facebook.presto.hive.s3.PrestoS3SelectClient; @@ -217,7 +218,7 @@ String getFieldDelimiter(Properties schema) * For example, "Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403 ..." */ @VisibleForTesting - static class UnrecoverableS3OperationException + public static class UnrecoverableS3OperationException extends RuntimeException { public UnrecoverableS3OperationException(String bucket, String key, Throwable cause) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectPushdown.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java rename to presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectPushdown.java index 1470c8a72473f..85956817e19d9 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectPushdown.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectPushdown.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.facebook.airlift.log.Logger; import com.facebook.presto.hive.metastore.Column; @@ -125,7 +125,7 @@ private static boolean isS3Storage(String path) return SUPPORTED_S3_PREFIXES.stream().anyMatch(path::startsWith); } - static boolean shouldEnablePushdownForTable(ConnectorSession session, Table table, String path, Optional optionalPartition) + public static boolean shouldEnablePushdownForTable(ConnectorSession session, Table table, String path, Optional optionalPartition) { if (!isS3SelectPushdownEnabled(session)) { return false; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursor.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursor.java rename to presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursor.java index 3bfe2e311214f..75fa758ecba71 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursor.java @@ -11,9 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.hive.GenericHiveRecordCursor; +import com.facebook.presto.hive.HiveColumnHandle; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; @@ -35,7 +37,7 @@ import static org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_DDL; -class S3SelectRecordCursor +public class S3SelectRecordCursor extends GenericHiveRecordCursor { private static final String THRIFT_STRUCT = "struct"; @@ -59,7 +61,7 @@ public S3SelectRecordCursor( // since s3select only returns the required column, not the whole columns // we need to update the split schema to include only the required columns // otherwise, Serde could not deserialize output from s3select to row data correctly - static Properties updateSplitSchema(Properties splitSchema, List columns) + public static Properties updateSplitSchema(Properties splitSchema, List columns) { requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(columns, "columns is null"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursorProvider.java similarity index 91% rename from presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java rename to presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursorProvider.java index 0b08f9880e90a..60ca82431c602 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/s3select/S3SelectRecordCursorProvider.java @@ -11,10 +11,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.hive.HdfsEnvironment; +import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.HiveFileSplit; +import com.facebook.presto.hive.HiveRecordCursorProvider; +import com.facebook.presto.hive.IonSqlQueryBuilder; import com.facebook.presto.hive.s3.PrestoS3ClientFactory; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java index 59fd5d1edbac3..93076f4ec6e44 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java @@ -19,7 +19,7 @@ import com.facebook.presto.hive.HiveSplitPartitionInfo; import com.facebook.presto.hive.InternalHiveSplit; import com.facebook.presto.hive.InternalHiveSplit.InternalHiveBlock; -import com.facebook.presto.hive.S3SelectPushdown; +import com.facebook.presto.hive.s3select.S3SelectPushdown; import com.facebook.presto.spi.HostAddress; import com.facebook.presto.spi.schedule.NodeSelectionStrategy; import com.google.common.collect.ImmutableList; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectLineRecordReader.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectLineRecordReader.java similarity index 87% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectLineRecordReader.java rename to presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectLineRecordReader.java index 80cbef85a67a4..2a610f56959cf 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectLineRecordReader.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectLineRecordReader.java @@ -11,13 +11,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import org.testng.annotations.Test; import java.io.IOException; -import static com.facebook.presto.hive.S3SelectLineRecordReader.UnrecoverableS3OperationException; +import static com.facebook.presto.hive.s3select.S3SelectLineRecordReader.UnrecoverableS3OperationException; public class TestS3SelectLineRecordReader { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectPushdown.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java similarity index 97% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectPushdown.java rename to presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java index 545ee30809e11..294daa6df52e0 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectPushdown.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectPushdown.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectRecordCursor.java b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectRecordCursor.java similarity index 98% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectRecordCursor.java rename to presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectRecordCursor.java index 2f9e6d295c0f7..5affa3f89b0fe 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestS3SelectRecordCursor.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/s3select/TestS3SelectRecordCursor.java @@ -11,11 +11,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.hive.s3select; import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.common.type.TestingTypeManager; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.HiveType; import com.google.common.collect.ImmutableList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -34,7 +36,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveType.HIVE_INT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; -import static com.facebook.presto.hive.S3SelectRecordCursor.updateSplitSchema; +import static com.facebook.presto.hive.s3select.S3SelectRecordCursor.updateSplitSchema; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.joining;