diff --git a/pom.xml b/pom.xml index c6fd548ae0aeb..2824fa855550f 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ presto-rcfile presto-hive presto-hive-hadoop2 + presto-hive-metastore presto-teradata-functions presto-example-http presto-local-file @@ -375,6 +376,12 @@ test-jar + + com.facebook.presto + presto-hive-metastore + ${project.version} + + com.facebook.presto presto-matching diff --git a/presto-geospatial/pom.xml b/presto-geospatial/pom.xml index fc6067d3a650a..85ea9f36059ab 100644 --- a/presto-geospatial/pom.xml +++ b/presto-geospatial/pom.xml @@ -94,6 +94,12 @@ test + + com.facebook.presto + presto-hive-metastore + test + + com.facebook.presto presto-tpch diff --git a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java index 90df983e26fc5..7c40638014ecf 100644 --- a/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java +++ b/presto-geospatial/src/test/java/com/facebook/presto/plugin/geospatial/TestSpatialJoins.java @@ -20,6 +20,7 @@ import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveHdfsConfiguration; import com.facebook.presto.hive.HivePlugin; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.file.FileHiveMetastore; @@ -98,8 +99,9 @@ private static DistributedQueryRunner createQueryRunner() File baseDir = queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data").toFile(); HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig), ImmutableSet.of()); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); FileHiveMetastore metastore = new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); metastore.createDatabase(Database.builder() diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index 20a75d66bf686..4c0632b0389bc 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -22,6 +22,11 @@ presto-hive + + com.facebook.presto + presto-hive-metastore + + com.google.guava guava diff --git a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java index a1834d3e64bfb..8e1fa073ba3d3 100644 --- a/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java +++ b/presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystemS3.java @@ -44,12 +44,12 @@ protected void setup(String host, int port, String databaseName, String awsAcces super.setup(host, port, databaseName, this::createHdfsConfiguration, s3SelectPushdownEnabled); } - HdfsConfiguration createHdfsConfiguration(HiveClientConfig config) + HdfsConfiguration createHdfsConfiguration(HiveClientConfig config, MetastoreClientConfig metastoreConfig) { S3ConfigurationUpdater s3Config = new PrestoS3ConfigurationUpdater(new HiveS3Config() .setS3AwsAccessKey(awsAccessKey) .setS3AwsSecretKey(awsSecretKey)); - return new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, s3Config, ignored -> {}), ImmutableSet.of()); + return new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreConfig, s3Config, ignored -> {}), ImmutableSet.of()); } @Override diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml new file mode 100644 index 0000000000000..c67b73edffcd1 --- /dev/null +++ b/presto-hive-metastore/pom.xml @@ -0,0 +1,177 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.230-SNAPSHOT + + + presto-hive-metastore + presto-hive-metastore + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + + + + com.facebook.airlift + configuration + + + + io.airlift + units + + + + com.facebook.presto.hive + hive-apache + + + + org.apache.thrift + libthrift + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.inject + javax.inject + + + + javax.validation + validation-api + + + + com.facebook.hive + hive-dwrf + + + + org.openjdk.jol + jol-core + + + + com.fasterxml.jackson.core + jackson-annotations + + + + io.airlift + joda-to-java-time-bridge + runtime + + + + com.facebook.presto.hadoop + hadoop-apache2 + provided + + + + com.google.code.findbugs + jsr305 + true + provided + + + + org.jetbrains + annotations + provided + + + + + org.openjdk.jmh + jmh-core + test + + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-parser + test + + + + org.testng + testng + 6.9.6 + test + + + + com.facebook.airlift + testing + test + + + + com.facebook.airlift + json + test + + + + org.slf4j + slf4j-jdk14 + test + + + + org.slf4j + jcl-over-slf4j + test + + + + + + ci + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsConfiguration.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java similarity index 99% rename from presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java index 63a4692fe03ef..9ca2aaf9546dc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HdfsEnvironment.java @@ -45,7 +45,7 @@ public class HdfsEnvironment @Inject public HdfsEnvironment( HdfsConfiguration hdfsConfiguration, - HiveClientConfig config, + MetastoreClientConfig config, HdfsAuthentication hdfsAuthentication) { this.hdfsConfiguration = requireNonNull(hdfsConfiguration, "hdfsConfiguration is null"); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBasicStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java index ed0671cd1cc3a..d832fadbd12be 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveBucketProperty.java @@ -24,7 +24,7 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java similarity index 67% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java index 4e4523d041859..b56dc1d798768 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.hive; -import com.facebook.presto.spi.PrestoException; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -32,19 +31,10 @@ import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.data.JsonSerDe; -import java.util.List; - -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.lang.String.format; import static java.util.Objects.requireNonNull; public enum HiveStorageFormat @@ -127,45 +117,4 @@ public DataSize getEstimatedWriterSystemMemoryUsage() { return estimatedWriterSystemMemoryUsage; } - - public void validateColumns(List handles) - { - if (this == AVRO) { - for (HiveColumnHandle handle : handles) { - if (!handle.isPartitionKey()) { - validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName()); - } - } - } - } - - private static void validateAvroType(TypeInfo type, String columnName) - { - if (type.getCategory() == Category.MAP) { - TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); - if ((keyType.getCategory() != Category.PRIMITIVE) || - (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveCategory.STRING)) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); - } - } - else if (type.getCategory() == Category.PRIMITIVE) { - PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); - if (primitive == PrimitiveCategory.BYTE) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); - } - if (primitive == PrimitiveCategory.SHORT) { - throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); - } - } - } - - private static PrimitiveTypeInfo primitiveTypeInfo(TypeInfo typeInfo) - { - return (PrimitiveTypeInfo) typeInfo; - } - - private static MapTypeInfo mapTypeInfo(TypeInfo typeInfo) - { - return (MapTypeInfo) typeInfo; - } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java index 935f131a69eb0..91686c4dffc50 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveType.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveType.java @@ -233,7 +233,7 @@ private static TypeSignature getTypeSignature(TypeInfo typeInfo) List structFieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); List structFieldNames = structTypeInfo.getAllStructFieldNames(); if (structFieldTypeInfos.size() != structFieldNames.size()) { - throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, format("Invalid Hive struct type: %s", typeInfo)); + throw new PrestoException(MetastoreErrorCode.HIVE_INVALID_METADATA, format("Invalid Hive struct type: %s", typeInfo)); } ImmutableList.Builder typeSignatureBuilder = ImmutableList.builder(); for (int i = 0; i < structFieldTypeInfos.size(); i++) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java similarity index 93% rename from presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java index c22cd0f6bfbef..8ce4a32781f32 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeName.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/HiveTypeName.java @@ -21,8 +21,7 @@ public final class HiveTypeName { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(HivePartitionKey.class).instanceSize() + - ClassLayout.parseClass(String.class).instanceSize(); + private static final int INSTANCE_SIZE = ClassLayout.parseClass(String.class).instanceSize(); private final String value; diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java new file mode 100644 index 0000000000000..353a9d84b18d9 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.airlift.configuration.Config; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; + +import javax.validation.constraints.NotNull; + +import java.util.concurrent.TimeUnit; + +public class MetastoreClientConfig +{ + private HostAndPort metastoreSocksProxy; + private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS); + private boolean verifyChecksum = true; + + public HostAndPort getMetastoreSocksProxy() + { + return metastoreSocksProxy; + } + + @Config("hive.metastore.thrift.client.socks-proxy") + public MetastoreClientConfig setMetastoreSocksProxy(HostAndPort metastoreSocksProxy) + { + this.metastoreSocksProxy = metastoreSocksProxy; + return this; + } + + @NotNull + public Duration getMetastoreTimeout() + { + return metastoreTimeout; + } + + @Config("hive.metastore-timeout") + public MetastoreClientConfig setMetastoreTimeout(Duration metastoreTimeout) + { + this.metastoreTimeout = metastoreTimeout; + return this; + } + + public boolean isVerifyChecksum() + { + return verifyChecksum; + } + + @Config("hive.dfs.verify-checksum") + public MetastoreClientConfig setVerifyChecksum(boolean verifyChecksum) + { + this.verifyChecksum = verifyChecksum; + return this; + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java new file mode 100644 index 0000000000000..0e0cf95a1c1e2 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientModule.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreClientModule + implements Module +{ + public MetastoreClientModule() + { + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(MetastoreClientConfig.class); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java new file mode 100644 index 0000000000000..05a8c1d463eca --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreErrorCode.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.ErrorType; + +import static com.facebook.presto.spi.ErrorType.EXTERNAL; +import static com.facebook.presto.spi.ErrorType.USER_ERROR; + +public enum MetastoreErrorCode + implements ErrorCodeSupplier +{ + HIVE_METASTORE_ERROR(0, EXTERNAL), + HIVE_TABLE_OFFLINE(2, USER_ERROR), + HIVE_PARTITION_OFFLINE(6, USER_ERROR), + HIVE_INVALID_METADATA(12, EXTERNAL), + HIVE_FILESYSTEM_ERROR(16, EXTERNAL), + HIVE_UNSUPPORTED_FORMAT(19, EXTERNAL), + /* Shared error code with HiveErrorCode */; + + private final ErrorCode errorCode; + + public static final int ERROR_CODE_MASK = 0x0100_0000; + + MetastoreErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + ERROR_CODE_MASK, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java index 35d1b56a779c2..6f969c0a325a6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionOfflineException.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_OFFLINE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_PARTITION_OFFLINE; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/PartitionStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/PartitionStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/PartitionStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java similarity index 95% rename from presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java index b6188fda97896..174b21784ce91 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/TableOfflineException.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TableOfflineException.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_OFFLINE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_TABLE_OFFLINE; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/TypeTranslator.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/TypeTranslator.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/TypeTranslator.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/TypeTranslator.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/GenericExceptionAction.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HdfsAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/HiveMetastoreAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/authentication/NoHiveMetastoreAuthentication.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/BooleanStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Column.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Column.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Column.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Database.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Database.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Database.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DateStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DecimalStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/DoubleStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/ExtendedHiveMetastore.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HiveColumnStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/HivePrivilegeInfo.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/IntegerStatistics.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java similarity index 88% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java index 8bb0e64bf7201..27e84cd2de167 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java @@ -17,9 +17,12 @@ import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.PartitionOfflineException; import com.facebook.presto.hive.TableOfflineException; +import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; import com.google.common.collect.ImmutableList; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -33,13 +36,13 @@ import java.util.Optional; import java.util.Properties; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveSplitManager.PRESTO_OFFLINE; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import static org.apache.hadoop.hive.common.FileUtils.unescapePathName; import static org.apache.hadoop.hive.metastore.MetaStoreUtils.typeToThriftType; import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT; @@ -57,10 +60,34 @@ public class MetastoreUtil { + public static final String PRESTO_OFFLINE = "presto_offline"; + private MetastoreUtil() { } + public static boolean isArrayType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); + } + + public static boolean isMapType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.MAP); + } + + public static boolean isRowType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ROW); + } + + public static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) + { + if (!condition) { + throw new PrestoException(errorCode, format(formatString, args)); + } + } + public static Properties getHiveSchema(Table table) { // Mimics function in Hive: MetaStoreUtils.getTableMetadata(Table) @@ -335,6 +362,29 @@ public static void renameFile(FileSystem fileSystem, Path source, Path target) } } + public static List toPartitionValues(String partitionName) + { + // mimics Warehouse.makeValsFromName + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + int start = 0; + while (true) { + while (start < partitionName.length() && partitionName.charAt(start) != '=') { + start++; + } + start++; + int end = start; + while (end < partitionName.length() && partitionName.charAt(end) != '/') { + end++; + } + if (start > partitionName.length()) { + break; + } + resultBuilder.add(unescapePathName(partitionName.substring(start, end))); + start = end + 1; + } + return resultBuilder.build(); + } + private static String getRenameErrorMessage(Path source, Path target) { return format("Error moving data files from %s to final location %s", source, target); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Partition.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Partition.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Partition.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java similarity index 95% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java index 8f881775fe6db..7ea5e73a29b41 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PartitionWithStatistics.java @@ -15,7 +15,7 @@ import com.facebook.presto.hive.PartitionStatistics; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrestoTableType.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/PrincipalPrivileges.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java similarity index 97% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java index 019ed67aded6d..61ddf2c26be9a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/SortingColumn.java @@ -22,7 +22,7 @@ import java.util.Objects; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.spi.block.SortOrder.ASC_NULLS_FIRST; import static com.facebook.presto.spi.block.SortOrder.DESC_NULLS_LAST; import static com.google.common.base.MoreObjects.toStringHelper; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Storage.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Storage.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Storage.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Storage.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java similarity index 96% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java index 024f643280367..b29eb45a8a178 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/StorageFormat.java @@ -22,8 +22,8 @@ import java.util.Objects; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/Table.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/Table.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/Table.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveCluster.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClient.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastoreClient.java diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java similarity index 100% rename from presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java rename to presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/Transport.java diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestMetastoreClientConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestMetastoreClientConfig.java new file mode 100644 index 0000000000000..ac74eaf872000 --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestMetastoreClientConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive; + +import com.facebook.airlift.configuration.testing.ConfigAssertions; +import com.google.common.collect.ImmutableMap; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class TestMetastoreClientConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(MetastoreClientConfig.class) + .setMetastoreSocksProxy(null) + .setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS)) + .setVerifyChecksum(true)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.thrift.client.socks-proxy", "localhost:1080") + .put("hive.metastore-timeout", "20s") + .put("hive.dfs.verify-checksum", "false") + .build(); + + MetastoreClientConfig expected = new MetastoreClientConfig() + .setMetastoreSocksProxy(HostAndPort.fromParts("localhost", 1080)) + .setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS)) + .setVerifyChecksum(false); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestPartitionOfflineException.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java similarity index 100% rename from presto-hive/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java rename to presto-hive-metastore/src/test/java/com/facebook/presto/hive/TestTableOfflineException.java diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index e9dfdadfdf512..0d78922f8fafe 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -75,6 +75,11 @@ presto-memory-context + + com.facebook.presto + presto-hive-metastore + + com.facebook.presto presto-rcfile diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java index f89b955cd2d97..088de7b57f832 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/BackgroundHiveSplitLoader.java @@ -64,17 +64,17 @@ import static com.facebook.presto.hive.HiveBucketing.getVirtualBucketNumber; import static com.facebook.presto.hive.HiveColumnHandle.pathColumnHandle; import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_BUCKET_FILES; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; import static com.facebook.presto.hive.HiveSessionProperties.isForceLocalScheduling; -import static com.facebook.presto.hive.HiveUtil.checkCondition; import static com.facebook.presto.hive.HiveUtil.getFooterCount; import static com.facebook.presto.hive.HiveUtil.getHeaderCount; import static com.facebook.presto.hive.HiveUtil.getInputFormat; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.S3SelectPushdown.shouldEnablePushdownForTable; +import static com.facebook.presto.hive.metastore.MetastoreUtil.checkCondition; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.MetastoreUtil.getPartitionLocation; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java index 81a3dae489c3e..565d2769bf9af 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursorProvider.java @@ -31,7 +31,7 @@ import java.util.Optional; import java.util.Properties; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class GenericHiveRecordCursorProvider diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java index 62e61d808c396..991b364614cd5 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationInitializer.java @@ -61,19 +61,19 @@ public class HdfsConfigurationInitializer private int textMaxLineLength; @VisibleForTesting - public HdfsConfigurationInitializer(HiveClientConfig config) + public HdfsConfigurationInitializer(HiveClientConfig config, MetastoreClientConfig metastoreConfig) { - this(config, ignored -> {}, ignored -> {}); + this(config, metastoreConfig, ignored -> {}, ignored -> {}); } @Inject - public HdfsConfigurationInitializer(HiveClientConfig config, S3ConfigurationUpdater s3ConfigurationUpdater, GcsConfigurationInitializer gcsConfigurationInitialize) + public HdfsConfigurationInitializer(HiveClientConfig config, MetastoreClientConfig metastoreConfig, S3ConfigurationUpdater s3ConfigurationUpdater, GcsConfigurationInitializer gcsConfigurationInitialize) { requireNonNull(config, "config is null"); checkArgument(config.getDfsTimeout().toMillis() >= 1, "dfsTimeout must be at least 1 ms"); checkArgument(toIntExact(config.getTextMaxLineLength().toBytes()) >= 1, "textMaxLineLength must be at least 1 byte"); - this.socksProxy = config.getMetastoreSocksProxy(); + this.socksProxy = metastoreConfig.getMetastoreSocksProxy(); this.ipcPingInterval = config.getIpcPingInterval(); this.dfsTimeout = config.getDfsTimeout(); this.dfsConnectTimeout = config.getDfsConnectTimeout(); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java index ef28969bc3bc2..b4f574564e00d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveBucketing.java @@ -46,8 +46,8 @@ import java.util.stream.Collectors; import static com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveUtil.getRegularColumnHandles; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.slice.Slices.utf8Slice; import static java.lang.Double.doubleToLongBits; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 1d6068789d514..103c4f29c1054 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -21,7 +21,6 @@ import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; -import com.google.common.net.HostAndPort; import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.MaxDataSize; @@ -78,14 +77,11 @@ public class HiveClientConfig private long metastoreCacheMaximumSize = 10000; private long perTransactionMetastoreCacheMaximumSize = 1000; private int maxMetastoreRefreshThreads = 100; - private HostAndPort metastoreSocksProxy; - private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS); private Duration ipcPingInterval = new Duration(10, TimeUnit.SECONDS); private Duration dfsTimeout = new Duration(60, TimeUnit.SECONDS); private Duration dfsConnectTimeout = new Duration(500, TimeUnit.MILLISECONDS); private int dfsConnectMaxRetries = 5; - private boolean verifyChecksum = true; private String domainSocketPath; private S3FileSystemType s3FileSystemType = S3FileSystemType.PRESTO; @@ -453,31 +449,6 @@ public HiveClientConfig setMaxMetastoreRefreshThreads(int maxMetastoreRefreshThr return this; } - public HostAndPort getMetastoreSocksProxy() - { - return metastoreSocksProxy; - } - - @Config("hive.metastore.thrift.client.socks-proxy") - public HiveClientConfig setMetastoreSocksProxy(HostAndPort metastoreSocksProxy) - { - this.metastoreSocksProxy = metastoreSocksProxy; - return this; - } - - @NotNull - public Duration getMetastoreTimeout() - { - return metastoreTimeout; - } - - @Config("hive.metastore-timeout") - public HiveClientConfig setMetastoreTimeout(Duration metastoreTimeout) - { - this.metastoreTimeout = metastoreTimeout; - return this; - } - @Min(1) public int getMinPartitionBatchSize() { @@ -696,18 +667,6 @@ public HiveClientConfig setS3FileSystemType(S3FileSystemType s3FileSystemType) return this; } - public boolean isVerifyChecksum() - { - return verifyChecksum; - } - - @Config("hive.dfs.verify-checksum") - public HiveClientConfig setVerifyChecksum(boolean verifyChecksum) - { - this.verifyChecksum = verifyChecksum; - return this; - } - public boolean isUseOrcColumnNames() { return useOrcColumnNames; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index 18c130fc2a575..d73b113798e6b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -153,6 +153,7 @@ public void configure(Binder binder) fileWriterFactoryBinder.addBinding().to(RcFileFileWriterFactory.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(ParquetFileWriterConfig.class); + binder.install(new MetastoreClientModule()); } @ForHiveClient diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercers.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercers.java index 736f53dc3a163..6414f6bc8c36c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercers.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercers.java @@ -40,9 +40,9 @@ import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveUtil.extractStructFieldTypes; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.block.ColumnarArray.toColumnarArray; import static com.facebook.presto.spi.block.ColumnarMap.toColumnarMap; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java index 68b8ea9b1d580..ae95fd3126cb0 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveCoercionRecordCursor.java @@ -37,9 +37,9 @@ import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveUtil.extractStructFieldTypes; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static io.airlift.slice.Slices.utf8Slice; import static java.lang.Float.intBitsToFloat; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java index 321f3a23c9c84..962e82f03cc0b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveErrorCode.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.ErrorType; +import static com.facebook.presto.hive.MetastoreErrorCode.ERROR_CODE_MASK; import static com.facebook.presto.spi.ErrorType.EXTERNAL; import static com.facebook.presto.spi.ErrorType.INTERNAL_ERROR; import static com.facebook.presto.spi.ErrorType.USER_ERROR; @@ -24,26 +25,26 @@ public enum HiveErrorCode implements ErrorCodeSupplier { - HIVE_METASTORE_ERROR(0, EXTERNAL), + // HIVE_METASTORE_ERROR(0) moved to MetastoreErrorCode HIVE_CURSOR_ERROR(1, EXTERNAL), - HIVE_TABLE_OFFLINE(2, USER_ERROR), + // HIVE_TABLE_OFFLINE(2) moved to MetastoreErrorCode HIVE_CANNOT_OPEN_SPLIT(3, EXTERNAL), HIVE_FILE_NOT_FOUND(4, EXTERNAL), HIVE_UNKNOWN_ERROR(5, EXTERNAL), - HIVE_PARTITION_OFFLINE(6, USER_ERROR), + // HIVE_PARTITION_OFFLINE(6) moved to MetastoreErrorCode HIVE_BAD_DATA(7, EXTERNAL), HIVE_PARTITION_SCHEMA_MISMATCH(8, EXTERNAL), HIVE_MISSING_DATA(9, EXTERNAL), HIVE_INVALID_PARTITION_VALUE(10, EXTERNAL), HIVE_TIMEZONE_MISMATCH(11, EXTERNAL), - HIVE_INVALID_METADATA(12, EXTERNAL), + // HIVE_INVALID_METADATA(12) moved to MetastoreErrorCode HIVE_INVALID_VIEW_DATA(13, EXTERNAL), HIVE_DATABASE_LOCATION_ERROR(14, EXTERNAL), HIVE_PATH_ALREADY_EXISTS(15, EXTERNAL), - HIVE_FILESYSTEM_ERROR(16, EXTERNAL), + // HIVE_FILESYSTEM_ERROR(16) moved to MetastoreErrorCode // code HIVE_WRITER_ERROR(17) is deprecated HIVE_SERDE_NOT_FOUND(18, EXTERNAL), - HIVE_UNSUPPORTED_FORMAT(19, EXTERNAL), + // HIVE_UNSUPPORTED_FORMAT(19) moved to MetastoreErrorCode HIVE_PARTITION_READ_ONLY(20, USER_ERROR), HIVE_TOO_MANY_OPEN_PARTITIONS(21, USER_ERROR), HIVE_CONCURRENT_MODIFICATION_DETECTED(22, EXTERNAL), @@ -72,7 +73,7 @@ public enum HiveErrorCode HiveErrorCode(int code, ErrorType type) { - errorCode = new ErrorCode(code + 0x0100_0000, name(), type); + errorCode = new ErrorCode(code + ERROR_CODE_MASK, name(), type); } @Override diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index 9a61b750080be..be6bfa79a921e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -23,6 +23,7 @@ import com.facebook.presto.hive.metastore.HiveColumnStatistics; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.PrestoTableType; import com.facebook.presto.hive.metastore.PrincipalPrivileges; @@ -102,6 +103,11 @@ import com.google.common.collect.Sets; import io.airlift.slice.Slice; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.joda.time.DateTimeZone; import java.io.File; @@ -140,10 +146,8 @@ import static com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle; import static com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TIMEZONE_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HiveSessionProperties.getCompressionCodec; import static com.facebook.presto.hive.HiveSessionProperties.getHiveStorageFormat; @@ -184,19 +188,21 @@ import static com.facebook.presto.hive.HiveUtil.getPartitionKeyColumnHandles; import static com.facebook.presto.hive.HiveUtil.hiveColumnHandles; import static com.facebook.presto.hive.HiveUtil.schemaTableName; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypeForTemporaryTable; import static com.facebook.presto.hive.HiveUtil.translateHiveUnsupportedTypesForTemporaryTable; import static com.facebook.presto.hive.HiveUtil.verifyPartitionTypeSupported; import static com.facebook.presto.hive.HiveWriteUtils.checkTableIsWritable; import static com.facebook.presto.hive.HiveWriteUtils.isWritableType; import static com.facebook.presto.hive.HiveWriterFactory.getFileExtension; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.PartitionUpdate.UpdateMode.APPEND; import static com.facebook.presto.hive.PartitionUpdate.UpdateMode.NEW; import static com.facebook.presto.hive.PartitionUpdate.UpdateMode.OVERWRITE; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.toHivePrivilege; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; @@ -691,7 +697,7 @@ public void createTable(ConnectorSession session, ConnectorTableMetadata tableMe HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(tableMetadata.getProperties()); Map tableProperties = getEmptyTableProperties(tableMetadata, !partitionedBy.isEmpty(), new HdfsContext(session, schemaName, tableName)); - hiveStorageFormat.validateColumns(columnHandles); + validateColumns(hiveStorageFormat, columnHandles); Map columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName); List partitionColumns = partitionedBy.stream() @@ -769,7 +775,7 @@ public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List< translateHiveUnsupportedTypesForTemporaryTable(columns, typeManager), ImmutableSet.of(), typeTranslator); - storageFormat.validateColumns(columnHandles); + validateColumns(storageFormat, columnHandles); Table table = Table.builder() .setDatabaseName(schemaName) @@ -796,6 +802,47 @@ public ConnectorTableHandle createTemporaryTable(ConnectorSession session, List< return new HiveTableHandle(schemaName, tableName); } + private void validateColumns(HiveStorageFormat hiveStorageFormat, List handles) + { + if (hiveStorageFormat == HiveStorageFormat.AVRO) { + for (HiveColumnHandle handle : handles) { + if (!handle.isPartitionKey()) { + validateAvroType(handle.getHiveType().getTypeInfo(), handle.getName()); + } + } + } + } + + private static void validateAvroType(TypeInfo type, String columnName) + { + if (type.getCategory() == ObjectInspector.Category.MAP) { + TypeInfo keyType = mapTypeInfo(type).getMapKeyTypeInfo(); + if ((keyType.getCategory() != ObjectInspector.Category.PRIMITIVE) || + (primitiveTypeInfo(keyType).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING)) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s has a non-varchar map key, which is not supported by Avro", columnName)); + } + } + else if (type.getCategory() == ObjectInspector.Category.PRIMITIVE) { + PrimitiveObjectInspector.PrimitiveCategory primitive = primitiveTypeInfo(type).getPrimitiveCategory(); + if (primitive == PrimitiveObjectInspector.PrimitiveCategory.BYTE) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s is tinyint, which is not supported by Avro. Use integer instead.", columnName)); + } + if (primitive == PrimitiveObjectInspector.PrimitiveCategory.SHORT) { + throw new PrestoException(NOT_SUPPORTED, format("Column %s is smallint, which is not supported by Avro. Use integer instead.", columnName)); + } + } + } + + private static PrimitiveTypeInfo primitiveTypeInfo(TypeInfo typeInfo) + { + return (PrimitiveTypeInfo) typeInfo; + } + + private static MapTypeInfo mapTypeInfo(TypeInfo typeInfo) + { + return (MapTypeInfo) typeInfo; + } + private Map getEmptyTableProperties(ConnectorTableMetadata tableMetadata, boolean partitioned, HdfsContext hdfsContext) { Builder tableProperties = ImmutableMap.builder(); @@ -1058,7 +1105,7 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH partitionValuesList = metastore.getPartitionNames(handle.getSchemaName(), handle.getTableName()) .orElseThrow(() -> new TableNotFoundException(((HiveTableHandle) tableHandle).getSchemaTableName())) .stream() - .map(HiveUtil::toPartitionValues) + .map(MetastoreUtil::toPartitionValues) .collect(toImmutableList()); } @@ -1113,7 +1160,7 @@ public HiveOutputTableHandle beginCreateTable(ConnectorSession session, Connecto // unpartitioned tables ignore the partition storage format HiveStorageFormat actualStorageFormat = partitionedBy.isEmpty() ? tableStorageFormat : partitionStorageFormat; - actualStorageFormat.validateColumns(columnHandles); + validateColumns(actualStorageFormat, columnHandles); Map columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName); List partitionColumns = partitionedBy.stream() diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java index df18d4aa695a5..84fbeee09c530 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSplitManager.java @@ -54,12 +54,12 @@ import static com.facebook.presto.hive.BackgroundHiveSplitLoader.BucketSplitInfo.createBucketSplitInfo; import static com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TRANSACTION_NOT_FOUND; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; import static com.facebook.presto.hive.HiveSessionProperties.isOfflineDataDebugModeEnabled; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; @@ -78,7 +78,6 @@ public class HiveSplitManager implements ConnectorSplitManager { - public static final String PRESTO_OFFLINE = "presto_offline"; public static final String OBJECT_NOT_READABLE = "object_not_readable"; private final HiveTransactionManager hiveTransactionManager; @@ -176,7 +175,7 @@ public ConnectorSplitSource getSplits( .orElseThrow(() -> new TableNotFoundException(tableName)); if (!isOfflineDataDebugModeEnabled(session)) { - // verify table is not marked as non-readable + // verify table is not marked as non-readable String tableNotReadable = table.getParameters().get(OBJECT_NOT_READABLE); if (!isNullOrEmpty(tableNotReadable)) { throw new HiveNotReadableException(tableName, Optional.empty(), tableNotReadable); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java index e08726d46dd1d..50727d52efe95 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTypeTranslator.java @@ -37,9 +37,9 @@ import static com.facebook.presto.hive.HiveType.HIVE_SHORT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveType.HIVE_TIMESTAMP; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 1cd610381da77..28b3539f194d4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -21,7 +21,6 @@ import com.facebook.presto.orc.OrcReader; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableHandle; -import com.facebook.presto.spi.ErrorCodeSupplier; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.RecordCursor; import com.facebook.presto.spi.SchemaTableName; @@ -108,13 +107,14 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_BAD_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_MISSING_COLUMN_NAMES; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_VIEW_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_SERDE_NOT_FOUND; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_BUCKETING_IS_IGNORED; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; +import static com.facebook.presto.hive.metastore.MetastoreUtil.checkCondition; import static com.facebook.presto.hive.util.ConfigurationUtils.copy; import static com.facebook.presto.hive.util.ConfigurationUtils.toJobConf; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; @@ -152,7 +152,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; -import static org.apache.hadoop.hive.common.FileUtils.unescapePathName; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; import static org.apache.hadoop.hive.serde.serdeConstants.DECIMAL_TYPE_NAME; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; @@ -163,7 +162,6 @@ public final class HiveUtil { private static final Pattern DEFAULT_HIVE_COLUMN_NAME_PATTERN = Pattern.compile("_col\\d+"); - public static final String PRESTO_VIEW_FLAG = "presto_view"; private static final String VIEW_PREFIX = "/* Presto View: "; @@ -646,21 +644,6 @@ public static Optional getDecimalType(String hiveTypeName) } } - public static boolean isArrayType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); - } - - public static boolean isMapType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.MAP); - } - - public static boolean isRowType(Type type) - { - return type.getTypeSignature().getBase().equals(StandardTypes.ROW); - } - public static boolean isStructuralType(Type type) { String baseName = type.getTypeSignature().getBase(); @@ -870,42 +853,12 @@ public static List getPartitionKeyColumnHandles(Table table) return columns.build(); } - public static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) - { - if (!condition) { - throw new PrestoException(errorCode, format(formatString, args)); - } - } - @Nullable public static String columnExtraInfo(boolean partitionKey) { return partitionKey ? "partition key" : null; } - public static List toPartitionValues(String partitionName) - { - // mimics Warehouse.makeValsFromName - ImmutableList.Builder resultBuilder = ImmutableList.builder(); - int start = 0; - while (true) { - while (start < partitionName.length() && partitionName.charAt(start) != '=') { - start++; - } - start++; - int end = start; - while (end < partitionName.length() && partitionName.charAt(end) != '/') { - end++; - } - if (start > partitionName.length()) { - break; - } - resultBuilder.add(unescapePathName(partitionName.substring(start, end))); - start = end + 1; - } - return resultBuilder.build(); - } - public static String getPrefilledColumnValue(HiveColumnHandle columnHandle, HivePartitionKey partitionKey, Path path, OptionalInt bucketNumber) { if (partitionKey != null) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index c9492ca352876..bba5b5e086500 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -110,18 +110,18 @@ import java.util.concurrent.TimeUnit; import static com.facebook.presto.hive.HiveErrorCode.HIVE_DATABASE_LOCATION_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_SERDE_NOT_FOUND; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; import static com.facebook.presto.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; import static com.facebook.presto.hive.HiveSessionProperties.getTemporaryStagingDirectoryPath; -import static com.facebook.presto.hive.HiveUtil.checkCondition; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.ParquetRecordWriterUtil.createParquetWriter; +import static com.facebook.presto.hive.metastore.MetastoreUtil.checkCondition; import static com.facebook.presto.hive.metastore.MetastoreUtil.getProtectMode; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyOnline; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java index b251a6e52b4b7..5bb78a7c6a5c3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriterFactory.java @@ -59,17 +59,17 @@ import java.util.function.Consumer; import static com.facebook.presto.hive.HiveCompressionCodec.NONE; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_READ_ONLY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PATH_ALREADY_EXISTS; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HiveSessionProperties.getSortedWriteTempPathSubdirectoryCount; import static com.facebook.presto.hive.HiveSessionProperties.isSortedWriteToTempPathEnabled; import static com.facebook.presto.hive.HiveType.toHiveTypes; import static com.facebook.presto.hive.HiveWriteUtils.createPartitionValues; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_EXISTING_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.PartitionUpdate.FileWriteInfo; import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java index bb8dc2f53e05c..c5f5fbba3c9a8 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java @@ -47,7 +47,6 @@ import java.util.concurrent.Callable; import java.util.function.Supplier; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED; import static com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize; @@ -60,6 +59,7 @@ import static com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize; import static com.facebook.presto.hive.HiveSessionProperties.getOrcStringStatisticsLimit; import static com.facebook.presto.hive.HiveType.toHiveTypes; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_UNSUPPORTED_FORMAT; import static com.facebook.presto.orc.OrcEncoding.DWRF; import static com.facebook.presto.orc.OrcEncoding.ORC; import static java.util.Locale.ENGLISH; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java index cd899ee0e8b28..eae2358a8f951 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/S3SelectRecordCursorProvider.java @@ -33,8 +33,8 @@ import java.util.Properties; import java.util.Set; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveUtil.getDeserializerClassName; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class S3SelectRecordCursorProvider diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java index 8c1f86704c620..b81f8b24dbd64 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/HivePartitionName.java @@ -23,8 +23,8 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java index 793b9f5315924..eb98b024bafdd 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/SemiTransactionalHiveMetastore.java @@ -63,19 +63,19 @@ import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PATH_ALREADY_EXISTS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; import static com.facebook.presto.hive.HiveUtil.isPrestoView; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.HiveWriteUtils.createDirectory; import static com.facebook.presto.hive.HiveWriteUtils.pathExists; import static com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.MetastoreUtil.getFileSystem; import static com.facebook.presto.hive.metastore.MetastoreUtil.renameFile; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.TEMPORARY_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java index bfe187b04876d..a2a01c3718b53 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/file/FileHiveMetastore.java @@ -22,6 +22,7 @@ import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveHdfsConfiguration; import com.facebook.presto.hive.HiveType; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.PartitionNotFoundException; import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.SchemaAlreadyExistsException; @@ -79,12 +80,12 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HivePartitionManager.extractPartitionValues; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; @@ -129,8 +130,9 @@ public class FileHiveMetastore public static FileHiveMetastore createTestingFileHiveMetastore(File catalogDirectory) { HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig), ImmutableSet.of()); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); return new FileHiveMetastore(hdfsEnvironment, catalogDirectory.toURI().toString(), "test"); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java index 46065c2123d78..2489c7d804cbe 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueExpressionUtil.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Set; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; public final class GlueExpressionUtil { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index 57c592e00e61e..a0bce78a9de23 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -55,7 +55,6 @@ import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HdfsEnvironment.HdfsContext; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.HiveUtil; import com.facebook.presto.hive.HiveWriteUtils; import com.facebook.presto.hive.PartitionNotFoundException; import com.facebook.presto.hive.PartitionStatistics; @@ -65,6 +64,7 @@ import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrincipalPrivileges; @@ -99,10 +99,10 @@ import java.util.concurrent.Future; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.MetastoreUtil.makePartName; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.MetastoreUtil.verifyCanDropColumn; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.PrestoTableType.VIRTUAL_VIEW; @@ -701,7 +701,7 @@ public Map> getPartitionsByNames(String databaseName List partitions = batchGetPartition(databaseName, tableName, partitionNames); Map> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(toMap(identity(), HiveUtil::toPartitionValues)); + .collect(toMap(identity(), MetastoreUtil::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = partitions.stream() .collect(toMap(Partition::getValues, identity())); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java index 1514b6d2f14a4..e11b9e743c6d3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java @@ -36,7 +36,7 @@ import java.util.Locale; import java.util.Optional; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.metastore.PrestoTableType.OTHER; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Strings.nullToEmpty; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java index 7fbeb38d52e08..edea7a4de850d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/BridgingHiveMetastore.java @@ -14,11 +14,11 @@ package com.facebook.presto.hive.metastore.thrift; import com.facebook.presto.hive.HiveType; -import com.facebook.presto.hive.HiveUtil; import com.facebook.presto.hive.PartitionStatistics; import com.facebook.presto.hive.metastore.Database; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePrivilegeInfo; +import com.facebook.presto.hive.metastore.MetastoreUtil; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.PartitionWithStatistics; import com.facebook.presto.hive.metastore.PrincipalPrivileges; @@ -265,7 +265,7 @@ public Map> getPartitionsByNames(String databaseName return ImmutableMap.of(); } Map> partitionNameToPartitionValuesMap = partitionNames.stream() - .collect(Collectors.toMap(identity(), HiveUtil::toPartitionValues)); + .collect(Collectors.toMap(identity(), MetastoreUtil::toPartitionValues)); Map, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(databaseName, tableName, partitionNames).stream() .map(ThriftMetastoreUtil::fromMetastoreApiPartition) .collect(Collectors.toMap(Partition::getValues, identity())); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java index a53b4b990f6e3..bef8a1331c597 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastore.java @@ -34,7 +34,7 @@ import java.util.Set; import java.util.function.Function; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; public interface HiveMetastore { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java index cc430ef3a917d..4182c2dfbf03d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/HiveMetastoreClientFactory.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; import com.google.common.net.HostAndPort; import io.airlift.units.Duration; @@ -47,9 +47,9 @@ public HiveMetastoreClientFactory( } @Inject - public HiveMetastoreClientFactory(HiveClientConfig config, HiveMetastoreAuthentication metastoreAuthentication) + public HiveMetastoreClientFactory(MetastoreClientConfig metastoreClientConfig, HiveMetastoreAuthentication metastoreAuthentication) { - this(Optional.empty(), Optional.ofNullable(config.getMetastoreSocksProxy()), config.getMetastoreTimeout(), metastoreAuthentication); + this(Optional.empty(), Optional.ofNullable(metastoreClientConfig.getMetastoreSocksProxy()), metastoreClientConfig.getMetastoreTimeout(), metastoreAuthentication); } public HiveMetastoreClient create(HostAndPort address) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java index 377fba8468861..0666c29b65a97 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftHiveMetastore.java @@ -75,8 +75,8 @@ import java.util.stream.Collectors; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveUtil.PRESTO_VIEW_FLAG; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.OWNERSHIP; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.createMetastoreColumnStatistics; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java index 0e5793b398c32..db99239aff3ea 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java @@ -88,9 +88,9 @@ import java.util.function.Predicate; import java.util.stream.Stream; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.HiveMetadata.AVRO_SCHEMA_URL_KEY; import static com.facebook.presto.hive.HiveStorageFormat.AVRO; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_INVALID_METADATA; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; @@ -300,7 +300,7 @@ public static Stream listApplicableTablePrivileges(SemiTransa Stream principals = Stream.concat( Stream.of(userPrincipal), listApplicableRoles(metastore, userPrincipal) - .map(role -> new PrestoPrincipal(ROLE, role))); + .map(role -> new PrestoPrincipal(ROLE, role))); return listTablePrivileges(metastore, databaseName, tableName, principals); } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java index b4d7b3c3b0ee2..2b5d8298f763b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/HdfsOrcDataSource.java @@ -22,9 +22,9 @@ import java.io.IOException; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java index a1bc6852f9cf8..2fde5c238fdc4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/HdfsParquetDataSource.java @@ -25,7 +25,7 @@ import java.io.IOException; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static com.google.common.base.Strings.nullToEmpty; import static java.lang.String.format; import static java.util.Objects.requireNonNull; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java index 0bce6aa7d5e65..f80308ef6904f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java @@ -28,8 +28,8 @@ import java.util.Deque; import java.util.Iterator; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; import static java.util.Objects.requireNonNull; public class HiveFileIterator diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 12e453a28c8fd..e85d8930205ae 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -217,7 +217,6 @@ import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.HiveType.toHiveType; import static com.facebook.presto.hive.HiveUtil.columnExtraInfo; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; import static com.facebook.presto.hive.HiveWriteUtils.createDirectory; import static com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; @@ -227,6 +226,7 @@ import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -799,13 +799,14 @@ protected void setupHive(String connectorId, String databaseName, String timeZon protected final void setup(String host, int port, String databaseName, String timeZone) { HiveClientConfig hiveClientConfig = getHiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = getMetastoreClientConfig(); hiveClientConfig.setTimeZone(timeZone); String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy"); if (proxy != null) { - hiveClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); + metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy)); } - HiveCluster hiveCluster = new TestingHiveCluster(hiveClientConfig, host, port); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port); ExtendedHiveMetastore metastore = new CachingHiveMetastore( new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster)), executor, @@ -813,18 +814,18 @@ protected final void setup(String host, int port, String databaseName, String ti Duration.valueOf("15s"), 10000); - setup(databaseName, hiveClientConfig, metastore); + setup(databaseName, hiveClientConfig, metastoreClientConfig, metastore); } - protected final void setup(String databaseName, HiveClientConfig hiveClientConfig, ExtendedHiveMetastore hiveMetastore) + protected final void setup(String databaseName, HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore hiveMetastore) { HiveConnectorId connectorId = new HiveConnectorId("hive-test"); setupHive(connectorId.toString(), databaseName, hiveClientConfig.getTimeZone()); metastoreClient = hiveMetastore; - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig), ImmutableSet.of()); - hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); locationService = new HiveLocationService(hdfsEnvironment); metadataFactory = new HiveMetadataFactory( metastoreClient, @@ -866,7 +867,7 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi hiveClientConfig.getSplitLoaderConcurrency(), false); pageSinkProvider = new HivePageSinkProvider( - getDefaultHiveFileWriterFactories(hiveClientConfig), + getDefaultHiveFileWriterFactories(hiveClientConfig, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastoreClient, @@ -879,8 +880,8 @@ protected final void setup(String databaseName, HiveClientConfig hiveClientConfi new HiveEventClient(), new HiveSessionProperties(hiveClientConfig, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), - getDefaultOrcFileWriterFactory(hiveClientConfig)); - pageSourceProvider = new HivePageSourceProvider(hiveClientConfig, hdfsEnvironment, getDefaultHiveRecordCursorProvider(hiveClientConfig), getDefaultHiveBatchPageSourceFactories(hiveClientConfig), getDefaultHiveSelectivePageSourceFactories(hiveClientConfig), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); + getDefaultOrcFileWriterFactory(hiveClientConfig, metastoreClientConfig)); + pageSourceProvider = new HivePageSourceProvider(hiveClientConfig, hdfsEnvironment, getDefaultHiveRecordCursorProvider(hiveClientConfig, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(hiveClientConfig, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(hiveClientConfig, metastoreClientConfig), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); } /** @@ -894,6 +895,11 @@ protected HiveClientConfig getHiveClientConfig() .setTemporaryTableSchema(database); } + protected MetastoreClientConfig getMetastoreClientConfig() + { + return new MetastoreClientConfig(); + } + protected ConnectorSession newSession() { return new TestingConnectorSession(new HiveSessionProperties(getHiveClientConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java index 75e5bb506ab8f..41af472bd8926 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClientLocal.java @@ -66,8 +66,9 @@ public void initialize() HiveClientConfig hiveConfig = new HiveClientConfig() .setTimeZone("America/Los_Angeles"); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); - setup(testDbName, hiveConfig, metastore); + setup(testDbName, hiveConfig, metastoreClientConfig, metastore); } @AfterClass(alwaysRun = true) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java index c303ce57c890c..5b596629f09be 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java @@ -60,7 +60,6 @@ import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.TestingConnectorSession; import com.facebook.presto.testing.TestingNodeManager; -import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; @@ -80,6 +79,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutorService; +import java.util.function.BiFunction; import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue; import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; @@ -131,6 +131,7 @@ public abstract class AbstractTestHiveFileSystem private ExecutorService executor; private HiveClientConfig config; + private MetastoreClientConfig metastoreClientConfig; @BeforeClass public void setUp() @@ -149,7 +150,7 @@ public void tearDown() protected abstract Path getBasePath(); - protected void setup(String host, int port, String databaseName, Function hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) + protected void setup(String host, int port, String databaseName, BiFunction hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) { database = databaseName; table = new SchemaTableName(database, "presto_test_external_fs"); @@ -158,19 +159,20 @@ protected void setup(String host, int port, String databaseName, Function getDefaultHiveBatchPageSourceFactories(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveBatchPageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new RcFilePageSourceFactory(TYPE_MANAGER, testHdfsEnvironment, stats, new HadoopFileOpener())) .add(new OrcBatchPageSourceFactory(TYPE_MANAGER, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) @@ -138,36 +140,36 @@ public static Set getDefaultHiveBatchPageSourceFacto .build(); } - public static Set getDefaultHiveSelectivePageSourceFactories(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveSelectivePageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { FileFormatDataSourceStats stats = new FileFormatDataSourceStats(); - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new OrcSelectivePageSourceFactory(TYPE_MANAGER, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) .add(new DwrfSelectivePageSourceFactory(TYPE_MANAGER, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), new StorageStripeMetadataSource(), new HadoopFileOpener())) .build(); } - public static Set getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveRecordCursorProvider(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new GenericHiveRecordCursorProvider(testHdfsEnvironment)) .build(); } - public static Set getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig) + public static Set getDefaultHiveFileWriterFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return ImmutableSet.builder() .add(new RcFileFileWriterFactory(testHdfsEnvironment, TYPE_MANAGER, new NodeVersion("test_version"), hiveClientConfig, new FileFormatDataSourceStats())) - .add(getDefaultOrcFileWriterFactory(hiveClientConfig)) + .add(getDefaultOrcFileWriterFactory(hiveClientConfig, metastoreClientConfig)) .build(); } - public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig) + public static OrcFileWriterFactory getDefaultOrcFileWriterFactory(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) { - HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig); + HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig); return new OrcFileWriterFactory( testHdfsEnvironment, TYPE_MANAGER, @@ -186,15 +188,16 @@ public static List getTypes(List columnHandles) return types.build(); } - public static HdfsEnvironment createTestHdfsEnvironment(HiveClientConfig config) + public static HdfsEnvironment createTestHdfsEnvironment(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig) { HdfsConfiguration hdfsConfig = new HiveHdfsConfiguration( new HdfsConfigurationInitializer( config, + metastoreClientConfig, new PrestoS3ConfigurationUpdater(new HiveS3Config()), new HiveGcsConfigurationInitializer(new HiveGcsConfig())), ImmutableSet.of()); - return new HdfsEnvironment(hdfsConfig, config, new NoHdfsAuthentication()); + return new HdfsEnvironment(hdfsConfig, metastoreClientConfig, new NoHdfsAuthentication()); } public static MapType mapType(Type keyType, Type valueType) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java index 790655dc66a5e..d97763c558f38 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestBackgroundHiveSplitLoader.java @@ -405,8 +405,8 @@ private static class TestingHdfsEnvironment public TestingHdfsEnvironment() { super( - new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig()), ImmutableSet.of()), - new HiveClientConfig(), + new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig(), new MetastoreClientConfig()), ImmutableSet.of()), + new MetastoreClientConfig(), new NoHdfsAuthentication()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java index e5c4d1b72e516..db7362fcbe5f6 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestFileSystemCache.java @@ -35,8 +35,8 @@ public void testFileSystemCache() ImpersonatingHdfsAuthentication auth = new ImpersonatingHdfsAuthentication(new SimpleHadoopAuthentication()); HdfsEnvironment environment = new HdfsEnvironment( - new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig()), ImmutableSet.of()), - new HiveClientConfig(), + new HiveHdfsConfiguration(new HdfsConfigurationInitializer(new HiveClientConfig(), new MetastoreClientConfig()), ImmutableSet.of()), + new MetastoreClientConfig(), auth); FileSystem fs1 = getFileSystem(environment, "user"); FileSystem fs2 = getFileSystem(environment, "user"); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index d1805b430e656..093f4d44c2c5e 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -20,7 +20,6 @@ import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.net.HostAndPort; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; @@ -54,8 +53,6 @@ public void testDefaults() .setMetastoreCacheMaximumSize(10000) .setPerTransactionMetastoreCacheMaximumSize(1000) .setMaxMetastoreRefreshThreads(100) - .setMetastoreSocksProxy(null) - .setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS)) .setMinPartitionBatchSize(10) .setMaxPartitionBatchSize(100) .setMaxInitialSplits(200) @@ -71,7 +68,6 @@ public void testDefaults() .setIpcPingInterval(new Duration(10, TimeUnit.SECONDS)) .setDfsConnectTimeout(new Duration(500, TimeUnit.MILLISECONDS)) .setDfsConnectMaxRetries(5) - .setVerifyChecksum(true) .setDomainSocketPath(null) .setS3FileSystemType(S3FileSystemType.PRESTO) .setResourceConfigFiles("") @@ -152,15 +148,12 @@ public void testExplicitPropertyMappings() .put("hive.metastore-cache-maximum-size", "5000") .put("hive.per-transaction-metastore-cache-maximum-size", "500") .put("hive.metastore-refresh-max-threads", "2500") - .put("hive.metastore.thrift.client.socks-proxy", "localhost:1080") - .put("hive.metastore-timeout", "20s") .put("hive.metastore.partition-batch-size.min", "1") .put("hive.metastore.partition-batch-size.max", "1000") .put("hive.dfs.ipc-ping-interval", "34s") .put("hive.dfs-timeout", "33s") .put("hive.dfs.connect.timeout", "20s") .put("hive.dfs.connect.max-retries", "10") - .put("hive.dfs.verify-checksum", "false") .put("hive.dfs.domain-socket-path", "/foo") .put("hive.s3-file-system-type", "EMRFS") .put("hive.config.resources", "/foo.xml,/bar.xml") @@ -246,8 +239,6 @@ public void testExplicitPropertyMappings() .setMetastoreCacheMaximumSize(5000) .setPerTransactionMetastoreCacheMaximumSize(500) .setMaxMetastoreRefreshThreads(2500) - .setMetastoreSocksProxy(HostAndPort.fromParts("localhost", 1080)) - .setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS)) .setMinPartitionBatchSize(1) .setMaxPartitionBatchSize(1000) .setMaxInitialSplits(10) @@ -263,7 +254,6 @@ public void testExplicitPropertyMappings() .setDfsTimeout(new Duration(33, TimeUnit.SECONDS)) .setDfsConnectTimeout(new Duration(20, TimeUnit.SECONDS)) .setDfsConnectMaxRetries(10) - .setVerifyChecksum(false) .setResourceConfigFiles(ImmutableList.of("/foo.xml", "/bar.xml")) .setHiveStorageFormat(HiveStorageFormat.SEQUENCEFILE) .setCompressionCodec(HiveCompressionCodec.NONE) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java index d292a087ed670..db19750b6961b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientFileMetastore.java @@ -29,9 +29,10 @@ protected ExtendedHiveMetastore createMetastore(File tempDir) { File baseDir = new File(tempDir, "metastore"); HiveClientConfig hiveConfig = new HiveClientConfig(); - HdfsConfigurationInitializer updator = new HdfsConfigurationInitializer(hiveConfig); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfigurationInitializer updator = new HdfsConfigurationInitializer(hiveConfig, metastoreClientConfig); HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(updator, ImmutableSet.of()); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveConfig, new NoHdfsAuthentication()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); return new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test"); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java index 6b01c91b10192..94d78a3dce8df 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java @@ -106,13 +106,14 @@ public void testAllFormats() throws Exception { HiveClientConfig config = new HiveClientConfig(); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); File tempDir = Files.createTempDir(); try { ExtendedHiveMetastore metastore = createTestingFileHiveMetastore(new File(tempDir, "metastore")); for (HiveStorageFormat format : HiveStorageFormat.values()) { config.setHiveStorageFormat(format); config.setCompressionCodec(NONE); - long uncompressedLength = writeTestFile(config, metastore, makeFileName(tempDir, config)); + long uncompressedLength = writeTestFile(config, metastoreClientConfig, metastore, makeFileName(tempDir, config)); assertGreaterThan(uncompressedLength, 0L); for (HiveCompressionCodec codec : HiveCompressionCodec.values()) { @@ -120,7 +121,7 @@ public void testAllFormats() continue; } config.setCompressionCodec(codec); - long length = writeTestFile(config, metastore, makeFileName(tempDir, config)); + long length = writeTestFile(config, metastoreClientConfig, metastore, makeFileName(tempDir, config)); assertTrue(uncompressedLength > length, format("%s with %s compressed to %s which is not less than %s", format, codec, length, uncompressedLength)); } } @@ -135,11 +136,11 @@ private static String makeFileName(File tempDir, HiveClientConfig config) return tempDir.getAbsolutePath() + "/" + config.getHiveStorageFormat().name() + "." + config.getCompressionCodec().name(); } - private static long writeTestFile(HiveClientConfig config, ExtendedHiveMetastore metastore, String outputPath) + private static long writeTestFile(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, String outputPath) { HiveTransactionHandle transaction = new HiveTransactionHandle(); HiveWriterStats stats = new HiveWriterStats(); - ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats); + ConnectorPageSink pageSink = createPageSink(transaction, config, metastoreClientConfig, metastore, new Path("file:///" + outputPath), stats); List columns = getTestColumns(); List columnTypes = columns.stream() .map(LineItemColumn::getType) @@ -188,7 +189,7 @@ private static long writeTestFile(HiveClientConfig config, ExtendedHiveMetastore File outputFile = getOnlyElement(files); long length = outputFile.length(); - ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile); + ConnectorPageSource pageSource = createPageSource(transaction, config, metastoreClientConfig, outputFile); List pages = new ArrayList<>(); while (!pageSource.isFinished()) { @@ -214,7 +215,7 @@ public static MaterializedResult toMaterializedResult(ConnectorSession session, return resultBuilder.build(); } - private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, File outputFile) + private static ConnectorPageSource createPageSource(HiveTransactionHandle transaction, HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, File outputFile) { HiveSplit split = new HiveSplit( SCHEMA_NAME, @@ -259,11 +260,11 @@ private static ConnectorPageSource createPageSource(HiveTransactionHandle transa Optional.empty(), false, "layout"))); - HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveBatchPageSourceFactories(config), getDefaultHiveSelectivePageSourceFactories(config), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); + HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config, metastoreClientConfig), getDefaultHiveRecordCursorProvider(config, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(config, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(config, metastoreClientConfig), TYPE_MANAGER, ROW_EXPRESSION_SERVICE); return provider.createPageSource(transaction, getSession(config), split, tableHandle.getLayout().get(), ImmutableList.copyOf(getColumnHandles())); } - private static ConnectorPageSink createPageSink(HiveTransactionHandle transaction, HiveClientConfig config, ExtendedHiveMetastore metastore, Path outputPath, HiveWriterStats stats) + private static ConnectorPageSink createPageSink(HiveTransactionHandle transaction, HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, Path outputPath, HiveWriterStats stats) { LocationHandle locationHandle = new LocationHandle(outputPath, outputPath, Optional.empty(), NEW, DIRECT_TO_TARGET_NEW_DIRECTORY); HiveOutputTableHandle handle = new HiveOutputTableHandle( @@ -281,9 +282,9 @@ private static ConnectorPageSink createPageSink(HiveTransactionHandle transactio "test", ImmutableMap.of()); JsonCodec partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class); - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(config, metastoreClientConfig); HivePageSinkProvider provider = new HivePageSinkProvider( - getDefaultHiveFileWriterFactories(config), + getDefaultHiveFileWriterFactories(config, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastore, @@ -296,7 +297,7 @@ private static ConnectorPageSink createPageSink(HiveTransactionHandle transactio new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), stats, - getDefaultOrcFileWriterFactory(config)); + getDefaultOrcFileWriterFactory(config, metastoreClientConfig)); return provider.createPageSink(transaction, getSession(config), handle, PageSinkProperties.defaultProperties()); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java index 69e009c5f2829..63ac5411c20a0 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveUtil.java @@ -32,7 +32,7 @@ import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.hive.HiveUtil.getDeserializer; import static com.facebook.presto.hive.HiveUtil.parseHiveTimestamp; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_CLASS; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java index 4ea7d1ea0e72a..13ef2a7f1cc78 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveWriteUtils.java @@ -31,7 +31,7 @@ public class TestHiveWriteUtils @Test public void testIsS3FileSystem() { - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig(), new MetastoreClientConfig()); assertTrue(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("s3://test-bucket/test-folder"))); assertFalse(isS3FileSystem(CONTEXT, hdfsEnvironment, new Path("/test-dir/test-folder"))); } @@ -39,7 +39,7 @@ public void testIsS3FileSystem() @Test public void testIsViewFileSystem() { - HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig()); + HdfsEnvironment hdfsEnvironment = createTestHdfsEnvironment(new HiveClientConfig(), new MetastoreClientConfig()); Path viewfsPath = new Path("viewfs://ns-default/test-folder"); Path nonViewfsPath = new Path("hdfs://localhost/test-dir/test-folder"); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java index 5efbfb1fe3d6c..547be25a0f3e8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestingSemiTransactionalHiveMetastore.java @@ -67,9 +67,10 @@ public static TestingSemiTransactionalHiveMetastore create() { // none of these values matter, as we never use them HiveClientConfig config = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of()); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, config, new NoHdfsAuthentication()); - HiveCluster hiveCluster = new TestingHiveCluster(config, HOST, PORT); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); + HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, HOST, PORT); ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster)); ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s")); ListeningExecutorService renameExecutor = listeningDecorator(executor); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java index 50a7cc516b114..6bf7d2b380b5c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/benchmark/HiveFileFormatBenchmark.java @@ -63,6 +63,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import static com.facebook.presto.hive.HiveTestUtils.METASTORE_CLIENT_CONFIG; import static com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment; import static com.facebook.presto.hive.HiveTestUtils.mapType; import static com.facebook.presto.spi.type.BigintType.BIGINT; @@ -99,7 +100,7 @@ public class HiveFileFormatBenchmark private static final ConnectorSession SESSION = new TestingConnectorSession(new HiveSessionProperties(CONFIG, new OrcFileWriterConfig(), new ParquetFileWriterConfig()) .getSessionProperties()); - private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(CONFIG); + private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(CONFIG, METASTORE_CLIENT_CONFIG); @Param({ "LINEITEM", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java similarity index 95% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java rename to presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java index 81f4163c65490..2ba6a0c384c91 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreUtil.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java @@ -34,11 +34,12 @@ import static com.facebook.presto.hive.HiveType.HIVE_DOUBLE; import static com.facebook.presto.hive.HiveType.HIVE_INT; import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema; import static com.facebook.presto.hive.metastore.MetastoreUtil.reconstructPartitionSchema; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; -public class TestMetastoreUtil +public class TestHiveMetastoreUtil { private static final List TEST_SCHEMA = ImmutableList.of( new FieldSchema("col1", "bigint", "comment1"), @@ -146,7 +147,7 @@ public void testPartitionRoundTrip() public void testHiveSchemaTable() { Properties expected = MetaStoreUtils.getTableMetadata(TEST_TABLE_WITH_UNSUPPORTED_FIELDS); - Properties actual = MetastoreUtil.getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); + Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); assertEquals(actual, expected); } @@ -154,7 +155,7 @@ public void testHiveSchemaTable() public void testHiveSchemaPartition() { Properties expected = MetaStoreUtils.getPartitionMetadata(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS, TEST_TABLE_WITH_UNSUPPORTED_FIELDS); - Properties actual = MetastoreUtil.getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS), ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); + Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS), ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA)); assertEquals(actual, expected); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java index 08e15645e3a12..a4461d9a34fdf 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestHiveClientGlueMetastore.java @@ -19,6 +19,7 @@ import com.facebook.presto.hive.HdfsEnvironment; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveHdfsConfiguration; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.google.common.collect.ImmutableSet; @@ -45,8 +46,9 @@ public TestHiveClientGlueMetastore() protected ExtendedHiveMetastore createMetastore(File tempDir) { HiveClientConfig hiveClientConfig = new HiveClientConfig(); - HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig), ImmutableSet.of()); - HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of()); + HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication()); GlueHiveMetastoreConfig glueConfig = new GlueHiveMetastoreConfig(); glueConfig.setDefaultWarehouseDir(tempDir.toURI().toString()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java index 5a6c64713b334..a0ddacdf2c0c2 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/InMemoryHiveMetastore.java @@ -53,7 +53,7 @@ import java.util.function.Function; import static com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics; -import static com.facebook.presto.hive.HiveUtil.toPartitionValues; +import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues; import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiPartition; import static com.facebook.presto.spi.StandardErrorCode.SCHEMA_NOT_EMPTY; import static com.google.common.base.MoreObjects.toStringHelper; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java similarity index 99% rename from presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java rename to presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java index 9dde3c8849caa..bf9a93a02d8c2 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftMetastoreUtil.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java @@ -59,7 +59,7 @@ import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.testng.Assert.assertEquals; -public class TestThriftMetastoreUtil +public class TestThriftHiveMetastoreUtil { @Test public void testLongStatsToColumnStatistics() diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java index 441f46048e084..570f66c9cb1e3 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/thrift/TestingHiveCluster.java @@ -13,7 +13,7 @@ */ package com.facebook.presto.hive.metastore.thrift; -import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.authentication.NoHiveMetastoreAuthentication; import com.google.common.net.HostAndPort; import org.apache.thrift.TException; @@ -25,12 +25,12 @@ public class TestingHiveCluster implements HiveCluster { - private final HiveClientConfig config; + private final MetastoreClientConfig metastoreClientConfig; private final HostAndPort address; - public TestingHiveCluster(HiveClientConfig config, String host, int port) + public TestingHiveCluster(MetastoreClientConfig metastoreClientConfig, String host, int port) { - this.config = requireNonNull(config, "config is null"); + this.metastoreClientConfig = requireNonNull(metastoreClientConfig, "metastore config is null"); this.address = HostAndPort.fromParts(requireNonNull(host, "host is null"), port); } @@ -38,7 +38,7 @@ public TestingHiveCluster(HiveClientConfig config, String host, int port) public HiveMetastoreClient createMetastoreClient() throws TException { - return new HiveMetastoreClientFactory(config, new NoHiveMetastoreAuthentication()).create(address); + return new HiveMetastoreClientFactory(metastoreClientConfig, new NoHiveMetastoreAuthentication()).create(address); } @Override diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java index dbdb0a3acf117..0545734891644 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/parquet/ParquetTester.java @@ -83,11 +83,12 @@ import static com.facebook.presto.hive.AbstractTestHiveFileFormats.getFieldFromCursor; import static com.facebook.presto.hive.HiveSessionProperties.getParquetMaxReadBlockSize; +import static com.facebook.presto.hive.HiveTestUtils.METASTORE_CLIENT_CONFIG; import static com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment; -import static com.facebook.presto.hive.HiveUtil.isArrayType; -import static com.facebook.presto.hive.HiveUtil.isMapType; -import static com.facebook.presto.hive.HiveUtil.isRowType; import static com.facebook.presto.hive.HiveUtil.isStructuralType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isArrayType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isMapType; +import static com.facebook.presto.hive.metastore.MetastoreUtil.isRowType; import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; import static com.facebook.presto.spi.type.Varchars.isVarcharType; @@ -114,7 +115,7 @@ public class ParquetTester public static final DateTimeZone HIVE_STORAGE_TIME_ZONE = DateTimeZone.forID("America/Bahia_Banderas"); private static final boolean OPTIMIZED = true; private static final HiveClientConfig HIVE_CLIENT_CONFIG = createHiveClientConfig(false); - private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG); + private static final HdfsEnvironment HDFS_ENVIRONMENT = createTestHdfsEnvironment(HIVE_CLIENT_CONFIG, METASTORE_CLIENT_CONFIG); private static final TestingConnectorSession SESSION = new TestingConnectorSession(new HiveSessionProperties(HIVE_CLIENT_CONFIG, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); private static final TestingConnectorSession SESSION_USE_NAME = new TestingConnectorSession(new HiveSessionProperties(createHiveClientConfig(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties()); private static final List TEST_COLUMN = singletonList("test"); diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 7a5a8c410ccbe..0252264ff725e 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -33,6 +33,17 @@ + + com.facebook.presto + presto-hive-metastore + + + com.facebook.presto.hive + hive-apache + + + + com.facebook.presto presto-jdbc diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java index 1f4637498be03..7cea65265a6e5 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/prestoaction/PrestoExceptionClassifier.java @@ -36,15 +36,15 @@ import static com.facebook.presto.connector.thrift.ThriftErrorCode.THRIFT_SERVICE_CONNECTION_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CURSOR_ERROR; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_FILE_NOT_FOUND; -import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TABLE_DROPPED_DURING_QUERY; import static com.facebook.presto.hive.HiveErrorCode.HIVE_TOO_MANY_OPEN_PARTITIONS; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_CLOSE_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_DATA_ERROR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_FILESYSTEM_ERROR; +import static com.facebook.presto.hive.MetastoreErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; import static com.facebook.presto.spi.StandardErrorCode.ABANDONED_TASK; import static com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE;