Skip to content

Upgrade to Hive 3.0.0 #15659

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rschlussel merged 1 commit into from
Mar 1, 2021
Merged

Upgrade to Hive 3.0.0 #15659

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
<dep.oracle.version>19.3.0.0</dep.oracle.version>
<dep.drift.version>1.32</dep.drift.version>
<dep.joda.version>2.10.6</dep.joda.version>
<dep.tempto.version>1.50</dep.tempto.version>
<dep.tempto.version>1.51</dep.tempto.version>
<dep.testng.version>6.10</dep.testng.version>
<dep.assertj-core.version>3.8.0</dep.assertj-core.version>
<dep.logback.version>1.2.3</dep.logback.version>
Expand Down Expand Up @@ -579,7 +579,7 @@
<dependency>
<groupId>com.facebook.presto.hive</groupId>
<artifactId>hive-apache</artifactId>
<version>1.2.2-2</version>
<version>3.0.0-2</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -1019,12 +1019,8 @@
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>0.9.1</version>
<version>0.9.3</version>
<exclusions>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
Expand Down
5 changes: 3 additions & 2 deletions presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import com.facebook.presto.spi.TableNotFoundException;
import com.facebook.presto.spi.statistics.ColumnStatisticType;
import com.google.common.base.CharMatcher;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
Expand Down Expand Up @@ -109,7 +110,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
import static org.apache.hadoop.hive.common.FileUtils.unescapePathName;
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.typeToThriftType;
import static org.apache.hadoop.hive.metastore.ColumnType.typeToThriftType;
import static org.apache.hadoop.hive.metastore.ProtectMode.getProtectModeFromString;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME;
Expand Down Expand Up @@ -217,7 +218,7 @@ public static Properties getHiveSchema(
if (storage.getBucketProperty().isPresent()) {
List<String> bucketedBy = storage.getBucketProperty().get().getBucketedBy();
if (!bucketedBy.isEmpty()) {
schema.setProperty(BUCKET_FIELD_NAME, bucketedBy.get(0));
schema.setProperty(BUCKET_FIELD_NAME, Joiner.on(",").join(bucketedBy));
}
schema.setProperty(BUCKET_COUNT, Integer.toString(storage.getBucketProperty().get().getBucketCount()));
}
Expand Down
3 changes: 2 additions & 1 deletion ...etastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Streams;
import com.google.common.primitives.Shorts;
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
Expand Down Expand Up @@ -822,7 +823,7 @@ public static Date toMetastoreDate(LocalDate date)

public static Decimal toMetastoreDecimal(BigDecimal decimal)
{
return new Decimal(ByteBuffer.wrap(decimal.unscaledValue().toByteArray()), (short) decimal.scale());
return new Decimal(Shorts.checkedCast(decimal.scale()), ByteBuffer.wrap(decimal.unscaledValue().toByteArray()));
Copy link
Copy Markdown
Contributor

@rschlussel rschlussel Feb 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reason for the changes for decimal?

Copy link
Copy Markdown
Contributor

@aweisberg aweisberg Feb 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Constructor argument order is reversed in Hive 3. Trino also just switches the parameters around.

}

private static OptionalLong toMetastoreDistinctValuesCount(OptionalLong distinctValuesCount, OptionalLong nullsCount)
Expand Down
5 changes: 4 additions & 1 deletion ...ive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestHiveMetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.testng.annotations.Test;

import java.util.List;
Expand All @@ -37,6 +37,7 @@
import static com.facebook.presto.hive.HiveType.HIVE_STRING;
import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveSchema;
import static com.facebook.presto.hive.metastore.MetastoreUtil.reconstructPartitionSchema;
import static org.apache.hadoop.hive.serde.serdeConstants.COLUMN_NAME_DELIMITER;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -149,6 +150,7 @@ public void testPartitionRoundTrip()
public void testHiveSchemaTable()
{
Properties expected = MetaStoreUtils.getTableMetadata(TEST_TABLE_WITH_UNSUPPORTED_FIELDS);
expected.remove(COLUMN_NAME_DELIMITER);
Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA));
assertEquals(actual, expected);
}
Expand All @@ -157,6 +159,7 @@ public void testHiveSchemaTable()
public void testHiveSchemaPartition()
{
Properties expected = MetaStoreUtils.getPartitionMetadata(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS, TEST_TABLE_WITH_UNSUPPORTED_FIELDS);
expected.remove(COLUMN_NAME_DELIMITER);
Properties actual = getHiveSchema(ThriftMetastoreUtil.fromMetastoreApiPartition(TEST_PARTITION_WITH_UNSUPPORTED_FIELDS, TEST_PARTITION_VERSION_FETCHER), ThriftMetastoreUtil.fromMetastoreApiTable(TEST_TABLE_WITH_UNSUPPORTED_FIELDS, TEST_SCHEMA));
assertEquals(actual, expected);
}
Expand Down
7 changes: 3 additions & 4 deletions .../src/test/java/com/facebook/presto/hive/metastore/thrift/TestThriftHiveMetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.testng.annotations.Test;

import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.time.LocalDate;
import java.util.Optional;
import java.util.OptionalDouble;
Expand All @@ -43,6 +41,7 @@
import static com.facebook.presto.hive.metastore.MetastoreUtil.getHiveBasicStatistics;
import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters;
import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreApiColumnStatistics;
import static com.facebook.presto.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreDecimal;
import static org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.binaryStats;
import static org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.booleanStats;
import static org.apache.hadoop.hive.metastore.api.ColumnStatisticsData.dateStats;
Expand Down Expand Up @@ -146,9 +145,9 @@ public void testDecimalStatsToColumnStatistics()
{
DecimalColumnStatsData decimalColumnStatsData = new DecimalColumnStatsData();
BigDecimal low = new BigDecimal("0");
decimalColumnStatsData.setLowValue(new Decimal(ByteBuffer.wrap(low.unscaledValue().toByteArray()), (short) low.scale()));
decimalColumnStatsData.setLowValue(toMetastoreDecimal(low));
BigDecimal high = new BigDecimal("100");
decimalColumnStatsData.setHighValue(new Decimal(ByteBuffer.wrap(high.unscaledValue().toByteArray()), (short) high.scale()));
decimalColumnStatsData.setHighValue(toMetastoreDecimal(high));
decimalColumnStatsData.setNumNulls(1);
decimalColumnStatsData.setNumDVs(20);
ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj("my_col", DECIMAL_TYPE_NAME, decimalStats(decimalColumnStatsData));
Expand Down
45 changes: 0 additions & 45 deletions presto-hive/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,6 @@
<dependency>
<groupId>com.facebook.presto</groupId>
<artifactId>presto-parquet</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-encoding</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand All @@ -53,38 +47,6 @@
<artifactId>presto-cache</artifactId>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${dep.parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>${dep.parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-hadoop-mr</artifactId>
Expand Down Expand Up @@ -291,13 +253,6 @@
<scope>runtime</scope>
</dependency>

<!-- used only by tests but also needed transitively -->
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-common</artifactId>
<version>${dep.parquet.version}</version>
</dependency>

<!-- Presto SPI -->
<dependency>
<groupId>com.facebook.presto</groupId>
Expand Down
1 change: 0 additions & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/GenericHiveRecordCursor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ class GenericHiveRecordCursor<K, V extends Writable>
private final K key;
private final V value;

@SuppressWarnings("deprecation")
private final Deserializer deserializer;

private final Type[] types;
Expand Down
2 changes: 1 addition & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/HiveCompressionCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.SnappyCodec;
import parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;

import java.util.Optional;
import java.util.function.Predicate;
Expand Down
7 changes: 1 addition & 6 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ public static boolean isSplittable(InputFormat<?, ?> inputFormat, FileSystem fil
}
}

public static StructObjectInspector getTableObjectInspector(@SuppressWarnings("deprecation") Deserializer deserializer)
public static StructObjectInspector getTableObjectInspector(Deserializer deserializer)
{
try {
ObjectInspector inspector = deserializer.getObjectInspector();
Expand All @@ -428,7 +428,6 @@ public static String getDeserializerClassName(Properties schema)
return name;
}

@SuppressWarnings("deprecation")
public static Deserializer getDeserializer(Configuration configuration, Properties schema)
{
String name = getDeserializerClassName(schema);
Expand All @@ -438,7 +437,6 @@ public static Deserializer getDeserializer(Configuration configuration, Properti
return deserializer;
}

@SuppressWarnings("deprecation")
private static Class<? extends Deserializer> getDeserializerClass(String name)
{
// CDH uses different names for Parquet
Expand All @@ -461,7 +459,6 @@ private static Class<? extends Deserializer> getDeserializerClass(String name)
}
}

@SuppressWarnings("deprecation")
private static Deserializer createDeserializer(Class<? extends Deserializer> clazz)
{
try {
Expand All @@ -472,7 +469,6 @@ private static Deserializer createDeserializer(Class<? extends Deserializer> cla
}
}

@SuppressWarnings("deprecation")
private static void initializeDeserializer(Configuration configuration, Deserializer deserializer, Properties schema)
{
try {
Expand All @@ -485,7 +481,6 @@ private static void initializeDeserializer(Configuration configuration, Deserial
}
}

@SuppressWarnings("deprecation")
private static void validate(Deserializer deserializer)
{
if (deserializer instanceof AbstractSerDe && !((AbstractSerDe) deserializer).getConfigurationErrors().isEmpty()) {
Expand Down
1 change: 0 additions & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,6 @@ public void close(boolean abort)
};
}

@SuppressWarnings("deprecation")
public static Serializer initializeSerializer(Configuration conf, Properties properties, String serializerName)
{
try {
Expand Down
7 changes: 2 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/OrcFileWriterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@
import io.airlift.slice.Slice;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.orc.OrcFile.OrcTableProperties;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.orc.OrcConf;
import org.joda.time.DateTimeZone;
import org.weakref.jmx.Flatten;
import org.weakref.jmx.Managed;
Expand Down Expand Up @@ -292,10 +292,7 @@ public DataSink createDataSink(ConnectorSession session, FileSystem fileSystem,

private static CompressionKind getCompression(Properties schema, JobConf configuration, OrcEncoding orcEncoding)
{
String compressionName = schema.getProperty(OrcTableProperties.COMPRESSION.getPropName());
if (compressionName == null) {
compressionName = configuration.get("hive.exec.orc.default.compress");
}
String compressionName = OrcConf.COMPRESS.getString(schema, configuration);
if (compressionName == null) {
return CompressionKind.ZLIB;
}
Expand Down
2 changes: 1 addition & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/ParquetFileWriterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import com.facebook.airlift.configuration.Config;
import io.airlift.units.DataSize;
import parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.ParquetWriter;

import static io.airlift.units.DataSize.Unit.BYTE;

Expand Down
6 changes: 3 additions & 3 deletions presto-hive/src/main/java/com/facebook/presto/hive/ParquetRecordWriterUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import parquet.hadoop.ParquetFileWriter;
import parquet.hadoop.ParquetOutputFormat;
import parquet.hadoop.ParquetRecordWriter;
import org.apache.parquet.hadoop.ParquetFileWriter;
import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.hadoop.ParquetRecordWriter;

import java.io.IOException;
import java.lang.reflect.Field;
Expand Down
1 change: 0 additions & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/RecordFileWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ public class RecordFileWriter
private final Path path;
private final JobConf conf;
private final int fieldCount;
@SuppressWarnings("deprecation")
private final Serializer serializer;
private final RecordWriter recordWriter;
private final SettableStructObjectInspector tableInspector;
Expand Down
2 changes: 1 addition & 1 deletion presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetFileWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import com.facebook.presto.parquet.writer.ParquetWriterOptions;
import com.facebook.presto.spi.PrestoException;
import com.google.common.collect.ImmutableList;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.openjdk.jol.info.ClassLayout;
import parquet.hadoop.metadata.CompressionCodecName;

import java.io.IOException;
import java.io.OutputStream;
Expand Down
4 changes: 2 additions & 2 deletions presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetFileWriterFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.parquet.hadoop.ParquetOutputFormat;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.joda.time.DateTimeZone;
import parquet.hadoop.ParquetOutputFormat;
import parquet.hadoop.metadata.CompressionCodecName;

import java.io.IOException;
import java.util.List;
Expand Down
8 changes: 1 addition & 7 deletions presto-hive/src/main/java/com/facebook/presto/hive/rcfile/RcFilePageSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import com.facebook.presto.spi.PrestoException;
import com.google.common.collect.ImmutableList;
import io.airlift.units.DataSize;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.List;
Expand Down Expand Up @@ -60,15 +59,10 @@ public class RcFilePageSource

private boolean closed;

public RcFilePageSource(
RcFileReader rcFileReader,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager)
public RcFilePageSource(RcFileReader rcFileReader, List<HiveColumnHandle> columns, TypeManager typeManager)
{
requireNonNull(rcFileReader, "rcReader is null");
requireNonNull(columns, "columns is null");
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
requireNonNull(typeManager, "typeManager is null");

this.rcFileReader = rcFileReader;
Expand Down
Loading