diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java deleted file mode 100644 index 4a3423f8f98c..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.google.inject.Inject; -import io.airlift.slice.BasicSliceInput; -import io.airlift.slice.DynamicSliceOutput; -import io.airlift.slice.SliceOutput; -import io.airlift.slice.Slices; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncodingSerde; - -import java.io.IOException; -import java.util.Base64; - -import static java.util.Objects.requireNonNull; - -// copy of TestingBlockJsonSerde -public final class BlockJsonSerde -{ - private BlockJsonSerde() {} - - public static class Serializer - extends JsonSerializer - { - private final BlockEncodingSerde blockEncodingSerde; - - @Inject - public Serializer(HiveBlockEncodingSerde blockEncodingSerde) - { - this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); - } - - @Override - public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException - { - SliceOutput output = new DynamicSliceOutput(64); - blockEncodingSerde.writeBlock(output, block); - String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); - jsonGenerator.writeString(encoded); - } - } - - public static class Deserializer - extends JsonDeserializer - { - private final BlockEncodingSerde blockEncodingSerde; - - @Inject - public Deserializer(HiveBlockEncodingSerde blockEncodingSerde) - { - this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); - } - - @Override - public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) - throws IOException - { - byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); - BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); - return blockEncodingSerde.readBlock(input); - } - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java deleted file mode 100644 index 00afd090724a..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.slice.SliceInput; -import io.airlift.slice.SliceOutput; -import io.trino.spi.block.ArrayBlockEncoding; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncoding; -import io.trino.spi.block.BlockEncodingSerde; -import io.trino.spi.block.ByteArrayBlockEncoding; -import io.trino.spi.block.DictionaryBlockEncoding; -import io.trino.spi.block.Fixed12BlockEncoding; -import io.trino.spi.block.Int128ArrayBlockEncoding; -import io.trino.spi.block.IntArrayBlockEncoding; -import io.trino.spi.block.LongArrayBlockEncoding; -import io.trino.spi.block.MapBlockEncoding; -import io.trino.spi.block.RowBlockEncoding; -import io.trino.spi.block.RunLengthBlockEncoding; -import io.trino.spi.block.ShortArrayBlockEncoding; -import io.trino.spi.block.VariableWidthBlockEncoding; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.SizeOf.SIZE_OF_INT; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.HyperLogLogType.HYPER_LOG_LOG; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; - -// This class is exactly the same as BlockEncodingManager -public final class HiveBlockEncodingSerde - implements BlockEncodingSerde -{ - private static final List TYPES = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, INTEGER, VARCHAR, VARBINARY, TIMESTAMP_MILLIS, TIMESTAMP_TZ_MILLIS, DATE, HYPER_LOG_LOG); - - private final ConcurrentMap blockEncodingsByName = new ConcurrentHashMap<>(); - private final ConcurrentMap, BlockEncoding> blockEncodingsByClass = new ConcurrentHashMap<>(); - - @Inject - public HiveBlockEncodingSerde() - { - addBlockEncoding(new VariableWidthBlockEncoding()); - addBlockEncoding(new ByteArrayBlockEncoding()); - addBlockEncoding(new ShortArrayBlockEncoding()); - addBlockEncoding(new IntArrayBlockEncoding()); - addBlockEncoding(new LongArrayBlockEncoding()); - addBlockEncoding(new Fixed12BlockEncoding()); - addBlockEncoding(new Int128ArrayBlockEncoding()); - addBlockEncoding(new DictionaryBlockEncoding()); - addBlockEncoding(new ArrayBlockEncoding()); - addBlockEncoding(new MapBlockEncoding()); - addBlockEncoding(new RowBlockEncoding()); - addBlockEncoding(new RunLengthBlockEncoding()); - } - - private void addBlockEncoding(BlockEncoding blockEncoding) - { - blockEncodingsByClass.put(blockEncoding.getBlockClass(), blockEncoding); - blockEncodingsByName.put(blockEncoding.getName(), blockEncoding); - } - - @Override - public Block readBlock(SliceInput input) - { - // read the encoding name - String encodingName = readLengthPrefixedString(input); - - // look up the encoding factory - BlockEncoding blockEncoding = blockEncodingsByName.get(encodingName); - checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); - - // load read the encoding factory from the output stream - return blockEncoding.readBlock(this, input); - } - - @Override - public long estimatedWriteSize(Block block) - { - while (true) { - BlockEncoding blockEncoding = blockEncodingsByClass.get(block.getClass()); - // see if a replacement block should be written instead - Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); - if (replacementBlock.isPresent()) { - block = replacementBlock.get(); - continue; - } - // length of encoding name + encoding name + block size - // TODO: improve this estimate by adding estimatedWriteSize to BlockEncoding interface - return SIZE_OF_INT + blockEncoding.getName().length() + block.getSizeInBytes(); - } - } - - @Override - public void writeBlock(SliceOutput output, Block block) - { - while (true) { - // look up the encoding factory - BlockEncoding blockEncoding = blockEncodingsByClass.get(block.getClass()); - checkArgument(blockEncoding != null, "Cannot write block %s as no encoding was found", block); - - // see if a replacement block should be written instead - Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); - if (replacementBlock.isPresent()) { - block = replacementBlock.get(); - continue; - } - - // write the name to the output - writeLengthPrefixedString(output, blockEncoding.getName()); - - // write the block to the output - blockEncoding.writeBlock(this, output, block); - - break; - } - } - - @Override - public Type readType(SliceInput sliceInput) - { - requireNonNull(sliceInput, "sliceInput is null"); - - TypeId id = TypeId.of(readLengthPrefixedString(sliceInput)); - for (Type type : TYPES) { - if (type.getTypeId().equals(id)) { - return type; - } - } - throw new IllegalArgumentException("Type not found: " + id); - } - - @Override - public void writeType(SliceOutput sliceOutput, Type type) - { - requireNonNull(sliceOutput, "sliceOutput is null"); - requireNonNull(type, "type is null"); - writeLengthPrefixedString(sliceOutput, type.getTypeId().getId()); - } - - private static String readLengthPrefixedString(SliceInput input) - { - int length = input.readInt(); - byte[] bytes = new byte[length]; - input.readBytes(bytes); - return new String(bytes, UTF_8); - } - - private static void writeLengthPrefixedString(SliceOutput output, String value) - { - byte[] bytes = value.getBytes(UTF_8); - output.writeInt(bytes.length); - output.writeBytes(bytes); - } -} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java deleted file mode 100644 index 924139de39cf..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import io.airlift.slice.BasicSliceInput; -import io.airlift.slice.DynamicSliceOutput; -import io.airlift.slice.Slice; -import io.airlift.slice.SliceOutput; -import io.trino.spi.block.Block; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.CharType.createCharType; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DecimalType.createDecimalType; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.createTimeType; -import static io.trino.spi.type.TimeWithTimeZoneType.createTimeWithTimeZoneType; -import static io.trino.spi.type.TimestampType.createTimestampType; -import static io.trino.spi.type.TimestampWithTimeZoneType.createTimestampWithTimeZoneType; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestHiveBlockEncodingSerde -{ - @Test - public void testSerialization() - { - testSerialization(BOOLEAN); - testSerialization(TINYINT); - testSerialization(SMALLINT); - testSerialization(INTEGER); - testSerialization(BIGINT); - testSerialization(REAL); - testSerialization(DOUBLE); - testSerialization(createDecimalType(7, 2)); - testSerialization(createDecimalType(37, 2)); - testSerialization(createCharType(10)); - testSerialization(createVarcharType(10)); - testSerialization(createUnboundedVarcharType()); - testSerialization(DATE); - testSerialization(createTimeType(0)); - testSerialization(createTimeType(3)); - testSerialization(createTimeType(12)); - testSerialization(createTimeWithTimeZoneType(0)); - testSerialization(createTimeWithTimeZoneType(3)); - testSerialization(createTimeWithTimeZoneType(12)); - testSerialization(createTimestampType(0)); - testSerialization(createTimestampType(3)); - testSerialization(createTimestampType(12)); - testSerialization(createTimestampWithTimeZoneType(0)); - testSerialization(createTimestampWithTimeZoneType(3)); - testSerialization(createTimestampWithTimeZoneType(12)); - } - - private void testSerialization(Type type) - { - Slice serialized; - try (SliceOutput sliceOutput = new DynamicSliceOutput(0)) { - Block block = type.createBlockBuilder(null, 0).build(); - new HiveBlockEncodingSerde().writeBlock(sliceOutput, block); - serialized = sliceOutput.slice(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - try (BasicSliceInput input = serialized.getInput()) { - Block read = new HiveBlockEncodingSerde().readBlock(input); - assertThat(read.getPositionCount()).isEqualTo(0); - } - } -}