Skip to content

Add boolean to varchar coercion for hive #19492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
findinpath wants to merge 1 commit into from
Closed

Add boolean to varchar coercion for hive #19492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/BooleanCoercer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.hive.coercions;

import io.airlift.slice.Slice;
import io.airlift.slice.Slices;
import io.trino.spi.TrinoException;
import io.trino.spi.block.Block;
import io.trino.spi.block.BlockBuilder;
import io.trino.spi.type.BooleanType;
import io.trino.spi.type.VarcharType;

import static io.airlift.slice.SliceUtf8.countCodePoints;
import static io.trino.spi.StandardErrorCode.INVALID_ARGUMENTS;
import static io.trino.spi.type.BooleanType.BOOLEAN;
import static java.lang.String.format;

public final class BooleanCoercer
{
private BooleanCoercer() {}

public static class BooleanToVarcharCoercer
extends TypeCoercer<BooleanType, VarcharType>
{
public BooleanToVarcharCoercer(VarcharType toType)
{
super(BOOLEAN, toType);
}

@Override
protected void applyCoercedValue(BlockBuilder blockBuilder, Block block, int position)
{
boolean value = BOOLEAN.getBoolean(block, position);
Slice converted = Slices.utf8Slice(value ? "TRUE" : "FALSE");
if (!toType.isUnbounded() && countCodePoints(converted) > toType.getBoundedLength()) {
throw new TrinoException(INVALID_ARGUMENTS, format("Varchar representation of %s exceeds %s bounds", value, toType));
}
toType.writeSlice(blockBuilder, converted);
}
}
}
5 changes: 5 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/coercions/CoercionUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableList;
import io.trino.plugin.hive.HiveTimestampPrecision;
import io.trino.plugin.hive.HiveType;
import io.trino.plugin.hive.coercions.BooleanCoercer.BooleanToVarcharCoercer;
import io.trino.plugin.hive.coercions.DateCoercer.VarcharToDateCoercer;
import io.trino.plugin.hive.coercions.TimestampCoercer.VarcharToLongTimestampCoercer;
import io.trino.plugin.hive.coercions.TimestampCoercer.VarcharToShortTimestampCoercer;
Expand All @@ -34,6 +35,7 @@
import io.trino.spi.block.RowBlock;
import io.trino.spi.type.ArrayType;
import io.trino.spi.type.BigintType;
import io.trino.spi.type.BooleanType;
import io.trino.spi.type.CharType;
import io.trino.spi.type.DateType;
import io.trino.spi.type.DecimalType;
Expand Down Expand Up @@ -115,6 +117,9 @@ public static Type createTypeFromCoercer(TypeManager typeManager, HiveType fromH
if (fromType instanceof VarcharType fromVarcharType && toType instanceof DateType toDateType) {
return Optional.of(new VarcharToDateCoercer(fromVarcharType, toDateType));
}
if (fromType instanceof BooleanType && toType instanceof VarcharType toVarcharType) {
return Optional.of(new BooleanToVarcharCoercer(toVarcharType));
Comment thread
ebyhr marked this conversation as resolved.
Outdated
}
if (fromType instanceof CharType fromCharType && toType instanceof CharType toCharType) {
if (narrowerThan(toCharType, fromCharType)) {
return Optional.of(new CharCoercer(fromCharType, toCharType));
Expand Down
5 changes: 5 additions & 0 deletions plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcTypeTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package io.trino.plugin.hive.orc;

import io.trino.orc.metadata.OrcType.OrcTypeKind;
import io.trino.plugin.hive.coercions.BooleanCoercer.BooleanToVarcharCoercer;
import io.trino.plugin.hive.coercions.DateCoercer.VarcharToDateCoercer;
import io.trino.plugin.hive.coercions.DoubleToVarcharCoercer;
import io.trino.plugin.hive.coercions.TimestampCoercer.LongTimestampToVarcharCoercer;
Expand All @@ -27,6 +28,7 @@

import java.util.Optional;

import static io.trino.orc.metadata.OrcType.OrcTypeKind.BOOLEAN;
import static io.trino.orc.metadata.OrcType.OrcTypeKind.DOUBLE;
import static io.trino.orc.metadata.OrcType.OrcTypeKind.STRING;
import static io.trino.orc.metadata.OrcType.OrcTypeKind.TIMESTAMP;
Expand Down Expand Up @@ -58,6 +60,9 @@ private OrcTypeTranslator() {}
if (fromOrcType == DOUBLE && toTrinoType instanceof VarcharType varcharType) {
return Optional.of(new DoubleToVarcharCoercer(varcharType, true));
}
if (fromOrcType == BOOLEAN && toTrinoType instanceof VarcharType varcharType) {
return Optional.of(new BooleanToVarcharCoercer(varcharType));
}
return Optional.empty();
}

Expand Down
4 changes: 3 additions & 1 deletion plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveCoercionPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.List;

import static com.google.common.base.Preconditions.checkArgument;
import static io.trino.plugin.hive.HiveType.HIVE_BOOLEAN;
import static io.trino.plugin.hive.HiveType.HIVE_BYTE;
import static io.trino.plugin.hive.HiveType.HIVE_DATE;
import static io.trino.plugin.hive.HiveType.HIVE_DOUBLE;
Expand Down Expand Up @@ -72,7 +73,8 @@ private boolean canCoerce(HiveType fromHiveType, HiveType toHiveType, HiveTimest
return toType instanceof CharType;
}
if (toType instanceof VarcharType) {
return fromHiveType.equals(HIVE_BYTE) ||
return fromHiveType.equals(HIVE_BOOLEAN) ||
fromHiveType.equals(HIVE_BYTE) ||
fromHiveType.equals(HIVE_SHORT) ||
fromHiveType.equals(HIVE_INT) ||
fromHiveType.equals(HIVE_LONG) ||
Expand Down
55 changes: 55 additions & 0 deletions plugin/trino-hive/src/test/java/io/trino/plugin/hive/coercions/TestBooleanCoercer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.hive.coercions;

import io.airlift.slice.Slice;
import io.trino.plugin.hive.coercions.CoercionUtils.CoercionContext;
import io.trino.spi.TrinoException;
import io.trino.spi.block.Block;
import io.trino.spi.type.Type;
import org.testng.annotations.Test;

import static io.airlift.slice.Slices.utf8Slice;
import static io.trino.plugin.hive.HiveTimestampPrecision.DEFAULT_PRECISION;
import static io.trino.plugin.hive.HiveType.toHiveType;
import static io.trino.plugin.hive.coercions.CoercionUtils.createCoercer;
import static io.trino.spi.predicate.Utils.blockToNativeValue;
import static io.trino.spi.predicate.Utils.nativeValueToBlock;
import static io.trino.spi.type.BooleanType.BOOLEAN;
import static io.trino.spi.type.VarcharType.createUnboundedVarcharType;
import static io.trino.spi.type.VarcharType.createVarcharType;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

public class TestBooleanCoercer
{
@Test
public void testBooleanToVarchar()
{
assertBooleanToVarcharCoercion(createUnboundedVarcharType(), true, utf8Slice("TRUE"));
assertBooleanToVarcharCoercion(createUnboundedVarcharType(), false, utf8Slice("FALSE"));
assertThatThrownBy(() -> assertBooleanToVarcharCoercion(createVarcharType(1), false, utf8Slice("FALSE")))
.isInstanceOf(TrinoException.class)
.hasMessageContaining("Varchar representation of false exceeds varchar(1) bounds");
}

private void assertBooleanToVarcharCoercion(Type toType, boolean valueToBeCoerced, Slice expectedValue)
{
Block coercedValue = createCoercer(TESTING_TYPE_MANAGER, toHiveType(BOOLEAN), toHiveType(toType), new CoercionContext(DEFAULT_PRECISION, false)).orElseThrow()
.apply(nativeValueToBlock(BOOLEAN, valueToBeCoerced));
assertThat(blockToNativeValue(toType, coercedValue))
.isEqualTo(expectedValue);
}
}
9 changes: 9 additions & 0 deletions ...g/trino-product-tests/src/main/java/io/trino/tests/product/hive/BaseTestHiveCoercion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ protected void doTestHiveCoercion(HiveTableDefinition tableDefinition)
"row_to_row",
"list_to_list",
"map_to_map",
"boolean_to_varchar",
"tinyint_to_smallint",
"tinyint_to_int",
"tinyint_to_bigint",
Expand Down Expand Up @@ -175,6 +176,7 @@ protected void insertTableRows(String tableName, String floatToDoubleType)
" CAST(ROW ('as is', -1, 100, 2323, 12345, 2) AS ROW(keep VARCHAR, ti2si TINYINT, si2int SMALLINT, int2bi INTEGER, bi2vc BIGINT, lower2uppercase BIGINT)), " +
" ARRAY [CAST(ROW (2, -101, 12345, 'removed') AS ROW (ti2int TINYINT, si2bi SMALLINT, bi2vc BIGINT, remove VARCHAR))], " +
" MAP (ARRAY [TINYINT '2'], ARRAY [CAST(ROW (-3, 2323, REAL '0.5') AS ROW (ti2bi TINYINT, int2bi INTEGER, float2double %2$s))]), " +
" TRUE, " +
" TINYINT '-1', " +
" TINYINT '2', " +
" TINYINT '-3', " +
Expand Down Expand Up @@ -225,6 +227,7 @@ protected void insertTableRows(String tableName, String floatToDoubleType)
" CAST(ROW (NULL, 1, -100, -2323, -12345, 2) AS ROW(keep VARCHAR, ti2si TINYINT, si2int SMALLINT, int2bi INTEGER, bi2vc BIGINT, lower2uppercase BIGINT)), " +
" ARRAY [CAST(ROW (-2, 101, -12345, NULL) AS ROW (ti2int TINYINT, si2bi SMALLINT, bi2vc BIGINT, remove VARCHAR))], " +
" MAP (ARRAY [TINYINT '-2'], ARRAY [CAST(ROW (null, -2323, REAL '-1.5') AS ROW (ti2bi TINYINT, int2bi INTEGER, float2double %2$s))]), " +
" FALSE, " +
" TINYINT '1', " +
" TINYINT '-2', " +
" NULL, " +
Expand Down Expand Up @@ -350,6 +353,9 @@ else if (getHiveVersionMajor() == 3 && isFormat.test("orc")) {
.addField("add", null)
.build()) :
"{-2:{\"ti2bi\":null,\"int2bi\":-2323,\"float2double\":-1.5,\"add\":null}}"))
.put("boolean_to_varchar", ImmutableList.of(
"TRUE",
"FALSE"))
.put("tinyint_to_smallint", ImmutableList.of(
-1,
1))
Expand Down Expand Up @@ -834,6 +840,7 @@ private void assertProperAlteredTableSchema(String tableName)
row("row_to_row", "row(keep varchar, ti2si smallint, si2int integer, int2bi bigint, bi2vc varchar, lower2uppercase bigint)"),
row("list_to_list", "array(row(ti2int integer, si2bi bigint, bi2vc varchar))"),
row("map_to_map", "map(integer, row(ti2bi bigint, int2bi bigint, float2double double, add tinyint))"),
row("boolean_to_varchar", "varchar(5)"),
row("tinyint_to_smallint", "smallint"),
row("tinyint_to_int", "integer"),
row("tinyint_to_bigint", "bigint"),
Expand Down Expand Up @@ -900,6 +907,7 @@ private void assertColumnTypes(
.put("row_to_row", engine == Engine.TRINO ? JAVA_OBJECT : STRUCT) // row
.put("list_to_list", ARRAY) // list
.put("map_to_map", JAVA_OBJECT) // map
.put("boolean_to_varchar", VARCHAR)
.put("tinyint_to_smallint", SMALLINT)
.put("tinyint_to_int", INTEGER)
.put("tinyint_to_bigint", BIGINT)
Expand Down Expand Up @@ -965,6 +973,7 @@ private static void alterTableColumnTypes(String tableName)
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN row_to_row row_to_row struct<keep:string, ti2si:smallint, si2int:int, int2bi:bigint, bi2vc:string, LOWER2UPPERCASE:bigint>", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN list_to_list list_to_list array<struct<ti2int:int, si2bi:bigint, bi2vc:string>>", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN map_to_map map_to_map map<int,struct<ti2bi:bigint, int2bi:bigint, float2double:double, add:tinyint>>", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN boolean_to_varchar boolean_to_varchar varchar(5)", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN tinyint_to_smallint tinyint_to_smallint smallint", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN tinyint_to_int tinyint_to_int int", tableName));
onHive().executeQuery(format("ALTER TABLE %s CHANGE COLUMN tinyint_to_bigint tinyint_to_bigint bigint", tableName));
Expand Down
1 change: 1 addition & 0 deletions ...t-tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnPartitionedTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionBui
" row_to_row STRUCT<keep: STRING, ti2si: TINYINT, si2int: SMALLINT, int2bi: INT, bi2vc: BIGINT, lower2uppercase: BIGINT>, " +
" list_to_list ARRAY<STRUCT<ti2int: TINYINT, si2bi: SMALLINT, bi2vc: BIGINT, remove: STRING>>, " +
" map_to_map MAP<TINYINT, STRUCT<ti2bi: TINYINT, int2bi: INT, float2double: " + floatType + ">>, " +
" boolean_to_varchar BOOLEAN," +
" tinyint_to_smallint TINYINT," +
" tinyint_to_int TINYINT," +
" tinyint_to_bigint TINYINT," +
Expand Down
2 changes: 2 additions & 0 deletions ...tests/src/main/java/io/trino/tests/product/hive/TestHiveCoercionOnUnpartitionedTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ private static HiveTableDefinition.HiveTableDefinitionBuilder tableDefinitionBui
row_to_row STRUCT<keep: STRING, ti2si: TINYINT, si2int: SMALLINT, int2bi: INT, bi2vc: BIGINT, lower2uppercase: BIGINT>,
list_to_list ARRAY<STRUCT<ti2int: TINYINT, si2bi: SMALLINT, bi2vc: BIGINT, remove: STRING>>,
map_to_map MAP<TINYINT, STRUCT<ti2bi: TINYINT, int2bi: INT, float2double: FLOAT>>,
boolean_to_varchar BOOLEAN,
boolean_to_varchar BOOLEAN,
tinyint_to_smallint TINYINT,
tinyint_to_int TINYINT,
tinyint_to_bigint TINYINT,
Expand Down