diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp index d94a69f9ed5b2..10b7a483430e9 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp @@ -37,6 +37,7 @@ #include "velox/common/compression/Compression.h" // clang-format on +#include #include using namespace facebook::velox; @@ -799,6 +800,56 @@ std::unique_ptr toFilter( VELOX_UNSUPPORTED("Unsupported filter found."); } +template +TypePtr fieldNamesToLowerCase(const TypePtr& type) { + return type; +} + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type); + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type); + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type); + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type) { + auto& elementType = type->childAt(0); + return std::make_shared(VELOX_DYNAMIC_TYPE_DISPATCH( + fieldNamesToLowerCase, elementType->kind(), elementType)); +} + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type) { + auto& keyType = type->childAt(0); + auto& valueType = type->childAt(1); + return std::make_shared( + VELOX_DYNAMIC_TYPE_DISPATCH( + fieldNamesToLowerCase, keyType->kind(), keyType), + VELOX_DYNAMIC_TYPE_DISPATCH( + fieldNamesToLowerCase, valueType->kind(), valueType)); +} + +template <> +TypePtr fieldNamesToLowerCase(const TypePtr& type) { + auto& rowType = type->asRow(); + std::vector names; + std::vector types; + names.reserve(type->size()); + types.reserve(type->size()); + for (int i = 0; i < rowType.size(); i++) { + std::string name = rowType.nameOf(i); + folly::toLowerAscii(name); + names.push_back(std::move(name)); + auto& childType = rowType.childAt(i); + types.push_back(VELOX_DYNAMIC_TYPE_DISPATCH( + fieldNamesToLowerCase, childType->kind(), childType)); + } + return std::make_shared(std::move(names), std::move(types)); +} + std::shared_ptr toConnectorTableHandle( const protocol::TableHandle& tableHandle, const VeloxExprConverter& exprConverter, @@ -839,8 +890,15 @@ std::shared_ptr toConnectorTableHandle( names.reserve(hiveLayout->dataColumns.size()); types.reserve(hiveLayout->dataColumns.size()); for (auto& column : hiveLayout->dataColumns) { - names.push_back(column.name); - types.push_back(typeParser.parse(column.type)); + std::string name = column.name; + folly::toLowerAscii(name); + names.emplace_back(std::move(name)); + auto parsedType = typeParser.parse(column.type); + // The type from the metastore may have upper case letters + // in field names, convert them all to lower case to be + // compatible with Presto. + types.push_back(VELOX_DYNAMIC_TYPE_DISPATCH( + fieldNamesToLowerCase, parsedType->kind(), parsedType)); } dataColumns = ROW(std::move(names), std::move(types)); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java index 0c3ba4339283a..34458eccf9c26 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java @@ -14,6 +14,10 @@ package com.facebook.presto.nativeworker; import com.facebook.presto.Session; +import com.facebook.presto.common.type.RowType; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; @@ -21,10 +25,18 @@ import org.testng.annotations.Test; import java.util.Map; +import java.util.Optional; import java.util.UUID; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.hive.HiveStorageFormat.DWRF; +import static com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY; +import static com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY; +import static com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY; +import static com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY; +import static com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createBucketedCustomer; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createBucketedLineitemAndOrders; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createCustomer; @@ -45,6 +57,7 @@ import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; +import static com.facebook.presto.transaction.TransactionBuilder.transaction; import static java.lang.String.format; import static org.testng.Assert.assertEquals; @@ -1274,6 +1287,50 @@ public void testUnionAllInsert() } } + @Test + public void testSelectFieldsWithCapitalLetters() + { + Session session = Session.builder(getSession()) + // This is needed for Spark. + .setCatalogSessionProperty("hive", "optimized_partition_update_serialization_enabled", "false") + .build(); + String tmpTableName = generateRandomTableName(); + try { + QueryRunner queryRunner = getQueryRunner(); + // We have to create the table through metadata, rather than + // through Presto SQL since, if we use the latter, Presto will + // convert the field names to lower case. + SchemaTableName table = new SchemaTableName(session.getSchema().get(), tmpTableName); + Map tableProperties = ImmutableMap.builder() + .put(STORAGE_FORMAT_PROPERTY, DWRF) + .put(PARTITIONED_BY_PROPERTY, ImmutableList.of()) + .put(BUCKETED_BY_PROPERTY, ImmutableList.of()) + .put(BUCKET_COUNT_PROPERTY, 0) + .put(SORTED_BY_PROPERTY, ImmutableList.of()) + .build(); + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(table, ImmutableList.of( + new ColumnMetadata("col", RowType.from(ImmutableList.of( + new RowType.Field(Optional.of("NationKey"), BIGINT), + new RowType.Field(Optional.of("NAME"), VARCHAR), + new RowType.Field(Optional.of("ReGiOnKeY"), BIGINT), + new RowType.Field(Optional.of("commenT"), VARCHAR))))), + tableProperties); + transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) + .singleStatement() + .execute(session, s -> { + queryRunner.getMetadata().createTable(s, s.getCatalog().get(), tableMetadata, false); + }); + + // Write some data so we can read it back. + queryRunner.execute(session, String.format("INSERT INTO %s SELECT cast(row(nationkey, name, regionkey, comment) as row(nationkey bigint, name varchar, regionkey bigint, comment varchar)) FROM nation", tmpTableName)); + // This should work since Presto is case insensitive. + assertQuery(String.format("SELECT col.nationkey, col.name, col.regionkey, col.comment FROM %s", tmpTableName), "SELECT nationkey, name, regionkey, comment FROM nation"); + } + finally { + dropTableIfExists(tmpTableName); + } + } + private void assertQueryResultCount(String sql, int expectedResultCount) { assertEquals(getQueryRunner().execute(sql).getRowCount(), expectedResultCount);