diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSink.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSink.java index ef0254248663..11ccbdc4dad3 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSink.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSink.java @@ -28,8 +28,6 @@ import io.trino.plugin.hive.FileWriter; import io.trino.plugin.hive.HdfsEnvironment; import io.trino.plugin.hive.HivePartitionKey; -import io.trino.plugin.hive.HiveType; -import io.trino.plugin.hive.HiveTypeName; import io.trino.plugin.hive.RecordFileWriter; import io.trino.plugin.hive.parquet.ParquetFileWriter; import io.trino.plugin.hive.util.HiveWriteUtils; @@ -47,7 +45,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.mapred.JobConf; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.joda.time.DateTimeZone; @@ -67,6 +64,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.airlift.slice.Slices.wrappedBuffer; import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_WRITE; +import static io.trino.plugin.deltalake.DeltaLakeSchemaProperties.buildHiveSchema; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getCompressionCodec; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetWriterBlockSize; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetWriterPageSize; @@ -80,7 +78,6 @@ import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; import static java.util.function.Function.identity; -import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.common.FileUtils.escapePathName; @@ -508,7 +505,7 @@ private FileWriter createParquetFileWriter(Path path) private FileWriter createRecordFileWriter(Path path) { - Properties schema = buildSchemaProperties(dataColumnNames, dataColumnTypes); + Properties schema = buildHiveSchema(dataColumnNames, dataColumnTypes); return new RecordFileWriter( path, dataColumnNames, @@ -521,18 +518,6 @@ private FileWriter createRecordFileWriter(Path path) session); } - static Properties buildSchemaProperties(List columnNames, List columnTypes) - { - Properties schema = new Properties(); - schema.setProperty(IOConstants.COLUMNS, String.join(",", columnNames)); - schema.setProperty(IOConstants.COLUMNS_TYPES, columnTypes.stream() - .map(DeltaHiveTypeTranslator::toHiveType) - .map(HiveType::getHiveTypeName) - .map(HiveTypeName::toString) - .collect(joining(":"))); - return schema; - } - private Page getDataPage(Page page) { Block[] blocks = new Block[dataColumnInputIndex.length]; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeSchemaProperties.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeSchemaProperties.java index 7467a049e370..88fca1beed05 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeSchemaProperties.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeSchemaProperties.java @@ -15,14 +15,20 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.trino.plugin.hive.HiveType; +import io.trino.plugin.hive.HiveTypeName; import io.trino.plugin.hive.metastore.Database; import io.trino.spi.session.PropertyMetadata; +import io.trino.spi.type.Type; +import org.apache.hadoop.hive.ql.io.IOConstants; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Properties; import static io.trino.spi.session.PropertyMetadata.stringProperty; +import static java.util.stream.Collectors.joining; public class DeltaLakeSchemaProperties { @@ -49,4 +55,16 @@ public static Optional getLocation(Map schemaProperties) { return Optional.ofNullable((String) schemaProperties.get(LOCATION_PROPERTY)); } + + public static Properties buildHiveSchema(List columnNames, List columnTypes) + { + Properties schema = new Properties(); + schema.setProperty(IOConstants.COLUMNS, String.join(",", columnNames)); + schema.setProperty(IOConstants.COLUMNS_TYPES, columnTypes.stream() + .map(DeltaHiveTypeTranslator::toHiveType) + .map(HiveType::getHiveTypeName) + .map(HiveTypeName::toString) + .collect(joining(":"))); + return schema; + } } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeUpdatablePageSource.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeUpdatablePageSource.java index 0f9b15582762..be85b9059c4a 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeUpdatablePageSource.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeUpdatablePageSource.java @@ -63,6 +63,7 @@ import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_DATA; import static io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_WRITE; import static io.trino.plugin.deltalake.DeltaLakePageSink.createPartitionValues; +import static io.trino.plugin.deltalake.DeltaLakeSchemaProperties.buildHiveSchema; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.getParquetMaxReadBlockSize; import static io.trino.plugin.deltalake.DeltaLakeSessionProperties.isParquetUseColumnIndex; import static io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.extractSchema; @@ -577,7 +578,7 @@ private DeltaLakeWriter createWriter(Path targetFile, List columnNames, List columnTypes) - { - // TODO copied out from DeltaLakePageSink. Extrat to utility class (https://github.com/trinodb/trino/issues/12030) - Properties schema = new Properties(); - schema.setProperty(IOConstants.COLUMNS, String.join(",", columnNames)); - schema.setProperty(IOConstants.COLUMNS_TYPES, columnTypes.stream() - .map(DeltaHiveTypeTranslator::toHiveType) - .map(HiveType::getHiveTypeName) - .map(HiveTypeName::toString) - .collect(joining(":"))); - return schema; - } - private void writeMetadataEntry(PageBuilder pageBuilder, RowType entryType, MetadataEntry metadataEntry) { pageBuilder.declarePosition();