diff --git a/pom.xml b/pom.xml index 1611d23588f03..7763591279753 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ 2.0.16 3.9.1 1.3.0 - 30.0.1 + 35.0.1 2.3.1 4.0.6 0.14.0 diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index aec5e2d885830..dde87dcd5b2dc 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -32,7 +32,7 @@ - org.hibernate + org.hibernate.validator hibernate-validator 8.0.3.Final @@ -41,6 +41,16 @@ jakarta.el 4.0.1 + + at.yawk.lz4 + lz4-java + 1.10.2 + + + org.mozilla + rhino + 1.8.1 + diff --git a/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java b/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java new file mode 100644 index 0000000000000..24d4fa716b6fe --- /dev/null +++ b/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java @@ -0,0 +1,170 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.druid.segment; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import org.apache.druid.collections.bitmap.BitmapFactory; +import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import org.apache.druid.query.OrderBy; +import org.apache.druid.segment.DimensionHandler; +import org.apache.druid.segment.Metadata; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.BaseColumnHolder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.data.Indexed; +import org.joda.time.Interval; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class PrestoQueryableIndex + implements QueryableIndex +{ + private final Interval dataInterval; + private final List columnNames; + private final Indexed availableDimensions; + private final BitmapFactory bitmapFactory; + private final Map> columns; + private final SmooshedFileMapper fileMapper; + @Nullable + private final Metadata metadata; + private final Supplier> dimensionHandlers; + + public PrestoQueryableIndex( + Interval dataInterval, + Indexed dimNames, + BitmapFactory bitmapFactory, + Map> columns, + SmooshedFileMapper fileMapper, + @Nullable Metadata metadata, + boolean lazy) + { + Preconditions.checkNotNull(columns.get(ColumnHolder.TIME_COLUMN_NAME)); + this.dataInterval = Preconditions.checkNotNull(dataInterval, "dataInterval"); + ImmutableList.Builder columnNamesBuilder = ImmutableList.builder(); + for (String column : columns.keySet()) { + if (!ColumnHolder.TIME_COLUMN_NAME.equals(column)) { + columnNamesBuilder.add(column); + } + } + this.columnNames = columnNamesBuilder.build(); + this.availableDimensions = dimNames; + this.bitmapFactory = bitmapFactory; + this.columns = columns; + this.fileMapper = fileMapper; + this.metadata = metadata; + + if (lazy) { + this.dimensionHandlers = Suppliers.memoize(() -> initDimensionHandlers(availableDimensions)); + } + else { + this.dimensionHandlers = () -> initDimensionHandlers(availableDimensions); + } + } + + @Override + public Interval getDataInterval() + { + return dataInterval; + } + + @Override + public int getNumRows() + { + return columns.get(ColumnHolder.TIME_COLUMN_NAME).get().getLength(); + } + + @Override + public List getColumnNames() + { + return columnNames; + } + + @Override + public Indexed getAvailableDimensions() + { + return availableDimensions; + } + + @Override + public BitmapFactory getBitmapFactoryForDimensions() + { + return bitmapFactory; + } + + @Nullable + @Override + public BaseColumnHolder getColumnHolder(String columnName) + { + Supplier columnHolderSupplier = (Supplier) this.columns.get(columnName); + return columnHolderSupplier == null ? null : (BaseColumnHolder) columnHolderSupplier.get(); + } + + @VisibleForTesting + public Map> getColumns() + { + return columns; + } + + @VisibleForTesting + public SmooshedFileMapper getFileMapper() + { + return fileMapper; + } + + @Override + public void close() + { + if (fileMapper != null) { + fileMapper.close(); + } + } + + @Override + public Metadata getMetadata() + { + return metadata; + } + + @Override + public List getOrdering() + { + return Collections.emptyList(); + } + + @Override + public Map getDimensionHandlers() + { + return dimensionHandlers.get(); + } + + private Map initDimensionHandlers(Indexed availableDimensions) + { + Map dimensionHandlerMap = Maps.newLinkedHashMap(); + for (String dim : availableDimensions) { + final ColumnHolder columnHolder = getColumnHolder(dim); + final DimensionHandler handler = columnHolder.getColumnFormat().getColumnHandler(dim); + dimensionHandlerMap.put(dim, handler); + } + return dimensionHandlerMap; + } +} diff --git a/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java b/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java index 23c1b5dcc4afd..f8e277dd313d7 100644 --- a/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java +++ b/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java @@ -22,13 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.common.collect.Streams; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.utils.SerializerUtils; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.segment.Metadata; import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.SimpleQueryableIndex; import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; @@ -66,7 +64,6 @@ public class V9SegmentIndexSource public V9SegmentIndexSource(SegmentColumnSource segmentColumnSource) { this.segmentColumnSource = requireNonNull(segmentColumnSource, "segmentColumnSource is null"); - NullHandling.initializeForTests(); } @Override @@ -74,10 +71,11 @@ public QueryableIndex loadIndex(List columnHandles) throws IOException { ByteBuffer indexBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(INDEX_METADATA_FILE_NAME)); - GenericIndexed.read(indexBuffer, STRING_STRATEGY); + GenericIndexed.read(indexBuffer, STRING_STRATEGY, null); GenericIndexed allDimensions = GenericIndexed.read( indexBuffer, - STRING_STRATEGY); + STRING_STRATEGY, + null); Interval dataInterval = Intervals.utc(indexBuffer.getLong(), indexBuffer.getLong()); @@ -114,8 +112,13 @@ public QueryableIndex loadIndex(List columnHandles) columns.put(TIME_COLUMN_NAME, () -> createColumnHolder(TIME_COLUMN_NAME)); Indexed indexed = new ListIndexed<>(availableDimensions); - // TODO: get rid of the time column by creating Presto's SimpleQueryableIndex impl - return new SimpleQueryableIndex( + /* + * Druid 35.0.1 made SimpleQueryableIndex abstract, so created PrestoQueryableIndex + * based on the original implementation. + * TODO: Refactor PrestoQueryableIndex to remove the dependency on the __time column + * and implement a fully Presto-specific QueryableIndex. + */ + return new PrestoQueryableIndex( dataInterval, indexed, segmentBitmapSerdeFactory.getBitmapFactory(), @@ -136,7 +139,7 @@ private ColumnHolder createColumnHolder(String columnName) try { ByteBuffer columnData = ByteBuffer.wrap(segmentColumnSource.getColumnData(columnName)); ColumnDescriptor columnDescriptor = readColumnDescriptor(columnData); - return columnDescriptor.read(columnData, ColumnConfig.DEFAULT, null); + return columnDescriptor.read(columnData, ColumnConfig.DEFAULT, null, null); } catch (IOException e) { throw new PrestoException(DRUID_SEGMENT_LOAD_ERROR, e);