diff --git a/pom.xml b/pom.xml index 8e671978c0df4..b1f01caf0b4be 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ 2.13.1 1.55 7.5 - 8.11.3 + 9.12.0 3.8.0 1.13.1 9.7.1 @@ -66,7 +66,7 @@ 313 2.0.16 3.9.1 - 0.11.0 + 1.3.0 30.0.1 2.3.1 4.0.5 @@ -80,7 +80,7 @@ 2.15.4 3.0.0 1.11.4 - 1.26.2 + 1.27.1 4.29.0 12.0.18 4.1.128.Final @@ -107,8 +107,14 @@ 4g 1.75.0 -missing - 1.17.1 + 1.17.2 2.32.9 + 1.19.0 + + 8.5.2 + 2.2.0 + 5.0.1 + true @@ -409,7 +415,7 @@ commons-io commons-io - 2.16.1 + 2.18.0 @@ -1485,7 +1491,7 @@ it.unimi.dsi fastutil - 8.5.2 + ${dep.fastutil.version} @@ -2130,6 +2136,24 @@ 9.6.3-4 + + org.apache.pinot + pinot-spi + ${dep.pinot.version} + + + + org.apache.pinot + pinot-common + ${dep.pinot.version} + + + + org.apache.pinot + pinot-core + ${dep.pinot.version} + + org.apache.kafka kafka_2.12 @@ -2278,22 +2302,16 @@ ${dep.kafka.version} - - org.apache.pinot - presto-pinot-driver - ${dep.pinot.version} - - org.xerial.snappy snappy-java - 1.1.10.4 + 1.1.10.7 com.github.luben zstd-jni - 1.5.2-3 + 1.5.6-9 @@ -2462,7 +2480,7 @@ org.apache.lucene - lucene-analyzers-common + lucene-analysis-common ${dep.lucene.version} @@ -2475,13 +2493,13 @@ org.locationtech.jts jts-core - 1.19.0 + ${dep.jts.version} org.locationtech.jts.io jts-io-common - 1.19.0 + ${dep.jts.version} junit @@ -2550,7 +2568,7 @@ com.clearspring.analytics stream - 2.9.5 + 2.9.8 @@ -2610,13 +2628,13 @@ org.apache.datasketches datasketches-memory - 2.2.0 + ${dep.datasketches-memory.version} org.apache.datasketches datasketches-java - 5.0.1 + ${dep.datasketches-java.version} diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index c7c51d65ce5cf..a951c5a0bd986 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -16,6 +16,7 @@ 7.17.27 2.24.3 true + 8.11.3 diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index d9003a304ae95..ec7b0f5011378 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -15,6 +15,7 @@ ${project.parent.basedir} 8 true + 8.11.3 @@ -306,6 +307,7 @@ org.apache.lucene lucene-analyzers-common + ${dep.lucene.version} diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml index 11eb4471d0dda..8458e1b26b6a8 100644 --- a/presto-pinot-toolkit/pom.xml +++ b/presto-pinot-toolkit/pom.xml @@ -17,6 +17,9 @@ 1.38.0 17 true + 1.20.0 + 8.5.15 + 3.0.2 @@ -26,269 +29,82 @@ javax.annotation-api 1.3.2 + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.2 + org.apache.pinot - presto-pinot-driver + pinot-spi - org.apache.lucene - lucene-core - - - commons-io - commons-io + org.apache.logging.log4j + log4j-1.2-api - com.jayway.jsonpath - json-path + org.apache.logging.log4j + log4j-slf4j2-impl - io.grpc - grpc-api - - - io.grpc - grpc-context - - - io.grpc - grpc-netty-shaded - - - org.codehaus.mojo - animal-sniffer-annotations + org.apache.logging.log4j + log4j-core - commons-codec - commons-codec - - - commons-codec - commons-lang3 - - - org.apache.commons - commons-lang3 - - - commons-logging - commons-logging - - - org.apache.httpcomponents - httpclient - - - org.apache.thrift - libthrift - - - com.github.luben - zstd-jni + org.slf4j + jcl-over-slf4j - org.glassfish.jersey.core - jersey-common + org.apache.logging.log4j + log4j-1.2-api org.glassfish.hk2.external aopalliance-repackaged - com.google.errorprone - error_prone_annotations - - - org.checkerframework - checker-compat-qual - - - org.reflections - reflections - - - org.slf4j - slf4j-api - - - jline - jline - - - org.slf4j - slf4j-log4j12 - - - org.antlr - antlr4-annotations - - - org.apache.kafka - kafka-clients - - - org.codehaus.jackson - jackson-mapper-asl - - - org.apache.kafka - kafka_2.10 - - - commons-beanutils - commons-beanutils-core - - - log4j - log4j - - - com.fasterxml.jackson.core - jackson-databind + joda-time + joda-time + + + + + org.apache.pinot + pinot-common + - com.fasterxml.jackson.core - jackson-core + org.beanshell + bsh - com.fasterxml.jackson.core - jackson-annotations + org.apache.thrift + libthrift - jakarta.validation - jakarta.validation-api + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml jakarta.servlet jakarta.servlet-api - - jakarta.validation - jakarta.validation-api - org.glassfish.hk2.external - jakarta.inject - - - jakarta.ws.rs - jakarta.ws.rs-api - - - jakarta.annotation - jakarta.annotation-api - - - com.google.guava - guava - - - joda-time - joda-time - - - org.apache.httpcomponents - httpcore - - - org.apache.zookeeper - zookeeper - - - commons-codec - commons-codec - - - org.slf4j - jcl-over-slf4j - - - jakarta.servlet - servlet-api - - - com.clearspring.analytics - stream - - - com.tdunning - t-digest - - - org.apache.datasketches - datasketches-java - - - org.roaringbitmap - RoaringBitmap - - - jakarta.inject - jakarta.inject-api - - - org.reflections - reflections - - - jakarta.validation - jakarta.validation-api - - - org.checkerframework - checker-qual - - - org.ow2.asm - asm - - - io.grpc - grpc-stub - - - io.grpc - grpc-protobuf - - - io.perfmark - perfmark-api - - - org.xerial.snappy - snappy-java - - - com.google.protobuf - protobuf-java - - - io.netty - netty-all - - - io.netty - netty-handler - - - io.netty - netty-transport-native-kqueue - - - io.netty - netty-transport-native-epoll - - - org.webjars - swagger-ui + aopalliance-repackaged + + org.apache.pinot + pinot-core + + org.apache.calcite calcite-core @@ -322,10 +138,6 @@ org.checkerframework checker-qual - - org.slf4j - slf4j-api - com.jayway.jsonpath json-path @@ -391,6 +203,13 @@ grpc-stub + + + org.apache-extras.beanshell + bsh + 2.0b6 + + io.grpc grpc-netty-shaded @@ -409,7 +228,6 @@ com.clearspring.analytics stream - test org.slf4j @@ -422,7 +240,6 @@ com.tdunning t-digest 3.2 - test org.slf4j @@ -434,19 +251,7 @@ org.apache.datasketches datasketches-java - test - - - org.slf4j - slf4j-api - - - - - - org.roaringbitmap - RoaringBitmap - test + 6.1.1 org.slf4j @@ -465,11 +270,6 @@ jmxutils - - javax.inject - javax.inject - - jakarta.inject jakarta.inject-api @@ -478,11 +278,23 @@ com.facebook.airlift bootstrap + + + javax.inject + javax.inject + + com.facebook.airlift json + + + javax.inject + javax.inject + + @@ -493,6 +305,12 @@ com.facebook.airlift configuration + + + javax.inject + javax.inject + + @@ -509,6 +327,12 @@ com.google.inject guice + + + javax.inject + javax.inject + + @@ -586,6 +410,20 @@ com.facebook.presto presto-main-base test + + + javax.inject + javax.inject + + + com.teradata + re2j-td + + + org.apache.lucene + lucene-analyzers-common + + @@ -598,6 +436,20 @@ presto-main-base test-jar test + + + javax.inject + javax.inject + + + com.teradata + re2j-td + + + org.apache.lucene + lucene-analyzers-common + + @@ -616,6 +468,12 @@ com.facebook.airlift http-server test + + + javax.inject + javax.inject + + @@ -649,6 +507,12 @@ com.facebook.airlift http-client + + + javax.inject + javax.inject + + @@ -697,25 +561,27 @@ + org.apache-extras.beanshell:bsh:: org.apache.commons:commons-lang3:: org.antlr:antlr4-runtime:: + com.clearspring.analytics:stream:: + org.glassfish.hk2.external:jakarta.inject:: + com.tdunning:t-digest:: + org.apache.datasketches:datasketches-java:: org.reflections:reflections:: org.apache.thrift:libthrift:: io.perfmark:perfmark-api:: org.apache.calcite:calcite-babel:: org.apache.calcite:calcite-core:: - org.apache.pinot:pinot-spi-jdk8:: - org.apache.pinot:pinot-common-jdk8:: - org.apache.pinot:pinot-core-jdk8:: + org.apache.pinot:pinot-spi:: + org.apache.pinot:pinot-common:: + org.apache.pinot:pinot-core:: com.google.protobuf:protobuf-java:: io.grpc::: commons-codec:commons-codec:: - - javax.inject:javax.inject - diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java index 68a55af824233..310a5ec4c1967 100644 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotPageSourceProvider.java @@ -28,7 +28,6 @@ import com.google.common.annotations.VisibleForTesting; import jakarta.inject.Inject; import org.apache.pinot.common.config.GrpcConfig; -import org.apache.pinot.connector.presto.grpc.PinotStreamingQueryClient; import java.util.ArrayList; import java.util.HashMap; diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java index c14f9a825aaef..863a7e92fe62d 100644 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotSegmentPageSource.java @@ -25,11 +25,10 @@ import com.facebook.presto.spi.PrestoException; import io.airlift.slice.Slice; import io.airlift.slice.Slices; +import org.apache.pinot.common.datatable.DataTable; +import org.apache.pinot.common.datatable.DataTableFactory; import org.apache.pinot.common.proto.Server; import org.apache.pinot.common.utils.DataSchema; -import org.apache.pinot.common.utils.DataTable; -import org.apache.pinot.connector.presto.grpc.PinotStreamingQueryClient; -import org.apache.pinot.core.common.datatable.DataTableFactory; import org.apache.pinot.spi.data.FieldSpec; import org.apache.pinot.spi.utils.ByteArray; import org.apache.pinot.spi.utils.CommonConstants; @@ -38,11 +37,9 @@ import java.math.BigDecimal; import java.nio.Buffer; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -102,13 +99,7 @@ public PinotSegmentPageSource( public static void checkExceptions(DataTable dataTable, PinotSplit split, boolean markDataFetchExceptionsAsRetriable) { - Map metadata = dataTable.getMetadata(); - List exceptions = new ArrayList<>(); - metadata.forEach((k, v) -> { - if (k.startsWith(DataTable.EXCEPTION_METADATA_KEY)) { - exceptions.add(v); - } - }); + List exceptions = dataTable.getExceptions().values().stream().toList(); if (!exceptions.isEmpty()) { throw new PinotException( markDataFetchExceptionsAsRetriable ? PINOT_DATA_FETCH_EXCEPTION : PINOT_EXCEPTION, diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java new file mode 100644 index 0000000000000..d74eb888b5f69 --- /dev/null +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.pinot; + +import org.apache.pinot.common.config.GrpcConfig; +import org.apache.pinot.common.proto.Server; +import org.apache.pinot.common.utils.grpc.GrpcQueryClient; +import org.apache.pinot.common.utils.grpc.GrpcRequestBuilder; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Grpc based Pinot query client. + * This is part of the presto-pinot driver from the official pinot repo. + * Support has been dropped in recent versions, so we've moved it here. + * Original reference + */ +public class PinotStreamingQueryClient +{ + private final Map grpcQueryClientMap = new HashMap<>(); + private final GrpcConfig config; + + public PinotStreamingQueryClient(GrpcConfig config) + { + this.config = config; + } + + public Iterator submit(String host, int port, GrpcRequestBuilder requestBuilder) + { + GrpcQueryClient client = getOrCreateGrpcQueryClient(host, port); + return client.submit(requestBuilder.build()); + } + + private GrpcQueryClient getOrCreateGrpcQueryClient(String host, int port) + { + String key = String.format("%s_%d", host, port); + if (!grpcQueryClientMap.containsKey(key)) { + grpcQueryClientMap.put(key, new GrpcQueryClient(host, port, config)); + } + return grpcQueryClientMap.get(key); + } +} diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java index 111c65421b158..a6b98514a191d 100644 --- a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java @@ -27,15 +27,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.protobuf.ByteString; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.apache.pinot.common.config.GrpcConfig; +import org.apache.pinot.common.datatable.DataTable; import org.apache.pinot.common.proto.Server; import org.apache.pinot.common.utils.DataSchema; -import org.apache.pinot.common.utils.DataTable; import org.apache.pinot.common.utils.grpc.GrpcRequestBuilder; -import org.apache.pinot.connector.presto.grpc.PinotStreamingQueryClient; -import org.apache.pinot.connector.presto.grpc.Utils; import org.apache.pinot.core.common.datatable.DataTableBuilder; import org.apache.pinot.core.common.datatable.DataTableBuilderV4; import org.apache.pinot.spi.data.DimensionFieldSpec; @@ -57,6 +56,10 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; +import static org.apache.pinot.common.utils.DataSchema.ColumnDataType.BYTES; +import static org.apache.pinot.common.utils.DataSchema.ColumnDataType.MAP; +import static org.apache.pinot.common.utils.DataSchema.ColumnDataType.OBJECT; +import static org.apache.pinot.common.utils.DataSchema.ColumnDataType.UNKNOWN; @Test(singleThreaded = true) public class TestPinotSegmentPageSource @@ -66,7 +69,7 @@ public class TestPinotSegmentPageSource protected static final int NUM_ROWS = 100; private static final Set UNSUPPORTED_TYPES = ImmutableSet.of( - DataSchema.ColumnDataType.OBJECT, DataSchema.ColumnDataType.BYTES); + OBJECT, BYTES, MAP, UNKNOWN); protected static final List ALL_TYPES = Arrays.stream(DataSchema.ColumnDataType.values()) .filter(x -> !UNSUPPORTED_TYPES.contains(x)).collect(toImmutableList()); private static final DataSchema.ColumnDataType[] ALL_TYPES_ARRAY = ALL_TYPES.toArray(new DataSchema.ColumnDataType[0]); @@ -470,7 +473,7 @@ public Server.ServerResponse next() if (index < dataTables.size()) { final DataTable dataTable = dataTables.get(index++); try { - return Server.ServerResponse.newBuilder().setPayload(Utils.toByteString(dataTable.toBytes())).putMetadata("responseType", "data").build(); + return Server.ServerResponse.newBuilder().setPayload(toByteString(dataTable.toBytes())).putMetadata("responseType", "data").build(); } catch (IOException e) { throw new RuntimeException(); @@ -483,4 +486,9 @@ public Server.ServerResponse next() }; } } + + private static ByteString toByteString(byte[] bytes) + { + return ByteString.copyFrom(bytes); + } } diff --git a/presto-pinot/pom.xml b/presto-pinot/pom.xml index 92376d1ba60ab..2aaa0aaed1337 100644 --- a/presto-pinot/pom.xml +++ b/presto-pinot/pom.xml @@ -16,6 +16,9 @@ ${project.parent.basedir} 17 true + 1.20.0 + 6.1.1 + 3.0.2 @@ -25,6 +28,11 @@ javax.annotation-api 1.3.2 + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.2 + @@ -33,6 +41,10 @@ com.facebook.presto presto-pinot-toolkit + + com.google.inject + guice + com.google.protobuf protobuf-java @@ -121,6 +133,20 @@ presto-main-base ${project.version} test + + + javax.inject + javax.inject + + + com.teradata + re2j-td + + + org.apache.lucene + lucene-analyzers-common + + @@ -129,6 +155,20 @@ ${project.version} test-jar test + + + javax.inject + javax.inject + + + com.teradata + re2j-td + + + org.apache.lucene + lucene-analyzers-common + +