diff --git a/query-service-api/build.gradle.kts b/query-service-api/build.gradle.kts index be6fd460..ffa987de 100644 --- a/query-service-api/build.gradle.kts +++ b/query-service-api/build.gradle.kts @@ -23,7 +23,7 @@ protobuf { // the identifier, which can be referred to in the "plugins" // container of the "generateProtoTasks" closure. id("grpc_java") { - artifact = "io.grpc:protoc-gen-grpc-java:1.42.0" + artifact = "io.grpc:protoc-gen-grpc-java:1.43.1" } if (generateLocalGoGrpcFiles) { @@ -66,8 +66,9 @@ tasks.test { } dependencies { - api("io.grpc:grpc-protobuf:1.42.0") - api("io.grpc:grpc-stub:1.42.0") + api(platform("io.grpc:grpc-bom:1.43.1")) + api("io.grpc:grpc-protobuf") + api("io.grpc:grpc-stub") api("javax.annotation:javax.annotation-api:1.3.2") testImplementation("org.junit.jupiter:junit-jupiter:5.7.1") diff --git a/query-service-client/build.gradle.kts b/query-service-client/build.gradle.kts index b7d28c73..b7529cd9 100644 --- a/query-service-client/build.gradle.kts +++ b/query-service-client/build.gradle.kts @@ -7,10 +7,10 @@ plugins { dependencies { api(project(":query-service-api")) - implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.6.2") + implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.7.0") // Logging - implementation("org.slf4j:slf4j-api:1.7.30") + implementation("org.slf4j:slf4j-api:1.7.32") // Config implementation("com.typesafe:config:1.4.1") } diff --git a/query-service-impl/build.gradle.kts b/query-service-impl/build.gradle.kts index 0d0c9769..c1a0f1b3 100644 --- a/query-service-impl/build.gradle.kts +++ b/query-service-impl/build.gradle.kts @@ -10,9 +10,6 @@ tasks.test { dependencies { constraints { - implementation("com.fasterxml.jackson.core:jackson-databind:2.12.2") { - because("Multiple vulnerabilities") - } implementation("io.netty:netty:3.10.6.Final") { because("https://snyk.io/vuln/SNYK-JAVA-IONETTY-30430") } @@ -28,21 +25,21 @@ dependencies { } api(project(":query-service-api")) api("com.typesafe:config:1.4.1") - implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.6.2") - implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.6.2") - implementation("org.hypertrace.core.grpcutils:grpc-server-rx-utils:0.6.2") + implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.7.0") + implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.7.0") + implementation("org.hypertrace.core.grpcutils:grpc-server-rx-utils:0.7.0") implementation("org.hypertrace.core.attribute.service:attribute-service-api:0.12.3") implementation("org.hypertrace.core.attribute.service:attribute-projection-registry:0.12.3") implementation("org.hypertrace.core.attribute.service:caching-attribute-service-client:0.12.3") - implementation("org.hypertrace.core.serviceframework:service-framework-spi:0.1.28") + implementation("org.hypertrace.core.serviceframework:service-framework-spi:0.1.33") implementation("com.google.inject:guice:5.0.1") implementation("org.apache.pinot:pinot-java-client:0.6.0") { // We want to use log4j2 impl so exclude the log4j binding of slf4j exclude("org.slf4j", "slf4j-log4j12") } - implementation("org.slf4j:slf4j-api:1.7.30") + implementation("org.slf4j:slf4j-api:1.7.32") implementation("commons-codec:commons-codec:1.15") - implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.28") + implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.33") implementation("com.google.protobuf:protobuf-java-util:3.15.6") implementation("com.google.guava:guava:30.1.1-jre") implementation("io.reactivex.rxjava3:rxjava:3.0.11") diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/AbstractQueryTransformation.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/AbstractQueryTransformation.java new file mode 100644 index 00000000..fe955ccd --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/AbstractQueryTransformation.java @@ -0,0 +1,174 @@ +package org.hypertrace.core.query.service; + +import static io.reactivex.rxjava3.core.Single.zip; + +import io.reactivex.rxjava3.core.Observable; +import io.reactivex.rxjava3.core.Single; +import java.util.List; +import org.hypertrace.core.query.service.api.AttributeExpression; +import org.hypertrace.core.query.service.api.ColumnIdentifier; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Function; +import org.hypertrace.core.query.service.api.LiteralConstant; +import org.hypertrace.core.query.service.api.OrderByExpression; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.slf4j.Logger; + +public abstract class AbstractQueryTransformation implements QueryTransformation { + + protected abstract Logger getLogger(); + + @Override + public Single transform( + QueryRequest queryRequest, QueryTransformationContext transformationContext) { + return zip( + this.transformExpressionList(queryRequest.getSelectionList()), + this.transformExpressionList(queryRequest.getAggregationList()), + this.transformFilter(queryRequest.getFilter()), + this.transformExpressionList(queryRequest.getGroupByList()), + this.transformOrderByList(queryRequest.getOrderByList()), + (selections, aggregations, filter, groupBys, orderBys) -> + this.rebuildRequest( + queryRequest, selections, aggregations, filter, groupBys, orderBys)) + .doOnSuccess(transformed -> this.debugLogIfRequestTransformed(queryRequest, transformed)); + } + + protected QueryRequest rebuildRequest( + QueryRequest original, + List selections, + List aggregations, + Filter filter, + List groupBys, + List orderBys) { + + QueryRequest.Builder builder = original.toBuilder(); + + if (Filter.getDefaultInstance().equals(filter)) { + builder.clearFilter(); + } else { + builder.setFilter(filter); + } + + return builder + .clearSelection() + .addAllSelection(selections) + .clearAggregation() + .addAllAggregation(aggregations) + .clearGroupBy() + .addAllGroupBy(groupBys) + .clearOrderBy() + .addAllOrderBy(orderBys) + .build(); + } + + protected Single transformExpression(Expression expression) { + switch (expression.getValueCase()) { + case COLUMNIDENTIFIER: + return this.transformColumnIdentifier(expression.getColumnIdentifier()); + case ATTRIBUTE_EXPRESSION: + return this.transformAttributeExpression(expression.getAttributeExpression()); + case FUNCTION: + return this.transformFunction(expression.getFunction()); + case ORDERBY: + return this.transformOrderBy(expression.getOrderBy()) + .map(expression.toBuilder()::setOrderBy) + .map(Expression.Builder::build); + case LITERAL: + return this.transformLiteral(expression.getLiteral()); + case VALUE_NOT_SET: + default: + return Single.just(expression); + } + } + + protected Single transformColumnIdentifier(ColumnIdentifier columnIdentifier) { + return Single.just(Expression.newBuilder().setColumnIdentifier(columnIdentifier).build()); + } + + protected Single transformAttributeExpression( + AttributeExpression attributeExpression) { + return Single.just(Expression.newBuilder().setAttributeExpression(attributeExpression).build()); + } + + protected Single transformLiteral(LiteralConstant literalConstant) { + return Single.just(Expression.newBuilder().setLiteral(literalConstant).build()); + } + + protected Single transformFunction(Function function) { + return this.transformExpressionList(function.getArgumentsList()) + .map(expressions -> function.toBuilder().clearArguments().addAllArguments(expressions)) + .map(Function.Builder::build) + .map(Expression.newBuilder()::setFunction) + .map(Expression.Builder::build); + } + + protected Single transformOrderBy(OrderByExpression orderBy) { + return this.transformExpression(orderBy.getExpression()) + .map(orderBy.toBuilder()::setExpression) + .map(OrderByExpression.Builder::build); + } + + protected Single transformFilter(Filter filter) { + if (filter.equals(Filter.getDefaultInstance())) { + return Single.just(filter); + } + + Single lhsSingle = this.transformExpression(filter.getLhs()); + Single rhsSingle = this.transformExpression(filter.getRhs()); + Single> childFilterListSingle = + Observable.fromIterable(filter.getChildFilterList()) + .concatMapSingle(this::transformFilter) + .toList(); + return zip( + lhsSingle, + rhsSingle, + childFilterListSingle, + (lhs, rhs, childFilterList) -> + this.rebuildFilterOmittingDefaults(filter, lhs, rhs, childFilterList)); + } + + private Single> transformExpressionList(List expressionList) { + return Observable.fromIterable(expressionList) + .concatMapSingle(this::transformExpression) + .toList(); + } + + private Single> transformOrderByList( + List orderByList) { + return Observable.fromIterable(orderByList).concatMapSingle(this::transformOrderBy).toList(); + } + + /** + * This doesn't change any functional behavior, but omits fields that aren't needed, shrinking the + * object and keeping it equivalent to the source object for equality checks. + */ + private Filter rebuildFilterOmittingDefaults( + Filter original, Expression lhs, Expression rhs, List childFilters) { + Filter.Builder builder = original.toBuilder(); + + if (Expression.getDefaultInstance().equals(lhs)) { + builder.clearLhs(); + } else { + builder.setLhs(lhs); + } + + if (Expression.getDefaultInstance().equals(rhs)) { + builder.clearRhs(); + } else { + builder.setRhs(rhs); + } + + return builder.clearChildFilter().addAllChildFilter(childFilters).build(); + } + + private void debugLogIfRequestTransformed(QueryRequest original, QueryRequest transformed) { + if (!original.equals(transformed)) { + getLogger() + .debug( + "Request transformation occurred. Original request: {} Transformed Request: {}", + original, + transformed); + } + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java index f63db9ee..9bd89207 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java @@ -1,12 +1,9 @@ package org.hypertrace.core.query.service; import static org.hypertrace.core.query.service.api.Expression.ValueCase.ATTRIBUTE_EXPRESSION; -import static org.hypertrace.core.query.service.api.Expression.ValueCase.COLUMNIDENTIFIER; -import static org.hypertrace.core.query.service.api.Expression.ValueCase.FUNCTION; import java.util.Optional; import org.hypertrace.core.query.service.api.AttributeExpression; -import org.hypertrace.core.query.service.api.ColumnIdentifier; import org.hypertrace.core.query.service.api.Expression; import org.hypertrace.core.query.service.api.Expression.ValueCase; import org.hypertrace.core.query.service.api.Filter; @@ -21,12 +18,6 @@ */ public class QueryRequestUtil { - public static Expression createColumnExpression(String columnName) { - return Expression.newBuilder() - .setColumnIdentifier(ColumnIdentifier.newBuilder().setColumnName(columnName)) - .build(); - } - public static Expression createStringLiteralExpression(String value) { return Expression.newBuilder() .setLiteral( @@ -145,10 +136,7 @@ public static Optional getAlias(Expression expression) { ? getLogicalColumnName(expression).get() : expression.getColumnIdentifier().getAlias()); case ATTRIBUTE_EXPRESSION: - return Optional.of( - expression.getAttributeExpression().getAlias().isBlank() - ? getLogicalColumnName(expression).get() - : expression.getAttributeExpression().getAlias()); + return Optional.of(getAlias(expression.getAttributeExpression())); case FUNCTION: // todo: handle recursive functions max(rollup(time,50) // workaround is to use alias for now @@ -160,4 +148,10 @@ public static Optional getAlias(Expression expression) { return Optional.empty(); } } + + public static String getAlias(AttributeExpression attributeExpression) { + return attributeExpression.getAlias().isBlank() + ? attributeExpression.getAttributeId() + : attributeExpression.getAlias(); + } } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java index 09c5d84e..9fb4c010 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java @@ -6,6 +6,7 @@ import javax.inject.Singleton; import org.hypertrace.core.attribute.service.cachingclient.CachingAttributeClient; import org.hypertrace.core.query.service.api.QueryServiceGrpc.QueryServiceImplBase; +import org.hypertrace.core.query.service.attribubteexpression.AttributeExpressionModule; import org.hypertrace.core.query.service.pinot.PinotModule; import org.hypertrace.core.query.service.projection.ProjectionModule; import org.hypertrace.core.query.service.prometheus.PrometheusModule; @@ -33,5 +34,6 @@ protected void configure() { install(new PinotModule()); install(new ProjectionModule()); install(new PrometheusModule()); + install(new AttributeExpressionModule()); } } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformation.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformation.java index 09428e0f..0c3d3049 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformation.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformation.java @@ -2,10 +2,20 @@ import io.reactivex.rxjava3.core.Single; import org.hypertrace.core.query.service.api.QueryRequest; +import org.jetbrains.annotations.NotNull; -public interface QueryTransformation { +public interface QueryTransformation extends Comparable { Single transform( QueryRequest queryRequest, QueryTransformationContext transformationContext); + default int getPriority() { + return 10; + } + + @Override + default int compareTo(@NotNull QueryTransformation other) { + return Integer.compare(this.getPriority(), other.getPriority()); + } + interface QueryTransformationContext {} } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformationPipeline.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformationPipeline.java index c497ae10..797be9b4 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformationPipeline.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTransformationPipeline.java @@ -24,6 +24,7 @@ Single transform(QueryRequest originalRequest, String tenantId) { QueryTransformationContext transformationContext = new DefaultQueryTransformationContext(tenantId); return Observable.fromIterable(transformations) + .sorted() .reduce( Single.just(originalRequest), (requestSingle, transformation) -> diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionModule.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionModule.java new file mode 100644 index 00000000..6cc3e929 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionModule.java @@ -0,0 +1,18 @@ +package org.hypertrace.core.query.service.attribubteexpression; + +import com.google.inject.AbstractModule; +import com.google.inject.multibindings.Multibinder; +import org.hypertrace.core.query.service.QueryTransformation; + +public class AttributeExpressionModule extends AbstractModule { + + @Override + protected void configure() { + Multibinder transformationMultibinder = + Multibinder.newSetBinder(binder(), QueryTransformation.class); + transformationMultibinder + .addBinding() + .to(AttributeExpressionSubpathExistsFilteringTransformation.class); + transformationMultibinder.addBinding().to(AttributeExpressionNormalizationTransformation.class); + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformation.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformation.java new file mode 100644 index 00000000..10cabf32 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformation.java @@ -0,0 +1,41 @@ +package org.hypertrace.core.query.service.attribubteexpression; + +import io.reactivex.rxjava3.core.Single; +import lombok.extern.slf4j.Slf4j; +import org.hypertrace.core.query.service.AbstractQueryTransformation; +import org.hypertrace.core.query.service.QueryRequestUtil; +import org.hypertrace.core.query.service.api.ColumnIdentifier; +import org.hypertrace.core.query.service.api.Expression; +import org.slf4j.Logger; + +@Slf4j +final class AttributeExpressionNormalizationTransformation extends AbstractQueryTransformation { + + @Override + public int getPriority() { + // Run before default transformations + return 1; + } + + @Override + protected Logger getLogger() { + return log; + } + + @Override + protected Single transformColumnIdentifier(ColumnIdentifier columnIdentifier) { + Expression.Builder expressionBuilder = + QueryRequestUtil.createSimpleAttributeExpression(columnIdentifier.getColumnName()); + if (columnIdentifier.getAlias().isBlank()) { + return Single.just(expressionBuilder.build()); + } + + return Single.just( + expressionBuilder + .setAttributeExpression( + expressionBuilder + .getAttributeExpressionBuilder() + .setAlias(columnIdentifier.getAlias())) + .build()); + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformation.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformation.java new file mode 100644 index 00000000..371ccf70 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformation.java @@ -0,0 +1,134 @@ +package org.hypertrace.core.query.service.attribubteexpression; + +import static org.hypertrace.core.query.service.QueryRequestUtil.createContainsKeyFilter; +import static org.hypertrace.core.query.service.QueryRequestUtil.isAttributeExpressionWithSubpath; + +import com.google.common.collect.ImmutableSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.hypertrace.core.query.service.AbstractQueryTransformation; +import org.hypertrace.core.query.service.QueryRequestUtil; +import org.hypertrace.core.query.service.api.AttributeExpression; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.OrderByExpression; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.slf4j.Logger; + +@Slf4j +final class AttributeExpressionSubpathExistsFilteringTransformation + extends AbstractQueryTransformation { + + @Override + protected Logger getLogger() { + return log; + } + + @Override + protected QueryRequest rebuildRequest( + QueryRequest original, + List selections, + List aggregations, + Filter originalFilter, + List groupBys, + List orderBys) { + Filter updatedFilter = + rebuildFilterForComplexAttributeExpression(originalFilter, orderBys, selections) + .orElse(originalFilter); + + return super.rebuildRequest( + original, selections, aggregations, updatedFilter, groupBys, orderBys); + } + + /* + * We need the CONTAINS_KEY filter in all filters, selections, and order bys dealing with complex + * attribute expressions as Pinot gives an error if a particular key is absent. Rest all work fine. + * To handle order bys and selections, we add the corresponding filter at the top and 'AND' it with the main filter. + * To handle filter, we modify each filter (say filter1) as: "CONTAINS_KEY AND filter1". + */ + private Optional rebuildFilterForComplexAttributeExpression( + Filter originalFilter, List orderBys, List selections) { + + Set rootFilters = + ImmutableSet.builder() + .add(updateFilterForComplexAttributeExpressionFromFilter(originalFilter)) + .addAll(createFilterForComplexAttributeExpressionFromOrderBy(orderBys)) + .addAll(createFilterForComplexAttributeExpressionFromSelection(selections)) + .build() + .stream() + .filter(Predicate.not(Filter.getDefaultInstance()::equals)) + .collect(ImmutableSet.toImmutableSet()); + + if (rootFilters.isEmpty()) { + return Optional.empty(); + } + + if (rootFilters.size() == 1) { + return rootFilters.stream().findFirst(); + } + return Optional.of( + Filter.newBuilder().setOperator(Operator.AND).addAllChildFilter(rootFilters).build()); + } + + private Filter updateFilterForComplexAttributeExpressionFromFilter(Filter originalFilter) { + /* + * If childFilter is present, then the expected operators comprise the logical operators. + * If childFilter is absent, then the filter is a leaf filter which will have lhs and rhs. + */ + if (originalFilter.getChildFilterCount() > 0) { + Filter.Builder builder = Filter.newBuilder(); + builder.setOperator(originalFilter.getOperator()); + originalFilter + .getChildFilterList() + .forEach( + childFilter -> + builder.addChildFilter( + updateFilterForComplexAttributeExpressionFromFilter(childFilter))); + return builder.build(); + } + if (isAttributeExpressionWithSubpath(originalFilter.getLhs())) { + Filter childFilter = + createContainsKeyFilter(originalFilter.getLhs().getAttributeExpression()); + return Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(originalFilter) + .addChildFilter(childFilter) + .build(); + } + return originalFilter; + } + + private List createFilterForComplexAttributeExpressionFromOrderBy( + List orderByExpressionList) { + return orderByExpressionList.stream() + .map(OrderByExpression::getExpression) + .filter(QueryRequestUtil::isAttributeExpressionWithSubpath) + .map(Expression::getAttributeExpression) + .map(QueryRequestUtil::createContainsKeyFilter) + .collect(Collectors.toList()); + } + + private List createFilterForComplexAttributeExpressionFromSelection( + List expressions) { + return expressions.stream() + .flatMap(this::getAnyAttributeExpression) + .map(QueryRequestUtil::createContainsKeyFilter) + .collect(Collectors.toList()); + } + + private Stream getAnyAttributeExpression(Expression expression) { + if (expression.hasFunction()) { + return expression.getFunction().getArgumentsList().stream() + .flatMap(this::getAnyAttributeExpression); + } + return Stream.of(expression) + .filter(QueryRequestUtil::isAttributeExpressionWithSubpath) + .map(Expression::getAttributeExpression); + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandler.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandler.java index 12d37ed4..0d1084ad 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandler.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandler.java @@ -279,7 +279,7 @@ private boolean doesSingleViewFilterMatchLeafQueryFilter( } ViewColumnFilter viewColumnFilter = - viewFilterMap.get(getLogicalColumnName(queryFilter.getLhs()).orElse(null)); + getLogicalColumnName(queryFilter.getLhs()).map(viewFilterMap::get).orElse(null); if (viewColumnFilter == null) { return false; } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/projection/ProjectionTransformation.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/projection/ProjectionTransformation.java index f1e9b819..1536dfe8 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/projection/ProjectionTransformation.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/projection/ProjectionTransformation.java @@ -2,23 +2,20 @@ import static io.reactivex.rxjava3.core.Single.zip; import static org.hypertrace.core.query.service.QueryRequestUtil.createBooleanLiteralExpression; -import static org.hypertrace.core.query.service.QueryRequestUtil.createColumnExpression; -import static org.hypertrace.core.query.service.QueryRequestUtil.createContainsKeyFilter; import static org.hypertrace.core.query.service.QueryRequestUtil.createDoubleLiteralExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createLongLiteralExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createNullNumberLiteralExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createNullStringLiteralExpression; +import static org.hypertrace.core.query.service.QueryRequestUtil.createSimpleAttributeExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createStringLiteralExpression; -import static org.hypertrace.core.query.service.QueryRequestUtil.isAttributeExpressionWithSubpath; import io.reactivex.rxjava3.core.Maybe; import io.reactivex.rxjava3.core.Observable; import io.reactivex.rxjava3.core.Single; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; import org.hypertrace.core.attribute.service.cachingclient.CachingAttributeClient; import org.hypertrace.core.attribute.service.projection.AttributeProjection; import org.hypertrace.core.attribute.service.projection.AttributeProjectionRegistry; @@ -28,23 +25,16 @@ import org.hypertrace.core.attribute.service.v1.Projection; import org.hypertrace.core.attribute.service.v1.ProjectionExpression; import org.hypertrace.core.attribute.service.v1.ProjectionOperator; +import org.hypertrace.core.query.service.AbstractQueryTransformation; import org.hypertrace.core.query.service.QueryFunctionConstants; import org.hypertrace.core.query.service.QueryRequestUtil; -import org.hypertrace.core.query.service.QueryTransformation; import org.hypertrace.core.query.service.api.AttributeExpression; -import org.hypertrace.core.query.service.api.ColumnIdentifier; import org.hypertrace.core.query.service.api.Expression; -import org.hypertrace.core.query.service.api.Filter; import org.hypertrace.core.query.service.api.Function; -import org.hypertrace.core.query.service.api.Operator; -import org.hypertrace.core.query.service.api.OrderByExpression; -import org.hypertrace.core.query.service.api.QueryRequest; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -final class ProjectionTransformation implements QueryTransformation { - - private static final Logger LOG = LoggerFactory.getLogger(ProjectionTransformation.class); +@Slf4j +final class ProjectionTransformation extends AbstractQueryTransformation { private final CachingAttributeClient attributeClient; private final AttributeProjectionRegistry projectionRegistry; @@ -57,96 +47,33 @@ final class ProjectionTransformation implements QueryTransformation { } @Override - public Single transform( - QueryRequest queryRequest, QueryTransformationContext transformationContext) { - return zip( - this.transformExpressionList(queryRequest.getSelectionList()), - this.transformExpressionList(queryRequest.getAggregationList()), - this.transformFilter(queryRequest.getFilter()), - this.transformExpressionList(queryRequest.getGroupByList()), - this.transformOrderByList(queryRequest.getOrderByList()), - (selections, aggregations, filter, groupBys, orderBys) -> - this.rebuildRequest( - queryRequest, selections, aggregations, filter, groupBys, orderBys)) - .doOnSuccess(transformed -> this.debugLogIfRequestTransformed(queryRequest, transformed)); + protected Logger getLogger() { + return log; } - private Single> transformExpressionList(List expressionList) { - return Observable.fromIterable(expressionList) - .concatMapSingle(this::transformExpression) - .toList(); - } - - private Single transformExpression(Expression expression) { - switch (expression.getValueCase()) { - case COLUMNIDENTIFIER: - return this.transformColumnIdentifier(expression.getColumnIdentifier()); - case ATTRIBUTE_EXPRESSION: - return this.transformAttributeExpression(expression.getAttributeExpression()); - case FUNCTION: - return this.transformFunction(expression.getFunction()) - .map(expression.toBuilder()::setFunction) - .map(Expression.Builder::build); - case ORDERBY: - return this.transformOrderBy(expression.getOrderBy()) - .map(expression.toBuilder()::setOrderBy) - .map(Expression.Builder::build); - case LITERAL: - case VALUE_NOT_SET: - default: - return Single.just(expression); - } - } - - private Single transformColumnIdentifier(ColumnIdentifier columnIdentifier) { - return this.projectAttributeIfPossible(columnIdentifier.getColumnName()) - .map(expression -> this.aliasToMatchOriginal(getOriginalKey(columnIdentifier), expression)) - .defaultIfEmpty(Expression.newBuilder().setColumnIdentifier(columnIdentifier).build()); - } - - private Single transformAttributeExpression(AttributeExpression attributeExpression) { + @Override + protected Single transformAttributeExpression( + AttributeExpression attributeExpression) { return this.projectAttributeIfPossible(attributeExpression.getAttributeId()) .map( - expression -> - this.aliasToMatchOriginal(getOriginalKey(attributeExpression), expression)) + projectedExpression -> + attributeExpression.hasSubpath() + ? this.addSubpathOrThrow(projectedExpression, attributeExpression.getSubpath()) + : projectedExpression) + .map(expression -> this.aliasToMatchOriginal(attributeExpression, expression)) .defaultIfEmpty( Expression.newBuilder().setAttributeExpression(attributeExpression).build()); } - private Single transformFunction(Function function) { - return this.transformExpressionList(function.getArgumentsList()) - .map(expressions -> function.toBuilder().clearArguments().addAllArguments(expressions)) - .map(Function.Builder::build); - } - - private Single> transformOrderByList( - List orderByList) { - return Observable.fromIterable(orderByList).concatMapSingle(this::transformOrderBy).toList(); - } - - private Single transformOrderBy(OrderByExpression orderBy) { - return this.transformExpression(orderBy.getExpression()) - .map(orderBy.toBuilder()::setExpression) - .map(OrderByExpression.Builder::build); - } - - private Single transformFilter(Filter filter) { - if (filter.equals(Filter.getDefaultInstance())) { - return Single.just(filter); + private Expression addSubpathOrThrow(Expression projectedExpression, String subpath) { + if (!QueryRequestUtil.isSimpleAttributeExpression(projectedExpression)) { + throw new IllegalArgumentException( + "Cannot use subpath for expression with non-trivial projection: " + projectedExpression); } - - Single lhsSingle = this.transformExpression(filter.getLhs()); - Single rhsSingle = this.transformExpression(filter.getRhs()); - Single> childFilterListSingle = - Observable.fromIterable(filter.getChildFilterList()) - .concatMapSingle(this::transformFilter) - .toList(); - return zip( - lhsSingle, - rhsSingle, - childFilterListSingle, - (lhs, rhs, childFilterList) -> - this.rebuildFilterOmittingDefaults(filter, lhs, rhs, childFilterList)); + return Expression.newBuilder() + .setAttributeExpression( + projectedExpression.getAttributeExpression().toBuilder().setSubpath(subpath)) + .build(); } private Maybe projectAttributeIfPossible(String attributeId) { @@ -172,7 +99,8 @@ private Single rewriteProjectionAsQueryExpression( Projection projection, AttributeKind expectedType) { switch (projection.getValueCase()) { case ATTRIBUTE_ID: - return this.transformExpression(createColumnExpression(projection.getAttributeId())); + return this.transformExpression( + createSimpleAttributeExpression(projection.getAttributeId()).build()); case LITERAL: return this.rewriteLiteralAsQueryExpression(projection.getLiteral(), expectedType); case EXPRESSION: @@ -280,13 +208,9 @@ private Single convertOperator(ProjectionOperator operator) { } } - private Expression aliasToMatchOriginal(String originalKey, Expression newExpression) { + private Expression aliasToMatchOriginal(AttributeExpression original, Expression newExpression) { + String originalKey = QueryRequestUtil.getAlias(original); switch (newExpression.getValueCase()) { - case COLUMNIDENTIFIER: - return newExpression.toBuilder() - .setColumnIdentifier( - newExpression.getColumnIdentifier().toBuilder().setAlias(originalKey)) - .build(); case ATTRIBUTE_EXPRESSION: return newExpression.toBuilder() .setAttributeExpression( @@ -303,166 +227,4 @@ private Expression aliasToMatchOriginal(String originalKey, Expression newExpres return newExpression; } } - - private void debugLogIfRequestTransformed(QueryRequest original, QueryRequest transformed) { - if (!original.equals(transformed)) { - LOG.debug( - "Request transformation occurred. Original request: {} Transformed Request: {}", - original, - transformed); - } - } - - /** - * This doesn't change any functional behavior, but omits fields that aren't needed, shrinking the - * object and keeping it equivalent to the source object for equality checks. - */ - private Filter rebuildFilterOmittingDefaults( - Filter original, Expression lhs, Expression rhs, List childFilters) { - Filter.Builder builder = original.toBuilder(); - - if (Expression.getDefaultInstance().equals(lhs)) { - builder.clearLhs(); - } else { - builder.setLhs(lhs); - } - - if (Expression.getDefaultInstance().equals(rhs)) { - builder.clearRhs(); - } else { - builder.setRhs(rhs); - } - - return builder.clearChildFilter().addAllChildFilter(childFilters).build(); - } - - private QueryRequest rebuildRequest( - QueryRequest original, - List selections, - List aggregations, - Filter originalFilter, - List groupBys, - List orderBys) { - - QueryRequest.Builder builder = original.toBuilder(); - Filter updatedFilter = - rebuildFilterForComplexAttributeExpression(originalFilter, orderBys, selections); - - if (Filter.getDefaultInstance().equals(updatedFilter)) { - builder.clearFilter(); - } else { - builder.setFilter(updatedFilter); - } - - return builder - .clearSelection() - .addAllSelection(selections) - .clearAggregation() - .addAllAggregation(aggregations) - .clearGroupBy() - .addAllGroupBy(groupBys) - .clearOrderBy() - .addAllOrderBy(orderBys) - .build(); - } - - /* - * We need the CONTAINS_KEY filter in all filters, selections and order bys dealing with complex - * attribute expressions as Pinot gives error if particular key is absent. Rest all work fine. - * To handle order bys and selections, we add the corresponding filter at the top and 'AND' it with the main filter. - * To handle filter, we modify each filter (say filter1) as : "CONTAINS_KEY AND filter1". - */ - private Filter rebuildFilterForComplexAttributeExpression( - Filter originalFilter, List orderBys, List selections) { - - Filter updatedFilter = updateFilterForComplexAttributeExpressionFromFilter(originalFilter); - List filterList = - Stream.concat( - createFilterForComplexAttributeExpressionFromOrderBy(orderBys), - createFilterForComplexAttributeExpressionFromSelection(selections)) - .collect(Collectors.toList()); - - if (filterList.isEmpty()) { - return updatedFilter; - } - - if (!updatedFilter.equals(Filter.getDefaultInstance())) { - return Filter.newBuilder() - .setOperator(Operator.AND) - .addChildFilter(updatedFilter) - .addAllChildFilter(filterList) - .build(); - } - - if (filterList.size() > 1) { - return Filter.newBuilder().setOperator(Operator.AND).addAllChildFilter(filterList).build(); - } else { - return filterList.get(0); - } - } - - private Filter updateFilterForComplexAttributeExpressionFromFilter(Filter originalFilter) { - /* - * If childFilter is present, then the expected operators comprise the logical operators. - * If childFilter is absent, then the filter is a leaf filter which will have lhs and rhs. - */ - if (originalFilter.getChildFilterCount() > 0) { - Filter.Builder builder = Filter.newBuilder(); - builder.setOperator(originalFilter.getOperator()); - originalFilter - .getChildFilterList() - .forEach( - childFilter -> - builder.addChildFilter( - updateFilterForComplexAttributeExpressionFromFilter(childFilter))); - return builder.build(); - } else if (isAttributeExpressionWithSubpath(originalFilter.getLhs())) { - Filter childFilter = - createContainsKeyFilter(originalFilter.getLhs().getAttributeExpression()); - return Filter.newBuilder() - .setOperator(Operator.AND) - .addChildFilter(originalFilter) - .addChildFilter(childFilter) - .build(); - } else { - return originalFilter; - } - } - - private Stream createFilterForComplexAttributeExpressionFromOrderBy( - List orderByExpressionList) { - return orderByExpressionList.stream() - .map(OrderByExpression::getExpression) - .filter(QueryRequestUtil::isAttributeExpressionWithSubpath) - .map(Expression::getAttributeExpression) - .map(QueryRequestUtil::createContainsKeyFilter); - } - - private Stream createFilterForComplexAttributeExpressionFromSelection( - List selections) { - return selections.stream() - .flatMap(this::getAnyAttributeExpression) - .map(QueryRequestUtil::createContainsKeyFilter); - } - - private Stream getAnyAttributeExpression(Expression selection) { - if (selection.hasFunction()) { - return selection.getFunction().getArgumentsList().stream() - .flatMap(this::getAnyAttributeExpression); - } else { - return Stream.of(selection) - .filter(QueryRequestUtil::isAttributeExpressionWithSubpath) - .map(Expression::getAttributeExpression); - } - } - - private String getOriginalKey(AttributeExpression attributeExpression) { - String alias = attributeExpression.getAlias(); - return alias.isEmpty() ? attributeExpression.getAttributeId() : alias; - } - - private String getOriginalKey(ColumnIdentifier columnIdentifier) { - String alias = columnIdentifier.getAlias(); - return alias.isEmpty() ? columnIdentifier.getColumnName() : alias; - } } diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryRequestBuilderUtils.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryRequestBuilderUtils.java index 51b739e1..84ac05da 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryRequestBuilderUtils.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryRequestBuilderUtils.java @@ -37,6 +37,13 @@ public static Expression createAliasedColumnExpression(String columnName, String .build(); } + public static Expression createAliasedAttributeExpression(String columnName, String alias) { + return Expression.newBuilder() + .setAttributeExpression( + AttributeExpression.newBuilder().setAttributeId(columnName).setAlias(alias)) + .build(); + } + public static Expression createCountByColumnSelection(String columnNames) { return createFunctionExpression("Count", createColumnExpression(columnNames).build()); } diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryTransformationPipelineTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryTransformationPipelineTest.java index 238b630e..49448eb9 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryTransformationPipelineTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryTransformationPipelineTest.java @@ -44,10 +44,14 @@ void callsEachTransformationWithPreviousResult() { when(secondTransformation.transform(same(firstResult), any(QueryTransformationContext.class))) .thenReturn(Single.just(secondResult)); + when(firstTransformation.compareTo(any())).thenCallRealMethod(); + when(firstTransformation.getPriority()).thenReturn(1); + when(secondTransformation.getPriority()).thenReturn(10); + // purposely flip order to make sure sorted assertSame( secondResult, new QueryTransformationPipeline( - new LinkedHashSet<>(List.of(firstTransformation, secondTransformation))) + new LinkedHashSet<>(List.of(secondTransformation, firstTransformation))) .transform(this.originalRequest, TEST_TENANT_ID) .blockingGet()); } diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformationTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformationTest.java new file mode 100644 index 00000000..00d714f6 --- /dev/null +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionNormalizationTransformationTest.java @@ -0,0 +1,75 @@ +package org.hypertrace.core.query.service.attribubteexpression; + +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedAttributeExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedFunctionExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestUtil.createSimpleAttributeExpression; +import static org.hypertrace.core.query.service.QueryRequestUtil.createStringLiteralValueExpression; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.hypertrace.core.query.service.QueryFunctionConstants; +import org.hypertrace.core.query.service.QueryTransformation.QueryTransformationContext; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.OrderByExpression; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AttributeExpressionNormalizationTransformationTest { + + @Mock QueryTransformationContext mockTransformationContext; + + private final AttributeExpressionNormalizationTransformation transformation = + new AttributeExpressionNormalizationTransformation(); + + @Test + void transformColumnInAllPositions() { + QueryRequest originalRequest = + QueryRequest.newBuilder() + .addSelection(createAliasedColumnExpression("select-col", "select-alias")) + .addAggregation( + createAliasedFunctionExpression( + QueryFunctionConstants.QUERY_FUNCTION_SUM, + "agg-alias", + createColumnExpression("agg-col").build())) + .setFilter( + Filter.newBuilder() + .setLhs(createAliasedColumnExpression("filter-col", "filter-alias")) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("val"))) + .addOrderBy( + OrderByExpression.newBuilder() + .setExpression(createAliasedColumnExpression("orderby-col", "orderby-alias"))) + .addGroupBy(createAliasedColumnExpression("groupby-col", "groupby-alias")) + .build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .addSelection(createAliasedAttributeExpression("select-col", "select-alias")) + .addAggregation( + createAliasedFunctionExpression( + QueryFunctionConstants.QUERY_FUNCTION_SUM, + "agg-alias", + createSimpleAttributeExpression("agg-col").build())) + .setFilter( + Filter.newBuilder() + .setLhs(createAliasedAttributeExpression("filter-col", "filter-alias")) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("val"))) + .addOrderBy( + OrderByExpression.newBuilder() + .setExpression( + createAliasedAttributeExpression("orderby-col", "orderby-alias"))) + .addGroupBy(createAliasedAttributeExpression("groupby-col", "groupby-alias")) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } +} diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformationTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformationTest.java new file mode 100644 index 00000000..f9a43702 --- /dev/null +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/attribubteexpression/AttributeExpressionSubpathExistsFilteringTransformationTest.java @@ -0,0 +1,235 @@ +package org.hypertrace.core.query.service.attribubteexpression; + +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createComplexAttributeExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createCompositeFilter; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createEqualsFilter; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createOrderByExpression; +import static org.hypertrace.core.query.service.QueryRequestUtil.createContainsKeyFilter; +import static org.hypertrace.core.query.service.QueryRequestUtil.createStringLiteralValueExpression; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.hypertrace.core.query.service.QueryTransformation.QueryTransformationContext; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.SortOrder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AttributeExpressionSubpathExistsFilteringTransformationTest { + + @Mock QueryTransformationContext mockTransformationContext; + + private final AttributeExpressionSubpathExistsFilteringTransformation transformation = + new AttributeExpressionSubpathExistsFilteringTransformation(); + + @Test + void transQueryWithComplexAttributeExpression_SingleFilter() { + Expression spanTags = createComplexAttributeExpression("Span.tags", "span.kind").build(); + Filter filter = + Filter.newBuilder() + .setLhs(spanTags) + .setOperator(Operator.EQ) + .setRhs(createColumnExpression("server")) + .build(); + QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .setFilter( + Filter.newBuilder() + .setOperator(Operator.AND) + .addAllChildFilter( + List.of(filter, createContainsKeyFilter(spanTags.getAttributeExpression()))) + .build()) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } + + @Test + void transQueryWithComplexAttributeExpression_MultipleFilter() { + Expression spanTags1 = createComplexAttributeExpression("Span.tags", "FLAGS").build(); + Expression spanTags2 = createComplexAttributeExpression("Span.tags", "span.kind").build(); + + Filter childFilter1 = + Filter.newBuilder() + .setLhs(spanTags1) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("0")) + .build(); + Filter childFilter2 = + Filter.newBuilder() + .setLhs(spanTags2) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("server")) + .build(); + + Filter.Builder filter = createCompositeFilter(Operator.AND, childFilter1, childFilter2); + QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); + + QueryRequest expectedTransform = + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet(); + List childFilterList = expectedTransform.getFilter().getChildFilterList(); + + assertTrue( + childFilterList + .get(0) + .getChildFilterList() + .contains(createContainsKeyFilter(spanTags1.getAttributeExpression()))); + assertTrue( + childFilterList + .get(1) + .getChildFilterList() + .contains(createContainsKeyFilter(spanTags2.getAttributeExpression()))); + } + + @Test + void transQueryWithComplexAttributeExpression_HierarchicalFilter() { + Expression spanTags1 = createComplexAttributeExpression("Span.tags", "FLAGS").build(); + Expression spanTags2 = createComplexAttributeExpression("Span.tags", "span.kind").build(); + + Filter filter1 = + Filter.newBuilder() + .setLhs(spanTags1) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("0")) + .build(); + Filter filter2 = + Filter.newBuilder() + .setLhs(spanTags2) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("server")) + .build(); + Filter filter = + createCompositeFilter( + Operator.AND, + createEqualsFilter("other-attribute", "otherValue"), + createCompositeFilter(Operator.AND, filter1, filter2).build()) + .build(); + + QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); + + QueryRequest expectedTransform = + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet(); + List childFilterList = + expectedTransform.getFilter().getChildFilterList().get(1).getChildFilterList(); + + assertTrue( + childFilterList + .get(0) + .getChildFilterList() + .contains(createContainsKeyFilter(spanTags1.getAttributeExpression()))); + assertTrue( + childFilterList + .get(1) + .getChildFilterList() + .contains(createContainsKeyFilter(spanTags2.getAttributeExpression()))); + } + + @Test + void transQueryWithComplexAttributeExpression_OrderByAndFilter() { + Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); + + Filter filter = + Filter.newBuilder() + .setLhs(spanTag) + .setOperator(Operator.EQ) + .setRhs(createStringLiteralValueExpression("server")) + .build(); + Filter containsKeyFilter = createContainsKeyFilter("Span.tags", "span.kind"); + + QueryRequest originalRequest = + QueryRequest.newBuilder() + .setFilter(filter) + .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) + .build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .setFilter( + createCompositeFilter( + Operator.AND, + createCompositeFilter(Operator.AND, filter, containsKeyFilter).build(), + containsKeyFilter)) + .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } + + @Test + void transQueryWithComplexAttributeExpression_SingleSelection() { + Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); + + QueryRequest originalRequest = QueryRequest.newBuilder().addSelection(spanTag).build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .addSelection(spanTag) + .setFilter(createContainsKeyFilter("Span.tags", "span.kind")) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } + + @Test + void transQueryWithComplexAttributeExpression_SingleOrderBy() { + Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); + + QueryRequest originalRequest = + QueryRequest.newBuilder() + .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) + .build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .setFilter(createContainsKeyFilter("Span.tags", "span.kind")) + .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } + + @Test + void transQueryWithComplexAttributeExpression_MultipleOrderBy() { + Expression.Builder spanTag1 = createComplexAttributeExpression("Span.tags", "span.kind"); + Expression.Builder spanTag2 = createComplexAttributeExpression("Span.tags", "FLAGS"); + + QueryRequest originalRequest = + QueryRequest.newBuilder() + .addOrderBy(createOrderByExpression(spanTag1, SortOrder.ASC)) + .addOrderBy(createOrderByExpression(spanTag2, SortOrder.ASC)) + .build(); + + QueryRequest expectedTransform = + QueryRequest.newBuilder() + .setFilter( + createCompositeFilter( + Operator.AND, + createContainsKeyFilter("Span.tags", "span.kind"), + createContainsKeyFilter("Span.tags", "FLAGS"))) + .addOrderBy(createOrderByExpression(spanTag1, SortOrder.ASC)) + .addOrderBy(createOrderByExpression(spanTag2, SortOrder.ASC)) + .build(); + + assertEquals( + expectedTransform, + this.transformation.transform(originalRequest, mockTransformationContext).blockingGet()); + } +} diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/MigrationTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/MigrationTest.java index 4f96cddd..9b328389 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/MigrationTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/MigrationTest.java @@ -9,6 +9,7 @@ import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFunctionExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createNullStringLiteralValueExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createOrderByExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createStringArrayLiteralValueExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createTimeFilterWithSimpleAttribute; import static org.hypertrace.core.query.service.QueryRequestUtil.createContainsKeyFilter; import static org.hypertrace.core.query.service.QueryRequestUtil.createSimpleAttributeExpression; @@ -20,6 +21,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map.Entry; import org.apache.pinot.client.Connection; import org.apache.pinot.client.Request; @@ -247,6 +249,35 @@ public void testQueryWithNotContainsKeyOperator() { executionContext); } + @Test + public void testQueryWithContainsKeyValueOperator() { + Builder builder = QueryRequest.newBuilder(); + Expression spanTag = createSimpleAttributeExpression("Span.tags").build(); + builder.addSelection(spanTag); + builder.setFilter( + Filter.newBuilder() + .setOperator(Operator.CONTAINS_KEYVALUE) + .setLhs(spanTag) + .setRhs(createStringArrayLiteralValueExpression(List.of("Flags", "0"))) + .build()); + + ViewDefinition viewDefinition = getDefaultViewDefinition(); + defaultMockingForExecutionContext(); + + assertPQLQuery( + builder.build(), + "SELECT tags__keys, tags__values FROM SpanEventView " + + "WHERE " + + viewDefinition.getTenantIdColumn() + + " = '" + + TENANT_ID + + "' " + + "AND tags__keys = 'flags' and tags__values = '0' " + + "AND mapvalue(tags__keys,'flags',tags__values) = '0'", + viewDefinition, + executionContext); + } + @Test public void testQueryWithOrderByWithPagination() { QueryRequest orderByQueryRequest = buildOrderByQuery(); diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/projection/ProjectionTransformationTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/projection/ProjectionTransformationTest.java index 4ded70c5..e381ea07 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/projection/ProjectionTransformationTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/projection/ProjectionTransformationTest.java @@ -10,23 +10,18 @@ import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_CONDITIONAL; import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_HASH; import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_STRINGEQUALS; -import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedAttributeExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createAliasedFunctionExpression; -import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression; -import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createComplexAttributeExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createCompositeFilter; -import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createEqualsFilter; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFilter; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFunctionExpression; -import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createInFilter; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createOrderByExpression; import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createStringArrayLiteralValueExpression; -import static org.hypertrace.core.query.service.QueryRequestUtil.createContainsKeyFilter; import static org.hypertrace.core.query.service.QueryRequestUtil.createNullNumberLiteralExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createNullStringLiteralExpression; +import static org.hypertrace.core.query.service.QueryRequestUtil.createSimpleAttributeExpression; import static org.hypertrace.core.query.service.QueryRequestUtil.createStringLiteralValueExpression; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import io.reactivex.rxjava3.core.Single; @@ -42,8 +37,6 @@ import org.hypertrace.core.attribute.service.v1.ProjectionExpression; import org.hypertrace.core.attribute.service.v1.ProjectionOperator; import org.hypertrace.core.query.service.QueryTransformation.QueryTransformationContext; -import org.hypertrace.core.query.service.api.Expression; -import org.hypertrace.core.query.service.api.Filter; import org.hypertrace.core.query.service.api.Operator; import org.hypertrace.core.query.service.api.QueryRequest; import org.hypertrace.core.query.service.api.SortOrder; @@ -78,256 +71,18 @@ void beforeEach() { new ProjectionTransformation(this.mockAttributeClient, new AttributeProjectionRegistry()); } - @Test - void transQueryWithComplexAttributeExpression_SingleFilter() { - this.mockAttribute("server", AttributeMetadata.getDefaultInstance()); - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression spanTags = createComplexAttributeExpression("Span.tags", "span.kind").build(); - Filter filter = - Filter.newBuilder() - .setLhs(spanTags) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("server")) - .build(); - QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); - - QueryRequest expectedTransform = - QueryRequest.newBuilder() - .setFilter( - Filter.newBuilder() - .setOperator(Operator.AND) - .addAllChildFilter( - List.of(filter, createContainsKeyFilter(spanTags.getAttributeExpression()))) - .build()) - .build(); - - assertEquals( - expectedTransform, - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet()); - } - - @Test - void transQueryWithComplexAttributeExpression_MultipleFilter() { - this.mockAttribute("server", AttributeMetadata.getDefaultInstance()); - this.mockAttribute("0", AttributeMetadata.getDefaultInstance()); - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression spanTags1 = createComplexAttributeExpression("Span.tags", "FLAGS").build(); - Expression spanTags2 = createComplexAttributeExpression("Span.tags", "span.kind").build(); - - Filter childFilter1 = - Filter.newBuilder() - .setLhs(spanTags1) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("0")) - .build(); - Filter childFilter2 = - Filter.newBuilder() - .setLhs(spanTags2) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("server")) - .build(); - - Filter.Builder filter = createCompositeFilter(Operator.AND, childFilter1, childFilter2); - QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); - - QueryRequest expectedTransform = - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet(); - List childFilterList = expectedTransform.getFilter().getChildFilterList(); - - assertTrue( - childFilterList - .get(0) - .getChildFilterList() - .contains(createContainsKeyFilter(spanTags1.getAttributeExpression()))); - assertTrue( - childFilterList - .get(1) - .getChildFilterList() - .contains(createContainsKeyFilter(spanTags2.getAttributeExpression()))); - } - - @Test - void transQueryWithComplexAttributeExpression_HierarchicalFilter() { - this.mockAttribute("server", AttributeMetadata.getDefaultInstance()); - this.mockAttribute("0", AttributeMetadata.getDefaultInstance()); - this.mockAttribute(SIMPLE_ATTRIBUTE_ID, AttributeMetadata.getDefaultInstance()); - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression spanTags1 = createComplexAttributeExpression("Span.tags", "FLAGS").build(); - Expression spanTags2 = createComplexAttributeExpression("Span.tags", "span.kind").build(); - - Filter filter1 = - Filter.newBuilder() - .setLhs(spanTags1) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("0")) - .build(); - Filter filter2 = - Filter.newBuilder() - .setLhs(spanTags2) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("server")) - .build(); - Filter filter = - createCompositeFilter( - Operator.AND, - createEqualsFilter(SIMPLE_ATTRIBUTE_ID, "otherValue"), - createCompositeFilter(Operator.AND, filter1, filter2).build()) - .build(); - - QueryRequest originalRequest = QueryRequest.newBuilder().setFilter(filter).build(); - - QueryRequest expectedTransform = - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet(); - List childFilterList = - expectedTransform.getFilter().getChildFilterList().get(1).getChildFilterList(); - - assertTrue( - childFilterList - .get(0) - .getChildFilterList() - .contains(createContainsKeyFilter(spanTags1.getAttributeExpression()))); - assertTrue( - childFilterList - .get(1) - .getChildFilterList() - .contains(createContainsKeyFilter(spanTags2.getAttributeExpression()))); - } - - @Test - void transQueryWithComplexAttributeExpression_OrderByAndFilter() { - this.mockAttribute("server", AttributeMetadata.getDefaultInstance()); - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); - - Filter filter = - Filter.newBuilder() - .setLhs(spanTag) - .setOperator(Operator.EQ) - .setRhs(createColumnExpression("server")) - .build(); - Filter containsKeyFilter = createContainsKeyFilter("Span.tags", "span.kind"); - - QueryRequest originalRequest = - QueryRequest.newBuilder() - .setFilter(filter) - .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) - .build(); - - QueryRequest expectedTransform = - QueryRequest.newBuilder() - .setFilter( - createCompositeFilter( - Operator.AND, - createCompositeFilter(Operator.AND, filter, containsKeyFilter).build(), - containsKeyFilter)) - .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) - .build(); - - assertEquals( - expectedTransform, - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet()); - } - - @Test - void transQueryWithComplexAttributeExpression_SingleSelection() { - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); - - QueryRequest originalRequest = QueryRequest.newBuilder().addSelection(spanTag).build(); - - QueryRequest expectedTransform = - QueryRequest.newBuilder() - .addSelection(spanTag) - .setFilter(createContainsKeyFilter("Span.tags", "span.kind")) - .build(); - - assertEquals( - expectedTransform, - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet()); - } - - @Test - void transQueryWithComplexAttributeExpression_SingleOrderBy() { - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression.Builder spanTag = createComplexAttributeExpression("Span.tags", "span.kind"); - - QueryRequest originalRequest = - QueryRequest.newBuilder() - .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) - .build(); - - QueryRequest expectedTransform = - QueryRequest.newBuilder() - .setFilter(createContainsKeyFilter("Span.tags", "span.kind")) - .addOrderBy(createOrderByExpression(spanTag, SortOrder.ASC)) - .build(); - - assertEquals( - expectedTransform, - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet()); - } - - @Test - void transQueryWithComplexAttributeExpression_MultipleOrderBy() { - this.mockAttribute("Span.tags", AttributeMetadata.getDefaultInstance()); - - Expression.Builder spanTag1 = createComplexAttributeExpression("Span.tags", "span.kind"); - Expression.Builder spanTag2 = createComplexAttributeExpression("Span.tags", "FLAGS"); - - QueryRequest originalRequest = - QueryRequest.newBuilder() - .addOrderBy(createOrderByExpression(spanTag1, SortOrder.ASC)) - .addOrderBy(createOrderByExpression(spanTag2, SortOrder.ASC)) - .build(); - - QueryRequest expectedTransform = - QueryRequest.newBuilder() - .setFilter( - createCompositeFilter( - Operator.AND, - createContainsKeyFilter("Span.tags", "span.kind"), - createContainsKeyFilter("Span.tags", "FLAGS"))) - .addOrderBy(createOrderByExpression(spanTag1, SortOrder.ASC)) - .addOrderBy(createOrderByExpression(spanTag2, SortOrder.ASC)) - .build(); - - assertEquals( - expectedTransform, - this.projectionTransformation - .transform(originalRequest, mockTransformationContext) - .blockingGet()); - } - @Test void transformsBasicAliasProjection() { this.mockAttribute(PROJECTED_ATTRIBUTE_ID, this.attributeMetadata); this.mockAttribute(SIMPLE_ATTRIBUTE_ID, AttributeMetadata.getDefaultInstance()); QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() .addSelection( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) .build(); assertEquals( @@ -357,7 +112,7 @@ PROJECTION_OPERATOR_HASH, attributeIdProjection(SIMPLE_ATTRIBUTE_ID))), QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = @@ -367,7 +122,8 @@ PROJECTION_OPERATOR_HASH, attributeIdProjection(SIMPLE_ATTRIBUTE_ID))), QUERY_FUNCTION_CONCAT, PROJECTED_ATTRIBUTE_ID, createFunctionExpression( - QUERY_FUNCTION_HASH, createColumnExpression(SIMPLE_ATTRIBUTE_ID).build()), + QUERY_FUNCTION_HASH, + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build()), createStringLiteralValueExpression("projectionLiteral"))) .build(); @@ -405,7 +161,7 @@ PROJECTION_OPERATOR_HASH, attributeIdProjection(SIMPLE_ATTRIBUTE_ID))), QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = @@ -416,10 +172,11 @@ PROJECTION_OPERATOR_HASH, attributeIdProjection(SIMPLE_ATTRIBUTE_ID))), PROJECTED_ATTRIBUTE_ID, createFunctionExpression( QUERY_FUNCTION_STRINGEQUALS, - createColumnExpression(SIMPLE_ATTRIBUTE_ID).build(), + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build(), createStringLiteralValueExpression("foo")), createFunctionExpression( - QUERY_FUNCTION_HASH, createColumnExpression(SIMPLE_ATTRIBUTE_ID).build()), + QUERY_FUNCTION_HASH, + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build()), createStringLiteralValueExpression("projectionLiteral"))) .build(); @@ -440,7 +197,7 @@ void transformsAggregations() { createAliasedFunctionExpression( QUERY_FUNCTION_AVG, "myAlias", - createColumnExpression(PROJECTED_ATTRIBUTE_ID).build())) + createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID).build())) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() @@ -448,7 +205,7 @@ void transformsAggregations() { createAliasedFunctionExpression( QUERY_FUNCTION_AVG, "myAlias", - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID))) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID))) .build(); assertEquals( @@ -464,18 +221,18 @@ void transformsOrderBys() { this.mockAttribute(SIMPLE_ATTRIBUTE_ID, AttributeMetadata.getDefaultInstance()); QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .addOrderBy( createOrderByExpression( - createColumnExpression(PROJECTED_ATTRIBUTE_ID), SortOrder.DESC)) + createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID), SortOrder.DESC)) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() .addSelection( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) .addOrderBy( createOrderByExpression( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID) .toBuilder(), SortOrder.DESC)) .build(); @@ -493,26 +250,36 @@ void transformsNestedFilters() { this.mockAttribute(SIMPLE_ATTRIBUTE_ID, AttributeMetadata.getDefaultInstance()); QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .setFilter( createCompositeFilter( Operator.OR, - createInFilter(PROJECTED_ATTRIBUTE_ID, List.of("foo", "bar")), - createEqualsFilter(SIMPLE_ATTRIBUTE_ID, "otherValue"))) + createFilter( + createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID).build(), + Operator.IN, + createStringArrayLiteralValueExpression(List.of("foo", "bar"))), + createFilter( + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build(), + Operator.EQ, + createStringLiteralValueExpression("otherValue")))) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() .addSelection( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) .setFilter( createCompositeFilter( Operator.OR, createFilter( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID), + createAliasedAttributeExpression( + SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID), Operator.IN, createStringArrayLiteralValueExpression(List.of("foo", "bar"))), - createEqualsFilter(SIMPLE_ATTRIBUTE_ID, "otherValue"))) + createFilter( + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build(), + Operator.EQ, + createStringLiteralValueExpression("otherValue")))) .build(); assertEquals( @@ -530,16 +297,18 @@ void transformsGroupBys() { QueryRequest.newBuilder() .addAggregation( createFunctionExpression( - QUERY_FUNCTION_AVG, createColumnExpression(PROJECTED_ATTRIBUTE_ID).build())) - .addGroupBy(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + QUERY_FUNCTION_AVG, + createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID).build())) + .addGroupBy(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() .addAggregation( createFunctionExpression( QUERY_FUNCTION_AVG, - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID))) - .addGroupBy(createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID))) + .addGroupBy( + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) .build(); assertEquals( @@ -562,16 +331,16 @@ void passesThroughExpressionsInOrder() { QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression("slow")) - .addSelection(createColumnExpression(SIMPLE_ATTRIBUTE_ID)) - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression("slow")) + .addSelection(createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expected = QueryRequest.newBuilder() - .addSelection(createColumnExpression("slow")) - .addSelection(createColumnExpression(SIMPLE_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression("slow")) + .addSelection(createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID)) .addSelection( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID)) .build(); assertEquals( expected, @@ -593,17 +362,19 @@ void passesThroughOrderBysInOrder() { QueryRequest originalRequest = QueryRequest.newBuilder() - .addOrderBy(createOrderByExpression(createColumnExpression("slow"), SortOrder.ASC)) + .addOrderBy( + createOrderByExpression(createSimpleAttributeExpression("slow"), SortOrder.ASC)) .addOrderBy( createOrderByExpression( - createColumnExpression(PROJECTED_ATTRIBUTE_ID), SortOrder.DESC)) + createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID), SortOrder.DESC)) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() - .addOrderBy(createOrderByExpression(createColumnExpression("slow"), SortOrder.ASC)) + .addOrderBy( + createOrderByExpression(createSimpleAttributeExpression("slow"), SortOrder.ASC)) .addOrderBy( createOrderByExpression( - createAliasedColumnExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID) + createAliasedAttributeExpression(SIMPLE_ATTRIBUTE_ID, PROJECTED_ATTRIBUTE_ID) .toBuilder(), SortOrder.DESC)) .build(); @@ -628,14 +399,14 @@ void transformNullDefinedAttributes() { QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) - .addSelection(createColumnExpression(SIMPLE_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = QueryRequest.newBuilder() .addSelection(createNullStringLiteralExpression()) - .addSelection(createColumnExpression(SIMPLE_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID)) .build(); assertEquals( @@ -656,7 +427,7 @@ void transformNullDefinedAttributes() { expectedTransform = QueryRequest.newBuilder() .addSelection(createNullNumberLiteralExpression()) - .addSelection(createColumnExpression(SIMPLE_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID)) .build(); assertEquals( @@ -687,7 +458,7 @@ void transformsNullProjectionArguments() { QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); QueryRequest expectedTransform = @@ -697,7 +468,7 @@ void transformsNullProjectionArguments() { QUERY_FUNCTION_CONDITIONAL, PROJECTED_ATTRIBUTE_ID, createStringLiteralValueExpression("true"), - createColumnExpression(SIMPLE_ATTRIBUTE_ID).build(), + createSimpleAttributeExpression(SIMPLE_ATTRIBUTE_ID).build(), createNullStringLiteralExpression())) .build(); @@ -714,7 +485,7 @@ void doesNotTransformMaterializedDefinition() { PROJECTED_ATTRIBUTE_ID, this.attributeMetadata.toBuilder().setMaterialized(true).build()); QueryRequest originalRequest = QueryRequest.newBuilder() - .addSelection(createColumnExpression(PROJECTED_ATTRIBUTE_ID)) + .addSelection(createSimpleAttributeExpression(PROJECTED_ATTRIBUTE_ID)) .build(); assertEquals( diff --git a/query-service/build.gradle.kts b/query-service/build.gradle.kts index edec2e29..76e19ebd 100644 --- a/query-service/build.gradle.kts +++ b/query-service/build.gradle.kts @@ -10,30 +10,19 @@ plugins { dependencies { implementation(project(":query-service-impl")) - implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.6.2") - implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.28") - implementation("org.slf4j:slf4j-api:1.7.30") + implementation("org.hypertrace.core.grpcutils:grpc-server-utils:0.7.0") + implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.33") + implementation("org.slf4j:slf4j-api:1.7.32") implementation("com.typesafe:config:1.4.1") runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.17.0") - runtimeOnly("io.grpc:grpc-netty:1.42.0") - constraints { - runtimeOnly("io.netty:netty-codec-http2:4.1.71.Final") { - because("https://snyk.io/vuln/SNYK-JAVA-IONETTY-1089809") - } - runtimeOnly("io.netty:netty-handler-proxy:4.1.71.Final") { - because("https://snyk.io/vuln/SNYK-JAVA-IONETTY-1089809") - } - } - + runtimeOnly("io.grpc:grpc-netty") integrationTestImplementation("com.google.protobuf:protobuf-java-util:3.17.3") - integrationTestImplementation("org.junit.jupiter:junit-jupiter-api:5.7.1") - integrationTestImplementation("org.junit.jupiter:junit-jupiter-params:5.7.1") - integrationTestImplementation("org.junit.jupiter:junit-jupiter-engine:5.7.1") - integrationTestImplementation("org.testcontainers:testcontainers:1.15.2") - integrationTestImplementation("org.testcontainers:junit-jupiter:1.15.2") - integrationTestImplementation("org.testcontainers:kafka:1.15.2") - integrationTestImplementation("org.hypertrace.core.serviceframework:integrationtest-service-framework:0.1.28") + integrationTestImplementation("org.junit.jupiter:junit-jupiter:5.7.1") + integrationTestImplementation("org.testcontainers:testcontainers:1.16.2") + integrationTestImplementation("org.testcontainers:junit-jupiter:1.16.2") + integrationTestImplementation("org.testcontainers:kafka:1.16.2") + integrationTestImplementation("org.hypertrace.core.serviceframework:integrationtest-service-framework:0.1.33") integrationTestImplementation("com.github.stefanbirkner:system-lambda:1.2.0") integrationTestImplementation("org.apache.kafka:kafka-clients:5.5.1-ccs") diff --git a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java index 9927e7eb..da0d3148 100644 --- a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java +++ b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java @@ -96,8 +96,7 @@ public static void setup() throws Exception { new GenericContainer<>(DockerImageName.parse("hypertrace/pinot-servicemanager:main")) .withNetwork(network) .withNetworkAliases("pinot-controller", "pinot-server", "pinot-broker") - .withExposedPorts(8099) - .withExposedPorts(9000) + .withExposedPorts(8099, 9000) .dependsOn(kafkaZk) .withStartupAttempts(CONTAINER_STARTUP_ATTEMPTS) .waitingFor(Wait.forLogMessage(".*Completed schema installation.*", 1))