Skip to content

Commit 436adec

Browse files
varsha-abhinandanvarsha-abhinandan
authored
Adding support for IN and NOT IN for non-string array values (#207)
* ENG-36266 Adding support for IN and NOT IN for non-string array values
1 parent 2c4635c commit 436adec

File tree

8 files changed

+296
-64
lines changed

8 files changed

+296
-64
lines changed

‎owasp-suppressions.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,15 @@
2828
<cve>CVE-2019-0201</cve>
2929
<cve>CVE-2023-44981</cve>
3030
</suppress>
31-
<suppress until="2023-10-31Z">
31+
<suppress until="2023-11-30Z">
3232
<notes><![CDATA[
3333
file name: jackson-databind-2.14.2.jar
3434
This is currently disputed.
3535
]]></notes>
3636
<packageUrl regex="true">^pkg:maven/com\.fasterxml\.jackson\.core/jackson\-databind@.*$</packageUrl>
3737
<cve>CVE-2023-35116</cve>
3838
</suppress>
39-
<suppress until="2023-10-31Z">
39+
<suppress until="2023-11-30Z">
4040
<notes><![CDATA[
4141
file name: netty-handler-4.1.94.Final.jar
4242
]]></notes>

‎query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverter.java‎

Lines changed: 32 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@
66
import static org.hypertrace.core.query.service.api.Expression.ValueCase.LITERAL;
77

88
import com.google.common.base.Joiner;
9-
import com.google.protobuf.ByteString;
109
import java.util.AbstractMap.SimpleEntry;
1110
import java.util.ArrayList;
1211
import java.util.LinkedHashSet;
1312
import java.util.List;
1413
import java.util.Map.Entry;
1514
import java.util.Optional;
15+
import java.util.function.Consumer;
1616
import org.hypertrace.core.query.service.ExecutionContext;
1717
import org.hypertrace.core.query.service.api.Expression;
1818
import org.hypertrace.core.query.service.api.Filter;
@@ -450,40 +450,28 @@ private String convertLiteralToString(LiteralConstant literal, Params.Builder pa
450450
String ret = null;
451451
switch (value.getValueType()) {
452452
case STRING_ARRAY:
453-
StringBuilder builder = new StringBuilder("(");
454-
String delim = "";
455-
for (String item : value.getStringArrayList()) {
456-
builder.append(delim);
457-
builder.append(QUESTION_MARK);
458-
paramsBuilder.addStringParam(item);
459-
delim = ", ";
460-
}
461-
builder.append(")");
462-
ret = builder.toString();
453+
ret = buildArrayValue(value.getStringArrayList(), paramsBuilder::addStringParam);
463454
break;
464455
case BYTES_ARRAY:
465-
builder = new StringBuilder("(");
466-
delim = "";
467-
for (ByteString item : value.getBytesArrayList()) {
468-
builder.append(delim);
469-
builder.append(QUESTION_MARK);
470-
paramsBuilder.addByteStringParam(item);
471-
delim = ", ";
472-
}
473-
builder.append(")");
474-
ret = builder.toString();
456+
ret = buildArrayValue(value.getBytesArrayList(), paramsBuilder::addByteStringParam);
475457
break;
476458
case BOOLEAN_ARRAY:
477-
builder = new StringBuilder("(");
478-
delim = "";
479-
for (Boolean item : value.getBooleanArrayList()) {
480-
builder.append(delim);
481-
builder.append(QUESTION_MARK);
482-
paramsBuilder.addStringParam(String.valueOf(item));
483-
delim = ", ";
484-
}
485-
builder.append(")");
486-
ret = builder.toString();
459+
ret =
460+
buildArrayValue(
461+
value.getBooleanArrayList(),
462+
item -> paramsBuilder.addStringParam(String.valueOf(item)));
463+
break;
464+
case LONG_ARRAY:
465+
ret = buildArrayValue(value.getLongArrayList(), paramsBuilder::addLongParam);
466+
break;
467+
case INT_ARRAY:
468+
ret = buildArrayValue(value.getIntArrayList(), paramsBuilder::addIntegerParam);
469+
break;
470+
case FLOAT_ARRAY:
471+
ret = buildArrayValue(value.getFloatArrayList(), paramsBuilder::addFloatParam);
472+
break;
473+
case DOUBLE_ARRAY:
474+
ret = buildArrayValue(value.getDoubleArrayList(), paramsBuilder::addDoubleParam);
487475
break;
488476
case STRING:
489477
ret = QUESTION_MARK;
@@ -525,13 +513,22 @@ private String convertLiteralToString(LiteralConstant literal, Params.Builder pa
525513
ret = QUESTION_MARK;
526514
paramsBuilder.addStringParam("null");
527515
break;
528-
case LONG_ARRAY:
529-
case INT_ARRAY:
530-
case FLOAT_ARRAY:
531-
case DOUBLE_ARRAY:
532516
case UNRECOGNIZED:
533517
break;
534518
}
535519
return ret;
536520
}
521+
522+
private <T> String buildArrayValue(List<T> values, Consumer<T> paramAdder) {
523+
StringBuilder builder = new StringBuilder("(");
524+
String delim = "";
525+
for (T item : values) {
526+
builder.append(delim);
527+
builder.append(QUESTION_MARK);
528+
paramAdder.accept(item);
529+
delim = ", ";
530+
}
531+
builder.append(")");
532+
return builder.toString();
533+
}
537534
}

‎query-service-impl/src/main/java/org/hypertrace/core/query/service/postgres/converters/DefaultColumnRequestConverter.java‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111
import static org.hypertrace.core.query.service.postgres.converters.ColumnRequestContext.createColumnRequestContext;
1212

1313
import com.google.common.base.Strings;
14-
import com.google.protobuf.ByteString;
1514
import java.util.ArrayList;
1615
import java.util.List;
16+
import java.util.function.Consumer;
1717
import java.util.stream.Collectors;
1818
import org.apache.commons.codec.DecoderException;
1919
import org.apache.commons.codec.binary.Hex;
@@ -453,28 +453,28 @@ private String convertLiteralToString(LiteralConstant literal, Builder paramsBui
453453
String ret = null;
454454
switch (value.getValueType()) {
455455
case STRING_ARRAY:
456-
StringBuilder builder = new StringBuilder("(");
457-
String delim = "";
458-
for (String item : value.getStringArrayList()) {
459-
builder.append(delim);
460-
builder.append(QUESTION_MARK);
461-
paramsBuilder.addStringParam(item);
462-
delim = ", ";
463-
}
464-
builder.append(")");
465-
ret = builder.toString();
456+
ret = buildArrayValue(value.getStringArrayList(), paramsBuilder::addStringParam);
466457
break;
467458
case BYTES_ARRAY:
468-
builder = new StringBuilder("(");
469-
delim = "";
470-
for (ByteString item : value.getBytesArrayList()) {
471-
builder.append(delim);
472-
builder.append(QUESTION_MARK);
473-
paramsBuilder.addByteStringParam(item);
474-
delim = ", ";
475-
}
476-
builder.append(")");
477-
ret = builder.toString();
459+
ret = buildArrayValue(value.getBytesArrayList(), paramsBuilder::addByteStringParam);
460+
break;
461+
case LONG_ARRAY:
462+
ret = buildArrayValue(value.getLongArrayList(), paramsBuilder::addLongParam);
463+
break;
464+
case INT_ARRAY:
465+
ret = buildArrayValue(value.getIntArrayList(), paramsBuilder::addIntegerParam);
466+
break;
467+
case FLOAT_ARRAY:
468+
ret = buildArrayValue(value.getFloatArrayList(), paramsBuilder::addFloatParam);
469+
break;
470+
case DOUBLE_ARRAY:
471+
ret = buildArrayValue(value.getDoubleArrayList(), paramsBuilder::addDoubleParam);
472+
break;
473+
case BOOLEAN_ARRAY:
474+
ret =
475+
buildArrayValue(
476+
value.getBooleanArrayList(),
477+
item -> paramsBuilder.addStringParam(String.valueOf(item)));
478478
break;
479479
case STRING:
480480
ret = QUESTION_MARK;
@@ -516,14 +516,22 @@ private String convertLiteralToString(LiteralConstant literal, Builder paramsBui
516516
ret = QUESTION_MARK;
517517
paramsBuilder.addStringParam("null");
518518
break;
519-
case LONG_ARRAY:
520-
case INT_ARRAY:
521-
case FLOAT_ARRAY:
522-
case DOUBLE_ARRAY:
523-
case BOOLEAN_ARRAY:
524519
case UNRECOGNIZED:
525520
break;
526521
}
527522
return ret;
528523
}
524+
525+
private <T> String buildArrayValue(List<T> values, Consumer<T> paramAdder) {
526+
StringBuilder builder = new StringBuilder("(");
527+
String delim = "";
528+
for (T item : values) {
529+
builder.append(delim);
530+
builder.append(QUESTION_MARK);
531+
paramAdder.accept(item);
532+
delim = ", ";
533+
}
534+
builder.append(")");
535+
return builder.toString();
536+
}
529537
}

‎query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryRequestBuilderUtils.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,22 @@ public static Filter createBooleanInFilter(String column, List<Boolean> values)
124124
return createFilter(column, Operator.IN, createBooleanArrayLiteralValueExpression(values));
125125
}
126126

127+
public static Filter createLongInFilter(String column, List<Long> values) {
128+
return createFilter(column, Operator.IN, createLongArrayLiteralValueExpression(values));
129+
}
130+
131+
public static Filter createIntNotInFilter(String column, List<Integer> values) {
132+
return createFilter(column, Operator.NOT_IN, createIntArrayLiteralValueExpression(values));
133+
}
134+
135+
public static Filter createFloatNotInFilter(String column, List<Float> values) {
136+
return createFilter(column, Operator.NOT_IN, createFloatArrayLiteralValueExpression(values));
137+
}
138+
139+
public static Filter createDoubleInFilter(String column, List<Double> values) {
140+
return createFilter(column, Operator.IN, createDoubleArrayLiteralValueExpression(values));
141+
}
142+
127143
public static Filter createNotInFilter(String column, List<String> values) {
128144
return createFilter(column, Operator.NOT_IN, createStringArrayLiteralValueExpression(values));
129145
}
@@ -228,6 +244,15 @@ public static Expression createLongLiteralValueExpression(long value) {
228244
.build();
229245
}
230246

247+
public static Expression createLongArrayLiteralValueExpression(List<Long> values) {
248+
return Expression.newBuilder()
249+
.setLiteral(
250+
LiteralConstant.newBuilder()
251+
.setValue(
252+
Value.newBuilder().addAllLongArray(values).setValueType(ValueType.LONG_ARRAY)))
253+
.build();
254+
}
255+
231256
public static Expression createTimestampLiteralValueExpression(long value) {
232257
return Expression.newBuilder()
233258
.setLiteral(
@@ -244,6 +269,17 @@ public static Expression createDoubleLiteralValueExpression(double value) {
244269
.build();
245270
}
246271

272+
public static Expression createDoubleArrayLiteralValueExpression(List<Double> values) {
273+
return Expression.newBuilder()
274+
.setLiteral(
275+
LiteralConstant.newBuilder()
276+
.setValue(
277+
Value.newBuilder()
278+
.addAllDoubleArray(values)
279+
.setValueType(ValueType.DOUBLE_ARRAY)))
280+
.build();
281+
}
282+
247283
public static Expression createFloatLiteralValueExpression(float value) {
248284
return Expression.newBuilder()
249285
.setLiteral(
@@ -252,6 +288,17 @@ public static Expression createFloatLiteralValueExpression(float value) {
252288
.build();
253289
}
254290

291+
public static Expression createFloatArrayLiteralValueExpression(List<Float> values) {
292+
return Expression.newBuilder()
293+
.setLiteral(
294+
LiteralConstant.newBuilder()
295+
.setValue(
296+
Value.newBuilder()
297+
.addAllFloatArray(values)
298+
.setValueType(ValueType.FLOAT_ARRAY)))
299+
.build();
300+
}
301+
255302
public static Expression createBoolLiteralValueExpression(boolean value) {
256303
return Expression.newBuilder()
257304
.setLiteral(
@@ -268,6 +315,15 @@ public static Expression createIntLiteralValueExpression(int value) {
268315
.build();
269316
}
270317

318+
public static Expression createIntArrayLiteralValueExpression(List<Integer> values) {
319+
return Expression.newBuilder()
320+
.setLiteral(
321+
LiteralConstant.newBuilder()
322+
.setValue(
323+
Value.newBuilder().addAllIntArray(values).setValueType(ValueType.INT_ARRAY)))
324+
.build();
325+
}
326+
271327
public static Expression createNullStringLiteralValueExpression() {
272328
return Expression.newBuilder()
273329
.setLiteral(

‎query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverterTest.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,13 @@
66
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression;
77
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createCompositeFilter;
88
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createCountByColumnSelection;
9+
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createDoubleInFilter;
910
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createEqualsFilter;
11+
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFloatNotInFilter;
1012
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFunctionExpression;
1113
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createInFilter;
14+
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createIntNotInFilter;
15+
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createLongInFilter;
1216
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createLongLiteralValueExpression;
1317
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createNotEqualsFilter;
1418
import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createNullNumberLiteralValueExpression;
@@ -1060,6 +1064,77 @@ public void testQueryWithLikeOperatorForResponseBodyHavingTextIndex() {
10601064
executionContext);
10611065
}
10621066

1067+
@Test
1068+
public void testQueryWithIntArrayFilter() {
1069+
ViewDefinition viewDefinition = getDefaultViewDefinition();
1070+
defaultMockingForExecutionContext();
1071+
1072+
assertPQLQuery(
1073+
buildSimpleQueryWithFilter(
1074+
createIntNotInFilter("Span.metrics.duration_millis", List.of(123))),
1075+
"Select span_id FROM SpanEventView WHERE "
1076+
+ viewDefinition.getTenantIdColumn()
1077+
+ " = '"
1078+
+ TENANT_ID
1079+
+ "' "
1080+
+ "AND duration_millis NOT IN (123)",
1081+
viewDefinition,
1082+
executionContext);
1083+
}
1084+
1085+
@Test
1086+
public void testQueryWithLongArrayFilter() {
1087+
ViewDefinition viewDefinition = getDefaultViewDefinition();
1088+
defaultMockingForExecutionContext();
1089+
1090+
assertPQLQuery(
1091+
buildSimpleQueryWithFilter(
1092+
createLongInFilter("Span.metrics.duration_millis", List.of(10L, 20L, 30L))),
1093+
"SELECT span_id FROM SpanEventView WHERE "
1094+
+ viewDefinition.getTenantIdColumn()
1095+
+ " = '"
1096+
+ TENANT_ID
1097+
+ "' "
1098+
+ "AND duration_millis IN (10, 20, 30)",
1099+
viewDefinition,
1100+
executionContext);
1101+
}
1102+
1103+
@Test
1104+
public void testQueryWithFloatArrayFilter() {
1105+
ViewDefinition viewDefinition = getDefaultViewDefinition();
1106+
defaultMockingForExecutionContext();
1107+
1108+
assertPQLQuery(
1109+
buildSimpleQueryWithFilter(createFloatNotInFilter("Span.user_latitude", List.of(10.05f))),
1110+
"SELECT span_id FROM SpanEventView WHERE "
1111+
+ viewDefinition.getTenantIdColumn()
1112+
+ " = '"
1113+
+ TENANT_ID
1114+
+ "' "
1115+
+ "AND user_latitude NOT IN (10.05)",
1116+
viewDefinition,
1117+
executionContext);
1118+
}
1119+
1120+
@Test
1121+
public void testQueryWithDoubleArrayFilter() {
1122+
1123+
ViewDefinition viewDefinition = getDefaultViewDefinition();
1124+
defaultMockingForExecutionContext();
1125+
1126+
assertPQLQuery(
1127+
buildSimpleQueryWithFilter(createDoubleInFilter("Span.user_longitude", List.of(10.87654))),
1128+
"SELECT span_id FROM SpanEventView WHERE "
1129+
+ viewDefinition.getTenantIdColumn()
1130+
+ " = '"
1131+
+ TENANT_ID
1132+
+ "' "
1133+
+ "AND user_longitude IN (10.87654)",
1134+
viewDefinition,
1135+
executionContext);
1136+
}
1137+
10631138
private QueryRequest buildSimpleQueryWithFilter(Filter filter) {
10641139
Builder builder = QueryRequest.newBuilder();
10651140
builder.addSelection(createColumnExpression("Span.id").build());

0 commit comments

Comments
 (0)