diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index d5f79b8946c..46bb91415dd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -62,6 +62,7 @@ public enum BuiltinFunctionName { /** Collection functions */ ARRAY(FunctionName.of("array")), ARRAY_LENGTH(FunctionName.of("array_length")), + MVAPPEND(FunctionName.of("mvappend")), MVJOIN(FunctionName.of("mvjoin")), FORALL(FunctionName.of("forall")), EXISTS(FunctionName.of("exists")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java new file mode 100644 index 00000000000..a8bc882855c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; + +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * MVAppend function that appends all elements from arguments to create an array. Always returns an + * array or null for consistent type behavior. + */ +public class MVAppendFunctionImpl extends ImplementorUDF { + + public MVAppendFunctionImpl() { + super(new MVAppendImplementor(), NullPolicy.ALL); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + + if (sqlOperatorBinding.getOperandCount() == 0) { + return typeFactory.createSqlType(SqlTypeName.NULL); + } + + RelDataType elementType = determineElementType(sqlOperatorBinding, typeFactory); + return createArrayType( + typeFactory, typeFactory.createTypeWithNullability(elementType, true), true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + private static RelDataType determineElementType( + SqlOperatorBinding sqlOperatorBinding, RelDataTypeFactory typeFactory) { + RelDataType mostGeneralType = null; + + for (int i = 0; i < sqlOperatorBinding.getOperandCount(); i++) { + RelDataType operandType = getComponentType(sqlOperatorBinding.getOperandType(i)); + + mostGeneralType = updateMostGeneralType(mostGeneralType, operandType, typeFactory); + } + + return mostGeneralType != null ? mostGeneralType : typeFactory.createSqlType(SqlTypeName.NULL); + } + + private static RelDataType getComponentType(RelDataType operandType) { + if (!operandType.isStruct() && operandType.getComponentType() != null) { + return operandType.getComponentType(); + } + return operandType; + } + + private static RelDataType updateMostGeneralType( + RelDataType current, RelDataType candidate, RelDataTypeFactory typeFactory) { + if (current == null) { + return candidate; + } + + if (!current.equals(candidate)) { + return typeFactory.createSqlType(SqlTypeName.ANY); + } else { + return current; + } + } + + public static class MVAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Expressions.newArrayInit(Object.class, translatedOperands)); + } + } + + public static Object mvappend(Object... args) { + List elements = collectElements(args); + return elements.isEmpty() ? null : elements; + } + + private static List collectElements(Object... args) { + List elements = new ArrayList<>(); + + for (Object arg : args) { + if (arg == null) { + continue; + } + + if (arg instanceof List) { + addListElements((List) arg, elements); + } else { + elements.add(arg); + } + } + + return elements; + } + + private static void addListElements(List list, List elements) { + for (Object item : list) { + if (item != null) { + elements.add(item); + } + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 1097a9a5b0d..f92f70f519a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -46,6 +46,7 @@ import org.opensearch.sql.expression.function.CollectionUDF.ExistsFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.FilterFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ForallFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.MVAppendFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ReduceFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.TransformFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; @@ -383,6 +384,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); public static final SqlOperator FILTER = new FilterFunctionImpl().toUDF("filter"); public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform"); public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index eeeed029b69..d9ba4276d41 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -144,6 +144,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; @@ -834,6 +835,7 @@ void populate() { PPLTypeChecker.family(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER)); registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); + registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND); registerOperator(ARRAY_LENGTH, SqlLibraryOperators.ARRAY_LENGTH); registerOperator(FORALL, PPLBuiltinOperators.FORALL); registerOperator(EXISTS, PPLBuiltinOperators.EXISTS); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java new file mode 100644 index 00000000000..31fda119961 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** Unit tests for MVAppendFunctionImpl. */ +public class MVAppendFunctionImplTest { + + @Test + public void testMvappendWithNoArguments() { + Object result = MVAppendFunctionImpl.mvappend(); + assertNull(result); + } + + @Test + public void testMvappendWithSingleElement() { + Object result = MVAppendFunctionImpl.mvappend(42); + assertEquals(Arrays.asList(42), result); + } + + @Test + public void testMvappendWithMultipleElements() { + Object result = MVAppendFunctionImpl.mvappend(1, 2, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithNullValues() { + Object result = MVAppendFunctionImpl.mvappend(null, 1, null); + assertEquals(Arrays.asList(1), result); + } + + @Test + public void testMvappendWithAllNulls() { + Object result = MVAppendFunctionImpl.mvappend(null, null); + assertNull(result); + } + + @Test + public void testMvappendWithArrayFlattening() { + List array1 = Arrays.asList(1, 2); + List array2 = Arrays.asList(3, 4); + Object result = MVAppendFunctionImpl.mvappend(array1, array2); + assertEquals(Arrays.asList(1, 2, 3, 4), result); + } + + @Test + public void testMvappendWithMixedTypes() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(array, 3, "hello"); + assertEquals(Arrays.asList(1, 2, 3, "hello"), result); + } + + @Test + public void testMvappendWithArrayAndNulls() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(null, array, null, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithSingleNull() { + Object result = MVAppendFunctionImpl.mvappend((Object) null); + assertNull(result); + } + + @Test + public void testMvappendWithEmptyArray() { + List emptyArray = Arrays.asList(); + Object result = MVAppendFunctionImpl.mvappend(emptyArray, 1); + assertEquals(Arrays.asList(1), result); + } +} diff --git a/docs/category.json b/docs/category.json index cb40f1ebbcd..c98909dc9ac 100644 --- a/docs/category.json +++ b/docs/category.json @@ -54,6 +54,7 @@ "user/ppl/cmd/top.rst", "user/ppl/cmd/trendline.rst", "user/ppl/cmd/where.rst", + "user/ppl/functions/collection.rst", "user/ppl/functions/condition.rst", "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", diff --git a/docs/user/ppl/functions/collection.rst b/docs/user/ppl/functions/collection.rst index 95d55fa7e2d..76931e53876 100644 --- a/docs/user/ppl/functions/collection.rst +++ b/docs/user/ppl/functions/collection.rst @@ -14,8 +14,6 @@ ARRAY Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array(value1, value2, value3...)`` create an array with input values. Currently we don't allow mixture types. We will infer a least restricted type, for example ``array(1, "demo")`` -> ["1", "demo"] Argument type: value1: ANY, value2: ANY, ... @@ -24,21 +22,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, 2, 3) | fields array | head 1 + os> source=people | eval array = array(1, 2, 3) | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | [1, 2, 3] | - +----------------------------------+ + +---------+ + | array | + |---------| + | [1,2,3] | + +---------+ - PPL> source=people | eval array = array(1, "demo") | fields array | head 1 + os> source=people | eval array = array(1, "demo") | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | ["1", "demo"] | - +----------------------------------+ + +----------+ + | array | + |----------| + | [1,demo] | + +----------+ ARRAY_LENGTH ------------ @@ -46,8 +44,6 @@ ARRAY_LENGTH Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array_length(array)`` returns the length of input array. Argument type: array:ARRAY @@ -56,13 +52,13 @@ Return type: INTEGER Example:: - PPL> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 + os> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 fetched rows / total rows = 1/1 - +---------------+ - | length | - |---------------| - | 4 | - +---------------+ + +--------+ + | length | + |--------| + | 3 | + +--------+ FORALL ------ @@ -70,8 +66,6 @@ FORALL Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``forall(array, function)`` check whether all element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -80,13 +74,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ EXISTS ------ @@ -94,8 +88,6 @@ EXISTS Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``exists(array, function)`` check whether existing one of element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -104,13 +96,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ FILTER ------ @@ -118,8 +110,6 @@ FILTER Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``filter(array, function)`` filter the element in the array by the lambda function. The function should return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -128,13 +118,13 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | [1, 3] | - +---------+ + +--------+ + | result | + |--------| + | [1,3] | + +--------+ TRANSFORM --------- @@ -142,8 +132,6 @@ TRANSFORM Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``transform(array, function)`` transform the element of array one by one using lambda. The lambda function can accept one single input or two input. If the lambda accepts two argument, the second one is the index of element in array. Argument type: array:ARRAY, function:LAMBDA @@ -152,21 +140,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [3, 0, 5] | - +------------+ + +---------+ + | result | + |---------| + | [3,0,5] | + +---------+ - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [1, -1, 5] | - +------------+ + +----------+ + | result | + |----------| + | [1,-1,5] | + +----------+ REDUCE ------ @@ -174,8 +162,6 @@ REDUCE Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``reduce(array, acc_base, function, )`` use lambda function to go through all element and interact with acc_base. The lambda function accept two argument accumulator and array element. If add one more reduce_function, will apply reduce_function to accumulator finally. The reduce function accept accumulator as the one argument. Argument type: array:ARRAY, acc_base:ANY, function:LAMBDA, reduce_function:LAMBDA @@ -184,21 +170,21 @@ Return type: ANY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 8 | - +------------+ + +--------+ + | result | + |--------| + | 12 | + +--------+ - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 80 | - +------------+ + +--------+ + | result | + |--------| + | 120 | + +--------+ MVJOIN ------ @@ -206,8 +192,6 @@ MVJOIN Description >>>>>>>>>>> -Version: 3.3.0 - Usage: mvjoin(array, delimiter) joins string array elements into a single string, separated by the specified delimiter. NULL elements are excluded from the output. Only string arrays are supported. Argument type: array: ARRAY of STRING, delimiter: STRING @@ -216,19 +200,104 @@ Return type: STRING Example:: - PPL> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + os> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | a,b,c | + +--------+ + + os> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + fetched rows / total rows = 1/1 + +-------------+ + | result | + |-------------| + | Amber, Duke | + +-------------+ + +MVAPPEND +-------- + +Description +>>>>>>>>>>> + +Usage: mvappend(value1, value2, value3...) appends all elements from arguments to create an array. Flattens array arguments and collects all individual elements. Always returns an array or null for consistent type behavior. + +Argument type: value1: ANY, value2: ANY, ... + +Return type: ARRAY + +Example:: + + os> source=people | eval result = mvappend(1, 1, 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,1,3] | + +---------+ + + os> source=people | eval result = mvappend(1, array(2, 3)) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(mvappend(1, 2), 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(42) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [42] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1), 2) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [2] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1)) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | null | + +--------+ + + os> source=people | eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | fields result | head 1 + fetched rows / total rows = 1/1 + +-----------+ + | result | + |-----------| + | [1,2,3,4] | + +-----------+ + + os> source=accounts | eval result = mvappend(firstname, lastname) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------+ - | result | - |------------------------------------| - | "a,b,c" | - +------------------------------------+ + +--------------+ + | result | + |--------------| + | [Amber,Duke] | + +--------------+ - PPL> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + os> source=people | eval result = mvappend(1, 'text', 2.5) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------------+ - | result | - |------------------------------------------| - | "Amber, Duke" | - +------------------------------------------+ - + +--------------+ + | result | + |--------------| + | [1,text,2.5] | + +--------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java new file mode 100644 index 00000000000..cf84cbe7db6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java @@ -0,0 +1,221 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteMVAppendFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.BANK); + } + + @Test + public void testMvappendWithMultipleElements() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2, 3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3))); + } + + @Test + public void testMvappendWithSingleElement() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(42) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(42))); + } + + @Test + public void testMvappendWithArrayFlattening() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | head" + + " 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithMixedArrayAndScalar() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr = array(1, 2), result = mvappend(arr, 3, 4) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithStringValues() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('hello', 'world') | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("hello", "world"))); + } + + @Test + public void testMvappendWithMixedTypes() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(1, 'text', 2.5) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, "text", 2.5))); + } + + @Test + public void testMvappendWithIntAndDouble() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2.5) | head 1 | fields result")); + + System.out.println(actual); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2.5))); + } + + @Test + public void testMvappendWithRealFields() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(firstname, lastname) | head 1 | fields firstname, lastname," + + " result")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("result", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithFieldsAndLiterals() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(age, 'years', 'old') | head 1 | fields age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, "years", "old"))); + } + + @Test + public void testMvappendWithEmptyArray() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval empty_arr = array(), result = mvappend(empty_arr, 1, 2) | head 1 | fields" + + " result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2))); + } + + @Test + public void testMvappendWithNestedArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array('a', 'b'), arr2 = array('c'), arr3 = array('d', 'e'), result =" + + " mvappend(arr1, arr2, arr3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("a", "b", "c", "d", "e"))); + } + + @Test + public void testMvappendWithNumericArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1.5, 2.5), arr2 = array(3.5), result = mvappend(arr1, arr2, 4.5)" + + " | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1.5, 2.5, 3.5, 4.5))); + } + + @Test + public void testMvappendInWhereClause() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval combined = mvappend(firstname, lastname) | where array_length(combined) = 2 |" + + " head 1 | fields firstname, lastname, combined")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("combined", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithComplexExpression() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(array(age), array(age * 2), age + 10) | head 1 | fields" + + " age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, 64, 42))); + } + + @Test + public void testMvappendWithNull() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('test', nullif(1, 1), 2) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("test", 2))); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 230297b21c0..a52a462dfcd 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -425,6 +425,7 @@ ISBLANK: 'ISBLANK'; // COLLECTION FUNCTIONS ARRAY: 'ARRAY'; ARRAY_LENGTH: 'ARRAY_LENGTH'; +MVAPPEND: 'MVAPPEND'; MVJOIN: 'MVJOIN'; FORALL: 'FORALL'; FILTER: 'FILTER'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index d702d366ae5..d8bd8708ce4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1011,6 +1011,7 @@ geoipFunctionName collectionFunctionName : ARRAY | ARRAY_LENGTH + | MVAPPEND | MVJOIN | FORALL | EXISTS @@ -1503,4 +1504,4 @@ searchableKeyWord | LEFT_HINT | RIGHT_HINT | PERCENTILE_SHORTCUT - ; \ No newline at end of file + ; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 4184d8211d9..566d7a50dfc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -676,6 +676,13 @@ public void testMvjoin() { anonymize("source=t | eval result=mvjoin(array('a', 'b', 'c'), ',') | fields result")); } + @Test + public void testMvappend() { + assertEquals( + "source=table | eval identifier=mvappend(identifier,***,***) | fields + identifier", + anonymize("source=t | eval result=mvappend(a, 'b', 'c') | fields result")); + } + @Test public void testRexWithOffsetField() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10);