diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index d5f79b8946c..46bb91415dd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -62,6 +62,7 @@ public enum BuiltinFunctionName { /** Collection functions */ ARRAY(FunctionName.of("array")), ARRAY_LENGTH(FunctionName.of("array_length")), + MVAPPEND(FunctionName.of("mvappend")), MVJOIN(FunctionName.of("mvjoin")), FORALL(FunctionName.of("forall")), EXISTS(FunctionName.of("exists")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java new file mode 100644 index 00000000000..a8bc882855c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; + +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * MVAppend function that appends all elements from arguments to create an array. Always returns an + * array or null for consistent type behavior. + */ +public class MVAppendFunctionImpl extends ImplementorUDF { + + public MVAppendFunctionImpl() { + super(new MVAppendImplementor(), NullPolicy.ALL); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + + if (sqlOperatorBinding.getOperandCount() == 0) { + return typeFactory.createSqlType(SqlTypeName.NULL); + } + + RelDataType elementType = determineElementType(sqlOperatorBinding, typeFactory); + return createArrayType( + typeFactory, typeFactory.createTypeWithNullability(elementType, true), true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + private static RelDataType determineElementType( + SqlOperatorBinding sqlOperatorBinding, RelDataTypeFactory typeFactory) { + RelDataType mostGeneralType = null; + + for (int i = 0; i < sqlOperatorBinding.getOperandCount(); i++) { + RelDataType operandType = getComponentType(sqlOperatorBinding.getOperandType(i)); + + mostGeneralType = updateMostGeneralType(mostGeneralType, operandType, typeFactory); + } + + return mostGeneralType != null ? mostGeneralType : typeFactory.createSqlType(SqlTypeName.NULL); + } + + private static RelDataType getComponentType(RelDataType operandType) { + if (!operandType.isStruct() && operandType.getComponentType() != null) { + return operandType.getComponentType(); + } + return operandType; + } + + private static RelDataType updateMostGeneralType( + RelDataType current, RelDataType candidate, RelDataTypeFactory typeFactory) { + if (current == null) { + return candidate; + } + + if (!current.equals(candidate)) { + return typeFactory.createSqlType(SqlTypeName.ANY); + } else { + return current; + } + } + + public static class MVAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Expressions.newArrayInit(Object.class, translatedOperands)); + } + } + + public static Object mvappend(Object... args) { + List elements = collectElements(args); + return elements.isEmpty() ? null : elements; + } + + private static List collectElements(Object... args) { + List elements = new ArrayList<>(); + + for (Object arg : args) { + if (arg == null) { + continue; + } + + if (arg instanceof List) { + addListElements((List) arg, elements); + } else { + elements.add(arg); + } + } + + return elements; + } + + private static void addListElements(List list, List elements) { + for (Object item : list) { + if (item != null) { + elements.add(item); + } + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 049f6c56533..750c455c682 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -46,6 +46,7 @@ import org.opensearch.sql.expression.function.CollectionUDF.ExistsFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.FilterFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ForallFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.MVAppendFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ReduceFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.TransformFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; @@ -383,6 +384,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); public static final SqlOperator FILTER = new FilterFunctionImpl().toUDF("filter"); public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform"); public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index ac910710eb8..5ba0951e6f4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -144,6 +144,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND; import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; @@ -843,6 +844,7 @@ void populate() { PPLTypeChecker.family(SqlTypeFamily.ARRAY, SqlTypeFamily.CHARACTER)); registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); + registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND); registerOperator(ARRAY_LENGTH, SqlLibraryOperators.ARRAY_LENGTH); registerOperator(FORALL, PPLBuiltinOperators.FORALL); registerOperator(EXISTS, PPLBuiltinOperators.EXISTS); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java new file mode 100644 index 00000000000..31fda119961 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImplTest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** Unit tests for MVAppendFunctionImpl. */ +public class MVAppendFunctionImplTest { + + @Test + public void testMvappendWithNoArguments() { + Object result = MVAppendFunctionImpl.mvappend(); + assertNull(result); + } + + @Test + public void testMvappendWithSingleElement() { + Object result = MVAppendFunctionImpl.mvappend(42); + assertEquals(Arrays.asList(42), result); + } + + @Test + public void testMvappendWithMultipleElements() { + Object result = MVAppendFunctionImpl.mvappend(1, 2, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithNullValues() { + Object result = MVAppendFunctionImpl.mvappend(null, 1, null); + assertEquals(Arrays.asList(1), result); + } + + @Test + public void testMvappendWithAllNulls() { + Object result = MVAppendFunctionImpl.mvappend(null, null); + assertNull(result); + } + + @Test + public void testMvappendWithArrayFlattening() { + List array1 = Arrays.asList(1, 2); + List array2 = Arrays.asList(3, 4); + Object result = MVAppendFunctionImpl.mvappend(array1, array2); + assertEquals(Arrays.asList(1, 2, 3, 4), result); + } + + @Test + public void testMvappendWithMixedTypes() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(array, 3, "hello"); + assertEquals(Arrays.asList(1, 2, 3, "hello"), result); + } + + @Test + public void testMvappendWithArrayAndNulls() { + List array = Arrays.asList(1, 2); + Object result = MVAppendFunctionImpl.mvappend(null, array, null, 3); + assertEquals(Arrays.asList(1, 2, 3), result); + } + + @Test + public void testMvappendWithSingleNull() { + Object result = MVAppendFunctionImpl.mvappend((Object) null); + assertNull(result); + } + + @Test + public void testMvappendWithEmptyArray() { + List emptyArray = Arrays.asList(); + Object result = MVAppendFunctionImpl.mvappend(emptyArray, 1); + assertEquals(Arrays.asList(1), result); + } +} diff --git a/docs/category.json b/docs/category.json index 5e37ab389f9..cd24a9e1213 100644 --- a/docs/category.json +++ b/docs/category.json @@ -53,6 +53,7 @@ "user/ppl/cmd/top.rst", "user/ppl/cmd/trendline.rst", "user/ppl/cmd/where.rst", + "user/ppl/functions/collection.rst", "user/ppl/functions/condition.rst", "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", diff --git a/docs/user/ppl/functions/collection.rst b/docs/user/ppl/functions/collection.rst index 95d55fa7e2d..76931e53876 100644 --- a/docs/user/ppl/functions/collection.rst +++ b/docs/user/ppl/functions/collection.rst @@ -14,8 +14,6 @@ ARRAY Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array(value1, value2, value3...)`` create an array with input values. Currently we don't allow mixture types. We will infer a least restricted type, for example ``array(1, "demo")`` -> ["1", "demo"] Argument type: value1: ANY, value2: ANY, ... @@ -24,21 +22,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, 2, 3) | fields array | head 1 + os> source=people | eval array = array(1, 2, 3) | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | [1, 2, 3] | - +----------------------------------+ + +---------+ + | array | + |---------| + | [1,2,3] | + +---------+ - PPL> source=people | eval array = array(1, "demo") | fields array | head 1 + os> source=people | eval array = array(1, "demo") | fields array | head 1 fetched rows / total rows = 1/1 - +----------------------------------+ - | array | - |----------------------------------| - | ["1", "demo"] | - +----------------------------------+ + +----------+ + | array | + |----------| + | [1,demo] | + +----------+ ARRAY_LENGTH ------------ @@ -46,8 +44,6 @@ ARRAY_LENGTH Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``array_length(array)`` returns the length of input array. Argument type: array:ARRAY @@ -56,13 +52,13 @@ Return type: INTEGER Example:: - PPL> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 + os> source=people | eval array = array(1, 2, 3) | eval length = array_length(array) | fields length | head 1 fetched rows / total rows = 1/1 - +---------------+ - | length | - |---------------| - | 4 | - +---------------+ + +--------+ + | length | + |--------| + | 3 | + +--------+ FORALL ------ @@ -70,8 +66,6 @@ FORALL Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``forall(array, function)`` check whether all element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -80,13 +74,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, 2, 3), result = forall(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ EXISTS ------ @@ -94,8 +88,6 @@ EXISTS Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``exists(array, function)`` check whether existing one of element inside array can meet the lambda function. The function should also return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -104,13 +96,13 @@ Return type: BOOLEAN Example:: - PPL> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(-1, -2, 3), result = exists(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | true | - +---------+ + +--------+ + | result | + |--------| + | True | + +--------+ FILTER ------ @@ -118,8 +110,6 @@ FILTER Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``filter(array, function)`` filter the element in the array by the lambda function. The function should return boolean. The lambda function accepts one single input. Argument type: array:ARRAY, function:LAMBDA @@ -128,13 +118,13 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = filter(array, x -> x > 0) | fields result | head 1 fetched rows / total rows = 1/1 - +---------+ - | result | - |---------| - | [1, 3] | - +---------+ + +--------+ + | result | + |--------| + | [1,3] | + +--------+ TRANSFORM --------- @@ -142,8 +132,6 @@ TRANSFORM Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``transform(array, function)`` transform the element of array one by one using lambda. The lambda function can accept one single input or two input. If the lambda accepts two argument, the second one is the index of element in array. Argument type: array:ARRAY, function:LAMBDA @@ -152,21 +140,21 @@ Return type: ARRAY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, x -> x + 2) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [3, 0, 5] | - +------------+ + +---------+ + | result | + |---------| + | [3,0,5] | + +---------+ - PPL> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = transform(array, (x, i) -> x + i) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | [1, -1, 5] | - +------------+ + +----------+ + | result | + |----------| + | [1,-1,5] | + +----------+ REDUCE ------ @@ -174,8 +162,6 @@ REDUCE Description >>>>>>>>>>> -Version: 3.1.0 - Usage: ``reduce(array, acc_base, function, )`` use lambda function to go through all element and interact with acc_base. The lambda function accept two argument accumulator and array element. If add one more reduce_function, will apply reduce_function to accumulator finally. The reduce function accept accumulator as the one argument. Argument type: array:ARRAY, acc_base:ANY, function:LAMBDA, reduce_function:LAMBDA @@ -184,21 +170,21 @@ Return type: ANY Example:: - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 8 | - +------------+ + +--------+ + | result | + |--------| + | 12 | + +--------+ - PPL> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 + os> source=people | eval array = array(1, -2, 3), result = reduce(array, 10, (acc, x) -> acc + x, acc -> acc * 10) | fields result | head 1 fetched rows / total rows = 1/1 - +------------+ - | result | - |------------| - | 80 | - +------------+ + +--------+ + | result | + |--------| + | 120 | + +--------+ MVJOIN ------ @@ -206,8 +192,6 @@ MVJOIN Description >>>>>>>>>>> -Version: 3.3.0 - Usage: mvjoin(array, delimiter) joins string array elements into a single string, separated by the specified delimiter. NULL elements are excluded from the output. Only string arrays are supported. Argument type: array: ARRAY of STRING, delimiter: STRING @@ -216,19 +200,104 @@ Return type: STRING Example:: - PPL> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + os> source=people | eval result = mvjoin(array('a', 'b', 'c'), ',') | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | a,b,c | + +--------+ + + os> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + fetched rows / total rows = 1/1 + +-------------+ + | result | + |-------------| + | Amber, Duke | + +-------------+ + +MVAPPEND +-------- + +Description +>>>>>>>>>>> + +Usage: mvappend(value1, value2, value3...) appends all elements from arguments to create an array. Flattens array arguments and collects all individual elements. Always returns an array or null for consistent type behavior. + +Argument type: value1: ANY, value2: ANY, ... + +Return type: ARRAY + +Example:: + + os> source=people | eval result = mvappend(1, 1, 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,1,3] | + +---------+ + + os> source=people | eval result = mvappend(1, array(2, 3)) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(mvappend(1, 2), 3) | fields result | head 1 + fetched rows / total rows = 1/1 + +---------+ + | result | + |---------| + | [1,2,3] | + +---------+ + + os> source=people | eval result = mvappend(42) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [42] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1), 2) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | [2] | + +--------+ + + os> source=people | eval result = mvappend(nullif(1, 1)) | fields result | head 1 + fetched rows / total rows = 1/1 + +--------+ + | result | + |--------| + | null | + +--------+ + + os> source=people | eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | fields result | head 1 + fetched rows / total rows = 1/1 + +-----------+ + | result | + |-----------| + | [1,2,3,4] | + +-----------+ + + os> source=accounts | eval result = mvappend(firstname, lastname) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------+ - | result | - |------------------------------------| - | "a,b,c" | - +------------------------------------+ + +--------------+ + | result | + |--------------| + | [Amber,Duke] | + +--------------+ - PPL> source=accounts | eval names_array = array(firstname, lastname) | eval result = mvjoin(names_array, ', ') | fields result | head 1 + os> source=people | eval result = mvappend(1, 'text', 2.5) | fields result | head 1 fetched rows / total rows = 1/1 - +------------------------------------------+ - | result | - |------------------------------------------| - | "Amber, Duke" | - +------------------------------------------+ - + +--------------+ + | result | + |--------------| + | [1,text,2.5] | + +--------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java new file mode 100644 index 00000000000..cf84cbe7db6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java @@ -0,0 +1,221 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteMVAppendFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.BANK); + } + + @Test + public void testMvappendWithMultipleElements() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2, 3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3))); + } + + @Test + public void testMvappendWithSingleElement() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(42) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(42))); + } + + @Test + public void testMvappendWithArrayFlattening() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1, 2), arr2 = array(3, 4), result = mvappend(arr1, arr2) | head" + + " 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithMixedArrayAndScalar() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr = array(1, 2), result = mvappend(arr, 3, 4) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2, 3, 4))); + } + + @Test + public void testMvappendWithStringValues() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('hello', 'world') | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("hello", "world"))); + } + + @Test + public void testMvappendWithMixedTypes() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(1, 'text', 2.5) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, "text", 2.5))); + } + + @Test + public void testMvappendWithIntAndDouble() throws IOException { + JSONObject actual = + executeQuery( + source(TEST_INDEX_BANK, "eval result = mvappend(1, 2.5) | head 1 | fields result")); + + System.out.println(actual); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2.5))); + } + + @Test + public void testMvappendWithRealFields() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(firstname, lastname) | head 1 | fields firstname, lastname," + + " result")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("result", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithFieldsAndLiterals() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(age, 'years', 'old') | head 1 | fields age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, "years", "old"))); + } + + @Test + public void testMvappendWithEmptyArray() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval empty_arr = array(), result = mvappend(empty_arr, 1, 2) | head 1 | fields" + + " result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1, 2))); + } + + @Test + public void testMvappendWithNestedArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array('a', 'b'), arr2 = array('c'), arr3 = array('d', 'e'), result =" + + " mvappend(arr1, arr2, arr3) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("a", "b", "c", "d", "e"))); + } + + @Test + public void testMvappendWithNumericArrays() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval arr1 = array(1.5, 2.5), arr2 = array(3.5), result = mvappend(arr1, arr2, 4.5)" + + " | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of(1.5, 2.5, 3.5, 4.5))); + } + + @Test + public void testMvappendInWhereClause() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval combined = mvappend(firstname, lastname) | where array_length(combined) = 2 |" + + " head 1 | fields firstname, lastname, combined")); + + verifySchema( + actual, + schema("firstname", "string"), + schema("lastname", "string"), + schema("combined", "array")); + + verifyDataRows( + actual, + rows("Amber JOHnny", "Duke Willmington", List.of("Amber JOHnny", "Duke Willmington"))); + } + + @Test + public void testMvappendWithComplexExpression() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend(array(age), array(age * 2), age + 10) | head 1 | fields" + + " age, result")); + + verifySchema(actual, schema("age", "int"), schema("result", "array")); + verifyDataRows(actual, rows(32, List.of(32, 64, 42))); + } + + @Test + public void testMvappendWithNull() throws IOException { + JSONObject actual = + executeQuery( + source( + TEST_INDEX_BANK, + "eval result = mvappend('test', nullif(1, 1), 2) | head 1 | fields result")); + + verifySchema(actual, schema("result", "array")); + verifyDataRows(actual, rows(List.of("test", 2))); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 8c4a8376fc4..368c32f250f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -425,6 +425,7 @@ ISBLANK: 'ISBLANK'; // COLLECTION FUNCTIONS ARRAY: 'ARRAY'; ARRAY_LENGTH: 'ARRAY_LENGTH'; +MVAPPEND: 'MVAPPEND'; MVJOIN: 'MVJOIN'; FORALL: 'FORALL'; FILTER: 'FILTER'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 774f897bcaa..ae596259fca 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1019,6 +1019,7 @@ mathematicalFunctionName collectionFunctionName : ARRAY | ARRAY_LENGTH + | MVAPPEND | MVJOIN | FORALL | EXISTS @@ -1560,4 +1561,4 @@ searchableKeyWord | LEFT_HINT | RIGHT_HINT | PERCENTILE_SHORTCUT - ; \ No newline at end of file + ; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 8aa1581e749..f87b50ddf4c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -688,6 +688,13 @@ public void testMvjoin() { anonymize("source=t | eval result=mvjoin(array('a', 'b', 'c'), ',') | fields result")); } + @Test + public void testMvappend() { + assertEquals( + "source=table | eval identifier=mvappend(identifier,***,***) | fields + identifier", + anonymize("source=t | eval result=mvappend(a, 'b', 'c') | fields result")); + } + @Test public void testRexWithOffsetField() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10);