opensearch-project · yuancu · Sep 3, 2025 · Aug 28, 2025
@@ -8,9 +8,12 @@
 - [ ] New functionality includes testing.
   - [ ] All tests pass, including unit test, integration test and doctest
 - [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-  - [ ] New functionality has user manual doc added
-- [ ] Commits are signed per the DCO using --signoff 
+ - [ ] New functionality has javadoc added.
+ - [ ] New functionality has a user manual doc added.
+- [ ] New PPL command [checklist](https://github.com/opensearch-project/sql/blob/main/docs/dev/ppl-commands.md) all confirmed.
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff` or `-s`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
@@ -51,3 +51,6 @@ http-client.env.json
 /doctest/sql-cli/
 /doctest/opensearch-job-scheduler/
 .factorypath
+
+.clinerules/
+.claude/
@@ -90,20 +90,14 @@ Firstly you need to add the following configuration to the JVM used by your IDE.
 License Header
 --------------
 
-Because our code is licensed under Apache 2, you need to add the following license header to all new source code files. To automate this whenever creating new file, you can follow instructions for your IDE::
-
-   /*
-    * Licensed under the Apache License, Version 2.0 (the "License").
-    * You may not use this file except in compliance with the License.
-    * A copy of the License is located at
-    * 
-    *    http://www.apache.org/licenses/LICENSE-2.0
-    * 
-    * or in the "license" file accompanying this file. This file is distributed 
-    * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
-    * express or implied. See the License for the specific language governing 
-    * permissions and limitations under the License.
-    */
+Because our code is licensed under Apache 2, you need to add the following license header to all new source code files. To automate this whenever creating new file, you can follow instructions for your IDE.
+
+.. code:: java
+
+  /*
+   * Copyright OpenSearch Contributors
+   * SPDX-License-Identifier: Apache-2.0
+   */
 
 For example, `here are the instructions for adding copyright profiles in IntelliJ IDEA <https://www.jetbrains.com/help/idea/copyright.html>`__.
 
@@ -211,6 +205,14 @@ Java files are formatted using `Spotless <https://github.com/diffplug/spotless>`
    * - Javadoc format can be maintained by wrapping javadoc with `<pre></pre>` HTML tags
    * - Strings can be formatted on multiple lines with a `+` with the correct indentation for the string.
 
+Development Guidelines
+----------------------
+
+For detailed development documentation, please refer to the `development documentation <docs/dev/index.md>`_. For specific guidance on implementing PPL components, see the following resources:
+
+- `PPL Commands <docs/dev/ppl-commands.md>`_: Guidelines for adding new commands to PPL
+- `PPL Functions <docs/dev/ppl-functions.md>`_: Instructions for implementing and integrating custom functions
+
 Building and Running Tests
 ==========================
 

@@ -77,7 +77,6 @@
 import org.apache.calcite.runtime.Hook;
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.server.CalciteServerStatement;
-import org.apache.calcite.sql.SqlAggFunction;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql2rel.SqlRexConvertletTable;
@@ -90,7 +89,7 @@
 import org.opensearch.sql.calcite.CalcitePlanContext;
 import org.opensearch.sql.calcite.plan.OpenSearchRules;
 import org.opensearch.sql.calcite.plan.Scannable;
-import org.opensearch.sql.calcite.udf.udaf.NullableSqlAvgAggFunction;
+import org.opensearch.sql.expression.function.PPLBuiltinOperators;
 
 /**
  * Calcite Tools Helper. This class is used to create customized: 1. Connection 2. JavaTypeFactory
@@ -186,7 +185,7 @@ public OpenSearchRelBuilder(Context context, RelOptCluster cluster, RelOptSchema
     public AggCall avg(boolean distinct, String alias, RexNode operand) {
       return aggregateCall(
           SqlParserPos.ZERO,
-          AVG_NULLABLE,
+          PPLBuiltinOperators.AVG_NULLABLE,
           distinct,
           false,
           false,
@@ -199,16 +198,6 @@ public AggCall avg(boolean distinct, String alias, RexNode operand) {
     }
   }
 
-  public static final SqlAggFunction AVG_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.AVG);
-  public static final SqlAggFunction STDDEV_POP_NULLABLE =
-      new NullableSqlAvgAggFunction(SqlKind.STDDEV_POP);
-  public static final SqlAggFunction STDDEV_SAMP_NULLABLE =
-      new NullableSqlAvgAggFunction(SqlKind.STDDEV_SAMP);
-  public static final SqlAggFunction VAR_POP_NULLABLE =
-      new NullableSqlAvgAggFunction(SqlKind.VAR_POP);
-  public static final SqlAggFunction VAR_SAMP_NULLABLE =
-      new NullableSqlAvgAggFunction(SqlKind.VAR_SAMP);
-
   public static class OpenSearchPrepareImpl extends CalcitePrepareImpl {
     /**
      * Similar to {@link CalcitePrepareImpl#perform(CalciteServerStatement, FrameworkConfig,

@@ -39,6 +39,10 @@ private PPLOperandTypes() {}
               OperandTypes.NUMERIC.or(
                   OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.CHARACTER)));
 
+  public static final UDFOperandMetadata ANY_OPTIONAL_INTEGER =
+      UDFOperandMetadata.wrap(
+          (CompositeOperandTypeChecker)
+              OperandTypes.ANY.or(OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.INTEGER)));
   public static final UDFOperandMetadata INTEGER_INTEGER =
       UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER_INTEGER);
   public static final UDFOperandMetadata STRING_STRING =
@@ -48,6 +52,12 @@ private PPLOperandTypes() {}
   public static final UDFOperandMetadata STRING_INTEGER =
       UDFOperandMetadata.wrap(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER));
 
+  public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC =
+      UDFOperandMetadata.wrap(
+          (CompositeOperandTypeChecker)
+              OperandTypes.NUMERIC_NUMERIC.or(
+                  OperandTypes.family(
+                      SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)));
   public static final UDFOperandMetadata NUMERIC_NUMERIC_NUMERIC =
       UDFOperandMetadata.wrap(
           OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC));

@@ -5,10 +5,13 @@
 
 package org.opensearch.sql.calcite.utils;
 
+import java.util.List;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.sql.type.ReturnTypes;
 import org.apache.calcite.sql.type.SqlReturnTypeInference;
 import org.apache.calcite.sql.type.SqlTypeTransforms;
+import org.apache.calcite.sql.type.SqlTypeUtil;
 import org.opensearch.sql.data.type.ExprCoreType;
 
 /**
@@ -39,4 +42,17 @@ private PPLReturnTypes() {}
         }
         return UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT;
       };
+  public static SqlReturnTypeInference ARG0_ARRAY =
+      opBinding -> {
+        RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
+
+        // Get argument types
+        List<RelDataType> argTypes = opBinding.collectOperandTypes();
+
+        if (argTypes.isEmpty()) {
+          throw new IllegalArgumentException("Function requires at least one argument.");
+        }
+        RelDataType firstArgType = argTypes.get(0);
+        return SqlTypeUtil.createArrayType(typeFactory, firstArgType, true);
+      };
 }
@@ -17,6 +17,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.TimeZone;
 import java.util.stream.Collectors;
@@ -28,7 +29,6 @@
 import org.apache.calcite.linq4j.tree.Expression;
 import org.apache.calcite.linq4j.tree.Expressions;
 import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.schema.impl.AggregateFunctionImpl;
@@ -94,14 +94,15 @@ public class UserDefinedFunctionUtils {
   public static SqlUserDefinedAggFunction createUserDefinedAggFunction(
       Class<? extends UserDefinedAggFunction<?>> udafClass,
       String functionName,
-      SqlReturnTypeInference returnType) {
+      SqlReturnTypeInference returnType,
+      @Nullable UDFOperandMetadata operandMetadata) {
     return new SqlUserDefinedAggFunction(
         new SqlIdentifier(functionName, SqlParserPos.ZERO),
         SqlKind.OTHER_FUNCTION,
         returnType,
         null,
-        null,
-        AggregateFunctionImpl.create(udafClass),
+        operandMetadata,
+        Objects.requireNonNull(AggregateFunctionImpl.create(udafClass)),
         false,
         false,
         Optionality.FORBIDDEN);
@@ -126,45 +127,6 @@ public static RelBuilder.AggCall makeAggregateCall(
     return relBuilder.aggregateCall(aggFunction, addArgList);
   }
 
-  /**
-   * Creates and registers a User Defined Aggregate Function (UDAF) and returns an AggCall that can
-   * be used in query plans.
-   *
-   * @param udafClass The class implementing the aggregate function behavior
-   * @param functionName The name of the aggregate function
-   * @param returnType The return type inference for determining the result type
-   * @param fields The primary fields to aggregate
-   * @param argList Additional arguments for the aggregate function
-   * @param relBuilder The RelBuilder instance used for building relational expressions
-   * @return An AggCall object representing the aggregate function call
-   */
-  public static RelBuilder.AggCall createAggregateFunction(
-      Class<? extends UserDefinedAggFunction<?>> udafClass,
-      String functionName,
-      SqlReturnTypeInference returnType,
-      List<RexNode> fields,
-      List<RexNode> argList,
-      RelBuilder relBuilder) {
-    SqlUserDefinedAggFunction udaf =
-        createUserDefinedAggFunction(udafClass, functionName, returnType);
-    return makeAggregateCall(udaf, fields, argList, relBuilder);
-  }
-
-  public static SqlReturnTypeInference getReturnTypeInferenceForArray() {
-    return opBinding -> {
-      RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
-
-      // Get argument types
-      List<RelDataType> argTypes = opBinding.collectOperandTypes();
-
-      if (argTypes.isEmpty()) {
-        throw new IllegalArgumentException("Function requires at least one argument.");
-      }
-      RelDataType firstArgType = argTypes.get(0);
-      return createArrayType(typeFactory, firstArgType, true);
-    };
-  }
-
   public static SqlTypeName convertRelDataTypeToSqlTypeName(RelDataType type) {
     if (type instanceof AbstractExprRelDataType) {
       AbstractExprRelDataType<?> exprType = (AbstractExprRelDataType<?>) type;
@@ -276,6 +238,38 @@ public UDFOperandMetadata getOperandMetadata() {
     };
   }
 
+  /**
+   * Adapts a method from the v2 implementation whose parameters include a {@link
+   * FunctionProperties} at the beginning to a Calcite-compatible UserDefinedFunctionBuilder.
+   */
+  public static ImplementorUDF adaptExprMethodWithPropertiesToUDF(
+      java.lang.reflect.Type type,
+      String methodName,
+      SqlReturnTypeInference returnTypeInference,
+      NullPolicy nullPolicy,
+      UDFOperandMetadata operandMetadata) {
+    NotNullImplementor implementor =
+        (translator, call, translatedOperands) -> {
+          List<Expression> operands =
+              convertToExprValues(
+                  translatedOperands, call.getOperands().stream().map(RexNode::getType).collect(Collectors.toList()));
+          List<Expression> operandsWithProperties = prependFunctionProperties(operands, translator);
+          Expression exprResult = Expressions.call(type, methodName, operandsWithProperties);
+          return Expressions.call(exprResult, "valueForCalcite");
+        };
+    return new ImplementorUDF(implementor, nullPolicy) {
+      @Override
+      public SqlReturnTypeInference getReturnTypeInference() {
+        return returnTypeInference;
+      }
+
+      @Override
+      public UDFOperandMetadata getOperandMetadata() {
+        return operandMetadata;
+      }
+    };
+  }
+
   /**
    * Adapt a static math function (e.g., Math.expm1, Math.rint) to a UserDefinedFunctionBuilder.
    * This method generates a Calcite-compatible UDF by boxing the operand, converting it to a
@@ -326,32 +320,4 @@ public static List<Expression> prependFunctionProperties(
     operandsWithProperties.add(0, properties);
     return Collections.unmodifiableList(operandsWithProperties);
   }
-
-  public static ImplementorUDF adaptExprMethodWithPropertiesToUDF(
-      java.lang.reflect.Type type,
-      String methodName,
-      SqlReturnTypeInference returnTypeInference,
-      NullPolicy nullPolicy,
-      UDFOperandMetadata operandMetadata) {
-    NotNullImplementor implementor =
-        (translator, call, translatedOperands) -> {
-          List<Expression> operands =
-              convertToExprValues(
-                  translatedOperands, call.getOperands().stream().map(RexNode::getType).collect(Collectors.toList()));
-          List<Expression> operandsWithProperties = prependFunctionProperties(operands, translator);
-          Expression exprResult = Expressions.call(type, methodName, operandsWithProperties);
-          return Expressions.call(exprResult, "valueForCalcite");
-        };
-    return new ImplementorUDF(implementor, nullPolicy) {
-      @Override
-      public SqlReturnTypeInference getReturnTypeInference() {
-        return returnTypeInference;
-      }
-
-      @Override
-      public UDFOperandMetadata getOperandMetadata() {
-        return operandMetadata;
-      }
-    };
-  }
 }
@@ -8,6 +8,7 @@
 import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodToUDF;
 import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodWithPropertiesToUDF;
 import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptMathFunctionToUDF;
+import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createUserDefinedAggFunction;
 
 import com.google.common.base.Suppliers;
 import java.lang.reflect.InvocationTargetException;
@@ -21,11 +22,17 @@
 import org.apache.calcite.avatica.util.TimeUnit;
 import org.apache.calcite.linq4j.tree.Expression;
 import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.type.ReturnTypes;
 import org.apache.calcite.sql.type.SqlTypeTransforms;
 import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable;
 import org.apache.calcite.util.BuiltInMethod;
+import org.opensearch.sql.calcite.udf.udaf.LogPatternAggFunction;
+import org.opensearch.sql.calcite.udf.udaf.NullableSqlAvgAggFunction;
+import org.opensearch.sql.calcite.udf.udaf.PercentileApproxFunction;
+import org.opensearch.sql.calcite.udf.udaf.TakeAggFunction;
 import org.opensearch.sql.calcite.utils.PPLOperandTypes;
 import org.opensearch.sql.calcite.utils.PPLReturnTypes;
 import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
@@ -397,6 +404,35 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
   public static final SqlOperator ENHANCED_COALESCE =
       new EnhancedCoalesceFunction().toUDF("COALESCE");
 
+      // Aggregation functions
+  public static final SqlAggFunction AVG_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.AVG);
+  public static final SqlAggFunction STDDEV_POP_NULLABLE =
+      new NullableSqlAvgAggFunction(SqlKind.STDDEV_POP);
+  public static final SqlAggFunction STDDEV_SAMP_NULLABLE =
+      new NullableSqlAvgAggFunction(SqlKind.STDDEV_SAMP);
+  public static final SqlAggFunction VAR_POP_NULLABLE =
+      new NullableSqlAvgAggFunction(SqlKind.VAR_POP);
+  public static final SqlAggFunction VAR_SAMP_NULLABLE =
+      new NullableSqlAvgAggFunction(SqlKind.VAR_SAMP);
+  public static final SqlAggFunction TAKE =
+      createUserDefinedAggFunction(
+          TakeAggFunction.class,
+          "TAKE",
+          PPLReturnTypes.ARG0_ARRAY,
+          PPLOperandTypes.ANY_OPTIONAL_INTEGER);
+  public static final SqlAggFunction PERCENTILE_APPROX =
+      createUserDefinedAggFunction(
+          PercentileApproxFunction.class,
+          "percentile_approx",
+          ReturnTypes.ARG0_FORCE_NULLABLE,
+          PPLOperandTypes.NUMERIC_NUMERIC_OPTIONAL_NUMERIC);
+  public static final SqlAggFunction INTERNAL_PATTERN =
+      createUserDefinedAggFunction(
+          LogPatternAggFunction.class,
+          "pattern",
+          ReturnTypes.explicit(UserDefinedFunctionUtils.nullablePatternAggList),
+          null);
+
   /**
    * Returns the PPL specific operator table, creating it if necessary.
    *