From bccf9f268313d1ea330e962b7639e3fdc0c13ded Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 00:26:38 -0700 Subject: [PATCH 01/13] xy series implmentation Signed-off-by: Asif Bashar --- .../org/opensearch/sql/analysis/Analyzer.java | 6 + .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/tree/Xyseries.java | 64 ++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 118 +++++++++++++ docs/user/ppl/cmd/xyseries.md | 155 ++++++++++++++++++ .../sql/calcite/remote/CalciteExplainIT.java | 42 +++++ .../sql/ppl/NewAddedCommandsIT.java | 83 +++++++++- .../sql/security/CrossClusterSearchIT.java | 25 +++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 17 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 34 ++++ .../sql/ppl/parser/AstBuilderTest.java | 73 +++++++++ 12 files changed, 618 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java create mode 100644 docs/user/ppl/cmd/xyseries.md diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 701d1545b76..916cc00bc4a 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -112,6 +112,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -842,6 +843,11 @@ public LogicalPlan visitChart(Chart node, AnalysisContext context) { throw getOnlyForCalciteException("Chart"); } + @Override + public LogicalPlan visitXyseries(Xyseries node, AnalysisContext context) { + throw getOnlyForCalciteException("Xyseries"); + } + @Override public LogicalPlan visitWindow(Window node, AnalysisContext context) { throw getOnlyForCalciteException("Window"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index acb6e105661..266f8f46f7d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -100,6 +100,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; /** AST nodes visitor Defines the traverse path. */ public abstract class AbstractNodeVisitor { @@ -520,4 +521,8 @@ public T visitMvExpand(MvExpand node, C context) { public T visitGraphLookup(GraphLookup node, C context) { return visitChildren(node, context); } + + public T visitXyseries(Xyseries node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java new file mode 100644 index 00000000000..84fb3019e0f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node representing the xyseries command. Converts row-oriented grouped results into a wide + * table where one field is the X axis (row key), one field provides pivot values for column naming, + * and one or more data fields fill the pivoted cells. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Xyseries extends UnresolvedPlan { + + /** The x-axis field (row key in output). */ + private final UnresolvedExpression xField; + + /** The y-name field whose values become part of the output column names. */ + private final UnresolvedExpression yNameField; + + /** Explicit pivot values from the IN (...) clause. */ + private final List pivotValues; + + /** One or more y-data fields whose values fill the pivoted cells. */ + private final List yDataFields; + + /** Separator between y-data-field name and pivot value in column names. Default ":". */ + private final String separator; + + /** Optional format template for output column names using $AGG$ and $VAL$ placeholders. */ + private final String format; + + @Setter private UnresolvedPlan child; + + @Override + public Xyseries attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitXyseries(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 94c40e5adb2..1b1a77428b0 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -171,6 +172,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.AliasFieldsWrappable; import org.opensearch.sql.calcite.plan.HighlightPushDown; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -4071,6 +4073,122 @@ static ChartConfig fromArguments(ArgumentMap argMap) { } } + @Override + public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { + visitChildren(node, context); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + // Resolve x-field and y-name-field names + String xFieldName = resolveFieldName(node.getXField()); + String yNameFieldName = resolveFieldName(node.getYNameField()); + + // Resolve y-data field names + List yDataFieldNames = + node.getYDataFields().stream().map(this::resolveFieldName).collect(Collectors.toList()); + + List pivotValues = node.getPivotValues() != null ? node.getPivotValues() : List.of(); + String separator = node.getSeparator(); + String format = node.getFormat(); + + // Build the pivot axis - cast to VARCHAR if needed for string comparison + RelDataType yNameType = yNameRef.getType(); + RexNode axis; + if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { + if (!SqlTypeUtil.isAtomic(yNameType)) { + throw new IllegalArgumentException( + "xyseries y-name-field must be a scalar type, got: " + yNameType.getSqlTypeName()); + } + RelDataType varchar = + rx.getTypeFactory() + .createTypeWithNullability( + rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), true); + axis = rx.makeCast(varchar, yNameRef, true); + } else { + axis = yNameRef; + } + + // Build aggregate calls - MAX for each y-data field + List aggCalls = + yDataFieldNames.stream() + .map(name -> b.max(b.field(name)).as(name)) + .collect(Collectors.toList()); + + // Build pivot value entries: alias -> [literal(value)] + // LinkedHashMap preserves insertion order for deterministic column ordering + LinkedHashMap> pivotValueMap = new LinkedHashMap<>(); + for (String val : pivotValues) { + pivotValueMap.put(val, ImmutableList.of(b.literal(val))); + } + + // Execute pivot: decomposes into GROUP BY x-field with FILTER-based aggregation + // Produces columns: x-field, {val1}_{agg1}, {val1}_{agg2}, {val2}_{agg1}, ... + b.pivot( + b.groupKey(b.field(xFieldName)), + aggCalls, + ImmutableList.of(axis), + pivotValueMap.entrySet()); + + // Pivot produces value-first column ordering: val1_agg1, val1_agg2, val2_agg1, ... + // Reorder to agg-first and apply custom column naming: agg1: val1, agg1: val2, ... + List reorderProjections = new ArrayList<>(); + List reorderNames = new ArrayList<>(); + + reorderProjections.add(b.field(xFieldName)); + reorderNames.add(xFieldName); + + for (String aggName : yDataFieldNames) { + for (String pivotVal : pivotValues) { + // Reference pivot output column by its generated name: {value}_{agg} + String pivotColName = pivotVal + "_" + aggName; + try { + reorderProjections.add(b.field(pivotColName)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "xyseries: expected pivot output column '" + pivotColName + "' not found", e); + } + reorderNames.add(generateColumnName(aggName, pivotVal, separator, format)); + } + } + // Fail fast with a clear message if the naming scheme produced collisions + // (e.g. a format template that omits $VAL$ or $AGG$ with multiple series). + Set seenNames = new HashSet<>(); + for (String name : reorderNames) { + if (!seenNames.add(name)) { + throw new IllegalArgumentException( + "xyseries produced duplicate output column name '" + + name + + "'. Use a format template containing both $AGG$ and $VAL$ so column names" + + " are unique."); + } + } + b.project(reorderProjections, reorderNames, true); + + // Order by x-field + b.sort(b.field(0)); + + return b.peek(); + } + + private String resolveFieldName(UnresolvedExpression expr) { + if (expr instanceof Field) { + return ((Field) expr).getField().toString(); + } + if (expr instanceof Alias) { + return ((Alias) expr).getName(); + } + return expr.toString(); + } + + private String generateColumnName( + String yDataFieldName, String pivotValue, String separator, String format) { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + return yDataFieldName + separator + pivotValue; + } + @Override public RelNode visitTrendline(Trendline node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/docs/user/ppl/cmd/xyseries.md b/docs/user/ppl/cmd/xyseries.md new file mode 100644 index 00000000000..a81ea7d604c --- /dev/null +++ b/docs/user/ppl/cmd/xyseries.md @@ -0,0 +1,155 @@ +# xyseries + +## Description + +The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. + +## Syntax + +```syntax +xyseries [sep=] [format=] in (, , ...) [, , ...] +``` + +## Parameters + +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | The field used as the row key in the output. Results are grouped and sorted by this field. | N/A | +| `` | Required | The field whose values are used to generate output column names. Only the values listed in the `in` clause are pivoted into columns. | N/A | +| `in (, , ...)` | Required | Explicit list of pivot values to select from ``. Each value generates one output column per ``. Values must be quoted strings. | N/A | +| `` | Required (at least one) | One or more fields containing the data to pivot. If multiple fields are specified, separate them with commas. | N/A | +| `sep` | Optional | Separator between the `` name and the pivot value in output column names. Ignored if `format` is specified. | `": "` | +| `format` | Optional | Naming template for output column names. Use `$AGG$` as a placeholder for the `` name and `$VAL$` as a placeholder for the pivot value. When specified, overrides `sep`. | N/A | + +## Notes + +The following considerations apply when using the `xyseries` command: + +* The `xyseries` command is typically used after a `stats` command that groups results by both the `` and ``. +* Output column names follow the pattern `` by default (for example, `host_cnt: 200`). Use the `format` option to customize this pattern. +* When a pivot value has no matching data for a given `` row, the output cell is `null`. +* The `` values are compared as strings. Non-string fields are cast to string automatically. +* Results are sorted by `` in ascending order. +* This command requires the Calcite engine to be enabled (`plugins.calcite.enabled: true`). + +## Example 1: Basic xyseries with a single data field + +This example pivots HTTP response codes into columns for a count of hosts per URL: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | +|--------+---------------+---------------+---------------| +| /page1 | 3 | 1 | null | +| /page2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+ +``` + +## Example 2: Multiple data fields + +This example pivots multiple aggregated fields at once: + +```ppl +source=weblogs +| stats count(host) as host_cnt, count(method) as method_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt, method_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | method_cnt: 200 | method_cnt: 404 | method_cnt: 500 | +|--------+---------------+---------------+---------------+------------------+------------------+------------------| +| /page1 | 3 | 1 | null | 3 | 1 | null | +| /page2 | 5 | null | 2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +``` + +## Example 3: Custom separator + +This example uses a custom separator between the data field name and pivot value in column names: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries sep="-" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | host_cnt-200 | host_cnt-404 | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 4: Format template + +This example uses a format template to customize output column names. `$VAL$` is replaced with the pivot value and `$AGG$` is replaced with the data field name: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries format="$VAL$_$AGG$" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | 200_host_cnt | 404_host_cnt | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 5: Partial pivot values + +When only a subset of values is specified in the `in` clause, rows with unmatched `` values produce `null` for the corresponding `` rows: + +```ppl +source=accounts +| stats avg(balance) as avg_balance by gender, state +| xyseries state gender in ("F") avg_balance +``` + +The query returns the following results: + +```text +fetched rows / total rows = 7/7 ++-------+-----------------+ +| state | avg_balance: F | +|-------+-----------------| +| IL | null | +| IN | 48086.0 | +| MD | null | +| PA | 40540.0 | +| TN | null | +| VA | 32838.0 | +| WA | null | ++-------+-----------------+ +``` + +## Limitations + +The `xyseries` command has the following limitations: + +* Pivot values must be explicitly provided in the `in` clause. Dynamic pivot (deriving column names from data at runtime) is not supported. +* This command is only available when the Calcite engine is enabled. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index b78a71e534c..04dc2b0e74b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -3028,6 +3028,48 @@ public void testHighlightOsdObjectFormatExplain() throws IOException { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void testXyseriesExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesMultipleDataFieldsExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance, count() as cnt by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance, cnt", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_multiple_data_fields.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesWithFormatExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state | xyseries" + + " format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_with_format.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void testExplainConsecutiveSortsAfterAggIssue5125() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 6b5ac0d4302..857841263f8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; import java.io.IOException; +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -551,17 +552,89 @@ public void testMvExpandInvalidLimitNegative() throws IOException { assertThat(error.getString("type"), equalTo("SyntaxCheckException")); } } - @Test public void testUnionUnsupportedInV2() throws IOException { + JSONObject result; + try { + result = + executeQuery( + String.format( + "| union [search source=%s | where age < 30] [search source=%s | where age >=" + + " 30]", + TEST_INDEX_BANK, TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + @Test + public void testXyseriesCommand() throws IOException { + JSONObject result; try { result = executeQuery( - String.format( - "| union [search source=%s | where age < 30] [search source=%s | where age >=" - + " 30]", - TEST_INDEX_BANK, TEST_INDEX_BANK)); + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandMultipleDataFields() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance, count() as cnt by" + + " gender, state | xyseries state gender in (\"F\", \"M\") avg_balance," + + " cnt", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithSep() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries sep=\"-\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithFormat() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\")" + + " avg_balance", + TEST_INDEX_BANK))); + } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index 0029921c1fc..8673c4221b3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -237,4 +237,29 @@ public void testCrossClusterConvertWithAlias() throws IOException { disableCalcite(); } + + @Test + public void testCrossClusterXyseries() throws IOException { + enableCalcite(); + + JSONObject result = + executeQuery( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in ('F', 'M') avg_balance", + TEST_INDEX_BANK_REMOTE)); + verifyColumn( + result, columnName("state"), columnName("avg_balance: F"), columnName("avg_balance: M")); + verifyDataRows( + result, + rows("IL", null, 39225.0), + rows("IN", 48086.0, null), + rows("MD", null, 4180.0), + rows("PA", 40540.0, null), + rows("TN", null, 5686.0), + rows("VA", 32838.0, null), + rows("WA", null, 16418.0)); + + disableCalcite(); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index b26751ad61b..1bc59116ed8 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -60,6 +60,7 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +XYSERIES: 'XYSERIES'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index eeaed6daf52..7d09064194e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -99,6 +99,7 @@ commands | fieldformatCommand | nomvCommand | graphLookupCommand + | xyseriesCommand | unionCommand | timewrapCommand ; @@ -153,6 +154,7 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | XYSERIES | TIMEWRAP | MAKERESULTS ; @@ -764,6 +766,19 @@ graphLookupArgs | (FILTER EQUAL LT_PRTHS logicalExpression RT_PRTHS) ; +xyseriesCommand + : XYSERIES xyseriesOption* xField = fieldExpression yNameField = fieldExpression IN LT_PRTHS xyseriesPivotValues RT_PRTHS yDataFields = fieldList + ; + +xyseriesOption + : SEP EQUAL sep = stringLiteral + | FORMAT EQUAL format = stringLiteral + ; + +xyseriesPivotValues + : stringLiteral (COMMA stringLiteral)* + ; + // clauses fromClause : SOURCE EQUAL tableOrSubqueryClause @@ -1856,4 +1871,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + | SEP + | FORMAT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e87264909c8..efdbf26a205 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -129,6 +129,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -1837,4 +1838,37 @@ public UnresolvedPlan visitGraphLookupCommand(OpenSearchPPLParser.GraphLookupCom .filter(filter) .build(); } + + /** Xyseries command. */ + @Override + public UnresolvedPlan visitXyseriesCommand(OpenSearchPPLParser.XyseriesCommandContext ctx) { + UnresolvedExpression xField = internalVisitExpression(ctx.xField); + UnresolvedExpression yNameField = internalVisitExpression(ctx.yNameField); + + // Parse pivot values from IN (...) clause + List pivotValues = + ctx.xyseriesPivotValues().stringLiteral().stream() + .map(s -> StringUtils.unquoteText(s.getText())) + .distinct() + .collect(Collectors.toList()); + + // Parse y-data fields + List yDataFields = + ctx.yDataFields.fieldExpression().stream() + .map(this::internalVisitExpression) + .collect(Collectors.toList()); + + // Parse options + String separator = ": "; + String format = null; + for (OpenSearchPPLParser.XyseriesOptionContext optCtx : ctx.xyseriesOption()) { + if (optCtx.SEP() != null) { + separator = StringUtils.unquoteText(optCtx.sep.getText()); + } else if (optCtx.FORMAT() != null) { + format = StringUtils.unquoteText(optCtx.format.getText()); + } + } + + return new Xyseries(xField, yNameField, pivotValues, yDataFields, separator, format); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5f45a96f8c..0650b4d1e4c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -83,6 +83,7 @@ import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.exception.SemanticCheckException; @@ -1827,6 +1828,78 @@ public void testMalformedPipeProducesSyntaxError() { plan("source=t | invalidCmd |"); } + // Xyseries tests + + @Test + public void testXyseriesCommandBasic() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries url response in (\"200\", \"404\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandMultipleDataFields() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt"), field("method_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries url response in (\"200\", \"404\") host_cnt, method_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSep() { + Xyseries expected = + new Xyseries( + field("url"), field("response"), List.of("200"), List.of(field("host_cnt")), "-", null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries sep=\"-\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200"), + List.of(field("host_cnt")), + ": ", + "$VAL$+$AGG$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries format=\"$VAL$+$AGG$\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSepAndFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + "-", + "$AGG$_$VAL$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries sep=\"-\" format=\"$AGG$_$VAL$\" url response in (\"200\", \"404\")" + + " host_cnt", + expected); + } + @Test public void testUnionWithSubsearches() { plan("| union [search source=t1 | where age > 30] " + "[search source=t2 | where age < 20]"); From 8ca6a57132444fd448efaa9c2c8efc4e6580229b Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 00:39:09 -0700 Subject: [PATCH 02/13] xy series implmentation Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 1bc59116ed8..a9a2cde2240 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -61,6 +61,8 @@ ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; XYSERIES: 'XYSERIES'; +SEP: 'SEP'; +FORMAT: 'FORMAT'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; From 7823512a492686cbd9810849f4d63219a8ee3ab4 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 00:45:59 -0700 Subject: [PATCH 03/13] xy series implmentation Signed-off-by: Asif Bashar --- .../sql/calcite/CalciteRelNodeVisitor.java | 2 +- .../org/opensearch/sql/ppl/NewAddedCommandsIT.java | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 1b1a77428b0..d9b20599146 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -4096,7 +4096,7 @@ public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { RelDataType yNameType = yNameRef.getType(); RexNode axis; if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { - if (!SqlTypeUtil.isAtomic(yNameType)) { + if (!SqlTypeUtil.isAtomic(yNameType)) { throw new IllegalArgumentException( "xyseries y-name-field must be a scalar type, got: " + yNameType.getSqlTypeName()); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 857841263f8..a32dd9fb990 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -552,21 +552,23 @@ public void testMvExpandInvalidLimitNegative() throws IOException { assertThat(error.getString("type"), equalTo("SyntaxCheckException")); } } + @Test public void testUnionUnsupportedInV2() throws IOException { JSONObject result; try { result = - executeQuery( - String.format( - "| union [search source=%s | where age < 30] [search source=%s | where age >=" - + " 30]", - TEST_INDEX_BANK, TEST_INDEX_BANK)); + executeQuery( + String.format( + "| union [search source=%s | where age < 30] [search source=%s | where age >=" + + " 30]", + TEST_INDEX_BANK, TEST_INDEX_BANK)); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); } verifyQuery(result); } + @Test public void testXyseriesCommand() throws IOException { From 48da2733202080b2de66245fa2c41fdfc1a03283 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 00:54:44 -0700 Subject: [PATCH 04/13] xy series implmentation Signed-off-by: Asif Bashar --- .../sql/calcite/CalciteRelNodeVisitor.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index d9b20599146..78dd3d77293 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -4148,7 +4148,8 @@ public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { throw new IllegalStateException( "xyseries: expected pivot output column '" + pivotColName + "' not found", e); } - reorderNames.add(generateColumnName(aggName, pivotVal, separator, format)); + reorderNames.add( + generateColumnName(aggName, pivotVal, separator, format, yDataFieldNames.size())); } } // Fail fast with a clear message if the naming scheme produced collisions @@ -4182,11 +4183,19 @@ private String resolveFieldName(UnresolvedExpression expr) { } private String generateColumnName( - String yDataFieldName, String pivotValue, String separator, String format) { - if (format != null) { - return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + String yDataFieldName, + String pivotValue, + String separator, + String format, + int yDataFieldCount) { + if (yDataFieldCount == 1) { + return pivotValue; + } else { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + return yDataFieldName + separator + pivotValue; } - return yDataFieldName + separator + pivotValue; } @Override From 9057493ff86fb79c80c14558e0a39abdb01dfd2c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 01:00:54 -0700 Subject: [PATCH 05/13] xy series implmentation Signed-off-by: Asif Bashar --- .../sql/calcite/CalciteRelNodeVisitor.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 78dd3d77293..4bfeb123936 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -4148,8 +4148,8 @@ public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { throw new IllegalStateException( "xyseries: expected pivot output column '" + pivotColName + "' not found", e); } - reorderNames.add( - generateColumnName(aggName, pivotVal, separator, format, yDataFieldNames.size())); + boolean singleDataField = yDataFieldNames.size() == 1; + reorderNames.add(generateColumnName(aggName, pivotVal, separator, format, singleDataField)); } } // Fail fast with a clear message if the naming scheme produced collisions @@ -4187,15 +4187,14 @@ private String generateColumnName( String pivotValue, String separator, String format, - int yDataFieldCount) { - if (yDataFieldCount == 1) { + boolean singleDataField) { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + if (singleDataField) { return pivotValue; - } else { - if (format != null) { - return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); - } - return yDataFieldName + separator + pivotValue; } + return yDataFieldName + separator + pivotValue; } @Override From 033181ca1dae2adfcd386455e6fd73121251f1db Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 01:04:04 -0700 Subject: [PATCH 06/13] index.md updated Signed-off-by: Asif Bashar --- docs/user/ppl/index.md | 113 +++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 3eed4181c61..feabd6fc9d0 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -35,62 +35,63 @@ source=accounts The following commands are available in PPL: **Note:** Experimental commands are ready for use, but specific parameters may change based on feedback. -| Command Name | Version Introduced | Current Status | Command Description | -| --- | --- | --- | --- | -| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | -| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | -| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | -| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | -| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | -| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | -| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | -| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | -| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | -| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | -| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | -| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | -| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | -| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | -| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | -| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | -| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | -| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | -| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | -| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | -| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | -| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | -| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | -| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | -| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | -| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | -| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | -| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | -| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | -| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | -| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | -| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | -| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | -| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | -| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | -| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | -| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | -| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | -| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | -| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | -| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | -| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | -| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | -| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | -| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | -| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | -| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | -| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | -| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | -| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | -| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| - - - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting +| Command Name | Version Introduced | Current Status | Command Description | +|----------------------------------------------------|--------------------|--------------------------| --- | +| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | +| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | +| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | +| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | +| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | +| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | +| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | +| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | +| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | +| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | +| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | +| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | +| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | +| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | +| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | +| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | +| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | +| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | +| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | +| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | +| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | +| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | +| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | +| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | +| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | +| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | +| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | +| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | +| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | +| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | +| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | +| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | +| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | +| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | +| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | +| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | +| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | +| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | +| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | +| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | +| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | +| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | +| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | +| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | +| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | +| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | +| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | +| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | +| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | +| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | +| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| +| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included.| + +- [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** - [Aggregation Functions](functions/aggregations.md) - [Collection Functions](functions/collection.md) From c7e3b31c0099e8b85f2fdb3b34097ffec7117f5c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 01:28:37 -0700 Subject: [PATCH 07/13] removed duplicate format added during merging Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 - ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 - 2 files changed, 2 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index a9a2cde2240..fb072ae134f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -62,7 +62,6 @@ ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; XYSERIES: 'XYSERIES'; SEP: 'SEP'; -FORMAT: 'FORMAT'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7d09064194e..a23a314537d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1872,5 +1872,4 @@ searchableKeyWord | DEPTH_FIELD | EDGE | SEP - | FORMAT ; From f9cc584b30e4e5577283d60312717c4a417c1257 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 01:44:53 -0700 Subject: [PATCH 08/13] fix compile issue Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 4bfeb123936..9995895bfa1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -4093,6 +4093,7 @@ public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { String format = node.getFormat(); // Build the pivot axis - cast to VARCHAR if needed for string comparison + RexNode yNameRef = b.field(yNameFieldName); RelDataType yNameType = yNameRef.getType(); RexNode axis; if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { From 48ec5c1e33902368617e3a402ea261cc52cc6453 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 02:22:52 -0700 Subject: [PATCH 09/13] fix test failure Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/security/CrossClusterSearchIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index 8673c4221b3..86e98c1523f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -248,8 +248,7 @@ public void testCrossClusterXyseries() throws IOException { "search source=%s | stats avg(balance) as avg_balance by gender, state" + " | xyseries state gender in ('F', 'M') avg_balance", TEST_INDEX_BANK_REMOTE)); - verifyColumn( - result, columnName("state"), columnName("avg_balance: F"), columnName("avg_balance: M")); + verifyColumn(result, columnName("state"), columnName("F"), columnName("M")); verifyDataRows( result, rows("IL", null, 39225.0), From ae0ee9f6965845bcdef87015696668da1e9c00d7 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 08:52:18 -0700 Subject: [PATCH 10/13] added missing explain expection output files missed during merge conflict. Signed-off-by: Asif Bashar --- .../expectedOutput/calcite/explain_xyseries.yaml | 15 +++++++++++++++ .../explain_xyseries_multiple_data_fields.yaml | 16 ++++++++++++++++ .../calcite/explain_xyseries_with_format.yaml | 15 +++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml new file mode 100644 index 00000000000..5fc1db286ed --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml new file mode 100644 index 00000000000..a8419925e99 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$3], cnt: F=[$2], cnt: M=[$4]) + LogicalAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + LogicalProject(avg_balance=[$2], cnt=[$3], state=[$1], $f4=[IS TRUE(=($0, 'F'))], $f5=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)], cnt=[COUNT()]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}], avg_balance: M=[$t3], cnt: F=[$t2], cnt: M=[$t4]) + EnumerableAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=['F'], expr#5=[=($t0, $t4)], expr#6=[IS TRUE($t5)], expr#7=['M'], expr#8=[=($t0, $t7)], expr#9=[IS TRUE($t8)], avg_balance=[$t2], cnt=[$t3], state=[$t1], $f4=[$t6], $f5=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml new file mode 100644 index 00000000000..eaf89c53c2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F_avg_balance=[$1], M_avg_balance=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) From 09cfc39e82b022564cfcb5dbe517e9a5de8f70d3 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 09:29:32 -0700 Subject: [PATCH 11/13] removed extra formatting changes Signed-off-by: Asif Bashar --- docs/user/ppl/index.md | 114 ++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index feabd6fc9d0..f83111cfb5a 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -31,65 +31,65 @@ source=accounts - [Identifiers](general/identifiers.md) - [Data Types](general/datatypes.md) * **Commands** - + The following commands are available in PPL: **Note:** Experimental commands are ready for use, but specific parameters may change based on feedback. - -| Command Name | Version Introduced | Current Status | Command Description | -|----------------------------------------------------|--------------------|--------------------------| --- | -| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | -| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | -| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | -| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | -| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | -| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | -| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | -| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | -| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | -| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | -| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | -| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | -| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | -| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | -| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | -| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | -| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | -| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | -| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | -| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | -| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | -| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | -| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | -| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | -| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | -| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | -| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | -| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | -| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | -| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | -| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | -| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | -| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | -| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | -| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | -| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | -| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | -| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | -| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | -| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | -| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | -| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | -| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | -| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | -| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | -| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | -| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | -| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | -| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | -| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | -| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| -| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included.| + +| Command Name | Version Introduced | Current Status | Command Description | +| --- | --- | --- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | +| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | +| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | +| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | +| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | +| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | +| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | +| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | +| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | +| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | +| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | +| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | +| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | +| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | +| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | +| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | +| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | +| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | +| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | +| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | +| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | +| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | +| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | +| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | +| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | +| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | +| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | +| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | +| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | +| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | +| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | +| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | +| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | +| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | +| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | +| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | +| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | +| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | +| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | +| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | +| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | +| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | +| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | +| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | +| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | +| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | +| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | +| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | +| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | +| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | +| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm. | +| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** From ea1ea99bd2c0f88a38359ab1fe39bbb683e233cd Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 09:40:35 -0700 Subject: [PATCH 12/13] removed extra formatting changes Signed-off-by: Asif Bashar --- docs/user/ppl/index.md | 116 ++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index f83111cfb5a..067d5f524fd 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -31,67 +31,67 @@ source=accounts - [Identifiers](general/identifiers.md) - [Data Types](general/datatypes.md) * **Commands** - + The following commands are available in PPL: **Note:** Experimental commands are ready for use, but specific parameters may change based on feedback. + +| Command Name | Version Introduced | Current Status | Command Description | +| --- | --- | --- | --- | +| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | +| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | +| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | +| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | +| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | +| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | +| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | +| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | +| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | +| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | +| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | +| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | +| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | +| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | +| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | +| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | +| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | +| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | +| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | +| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | +| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | +| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | +| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | +| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | +| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | +| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | +| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | +| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | +| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | +| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | +| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | +| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | +| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | +| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | +| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | +| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | +| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | +| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | +| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | +| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | +| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | +| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | +| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | +| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | +| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | +| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | +| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | +| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | +| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | +| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | +| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| +| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | -| Command Name | Version Introduced | Current Status | Command Description | -| --- | --- | --- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [search command](cmd/search.md) | 1.0 | stable (since 1.0) | Retrieve documents from the index. | -| [where command](cmd/where.md) | 1.0 | stable (since 1.0) | Filter the search result using boolean expressions. | -| [subquery command](cmd/subquery.md) | 3.0 | experimental (since 3.0) | Embed one PPL query inside another for complex filtering and data retrieval operations. | -| [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | -| [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | -| [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | -| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | -| [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | -| [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | -| [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | -| [expand command](cmd/expand.md) | 3.1 | experimental (since 3.1) | Transform a single document into multiple documents by expanding a nested array field. | -| [flatten command](cmd/flatten.md) | 3.1 | experimental (since 3.1) | Flatten a struct or an object field into separate fields in a document. | -| [table command](cmd/table.md) | 3.3 | experimental (since 3.3) | Keep or remove fields from the search result using enhanced syntax options. | -| [stats command](cmd/stats.md) | 1.0 | stable (since 1.0) | Calculate aggregation from search results. | -| [eventstats command](cmd/eventstats.md) | 3.1 | experimental (since 3.1) | Calculate aggregation statistics and add them as new fields to each event. | -| [streamstats command](cmd/streamstats.md) | 3.4 | experimental (since 3.4) | Calculate cumulative or rolling statistics as events are processed in order. | -| [bin command](cmd/bin.md) | 3.3 | experimental (since 3.3) | Group numeric values into buckets of equal intervals. | -| [timechart command](cmd/timechart.md) | 3.3 | experimental (since 3.3) | Create time-based charts and visualizations. | -| [chart command](cmd/chart.md) | 3.4 | experimental (since 3.4) | Apply statistical aggregations to search results and group the data for visualizations. | -| [trendline command](cmd/trendline.md) | 3.0 | experimental (since 3.0) | Calculate moving averages of fields. | -| [sort command](cmd/sort.md) | 1.0 | stable (since 1.0) | Sort all the search results by the specified fields. | -| [reverse command](cmd/reverse.md) | 3.2 | experimental (since 3.2) | Reverse the display order of search results. | -| [head command](cmd/head.md) | 1.0 | stable (since 1.0) | Return the first N number of specified results after an optional offset in search order. | -| [dedup command](cmd/dedup.md) | 1.0 | stable (since 1.0) | Remove identical documents defined by the field from the search result. | -| [top command](cmd/top.md) | 1.0 | stable (since 1.0) | Find the most common tuple of values of all fields in the field list. | -| [rare command](cmd/rare.md) | 1.0 | stable (since 1.0) | Find the least common tuple of values of all fields in the field list. | -| [parse command](cmd/parse.md) | 1.3 | stable (since 1.3) | Parse a text field with a regular expression and append the result to the search result. | -| [grok command](cmd/grok.md) | 2.4 | stable (since 2.4) | Parse a text field with a grok pattern and append the results to the search result. | -| [rex command](cmd/rex.md) | 3.3 | experimental (since 3.3) | Extract fields from a raw text field using regular expression named capture groups. | -| [regex command](cmd/regex.md) | 3.3 | experimental (since 3.3) | Filter search results by matching field values against a regular expression pattern. | -| [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | -| [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | -| [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | -| [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | -| [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | -| [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | -| [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | -| [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | -| [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | -| [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | -| [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | -| [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | -| [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | -| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | -| [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | -| [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | -| [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | -| [mvcombine command](cmd/mvcombine.md) | 3.5 | stable (since 3.4) | Combines values of a specified field across rows identical on all other fields. | -| [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | -| [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | -| [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm. | -| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | - -- [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting + - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** - [Aggregation Functions](functions/aggregations.md) - [Collection Functions](functions/collection.md) From cf5afad2f5c117d42ea95ad4208c9708620da91a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 10:26:37 -0700 Subject: [PATCH 13/13] fix explain test failure Signed-off-by: Asif Bashar --- .../resources/expectedOutput/calcite/explain_xyseries.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml index 5fc1db286ed..610d9aa1410 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) - LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$2]) + LogicalProject(state=[$0], F=[$1], M=[$2]) LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) @@ -12,4 +12,4 @@ calcite: CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file