diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 701d1545b76..916cc00bc4a 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -112,6 +112,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -842,6 +843,11 @@ public LogicalPlan visitChart(Chart node, AnalysisContext context) { throw getOnlyForCalciteException("Chart"); } + @Override + public LogicalPlan visitXyseries(Xyseries node, AnalysisContext context) { + throw getOnlyForCalciteException("Xyseries"); + } + @Override public LogicalPlan visitWindow(Window node, AnalysisContext context) { throw getOnlyForCalciteException("Window"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index acb6e105661..266f8f46f7d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -100,6 +100,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; /** AST nodes visitor Defines the traverse path. */ public abstract class AbstractNodeVisitor { @@ -520,4 +521,8 @@ public T visitMvExpand(MvExpand node, C context) { public T visitGraphLookup(GraphLookup node, C context) { return visitChildren(node, context); } + + public T visitXyseries(Xyseries node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java new file mode 100644 index 00000000000..84fb3019e0f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node representing the xyseries command. Converts row-oriented grouped results into a wide + * table where one field is the X axis (row key), one field provides pivot values for column naming, + * and one or more data fields fill the pivoted cells. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Xyseries extends UnresolvedPlan { + + /** The x-axis field (row key in output). */ + private final UnresolvedExpression xField; + + /** The y-name field whose values become part of the output column names. */ + private final UnresolvedExpression yNameField; + + /** Explicit pivot values from the IN (...) clause. */ + private final List pivotValues; + + /** One or more y-data fields whose values fill the pivoted cells. */ + private final List yDataFields; + + /** Separator between y-data-field name and pivot value in column names. Default ":". */ + private final String separator; + + /** Optional format template for output column names using $AGG$ and $VAL$ placeholders. */ + private final String format; + + @Setter private UnresolvedPlan child; + + @Override + public Xyseries attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitXyseries(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 94c40e5adb2..9995895bfa1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -171,6 +172,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.AliasFieldsWrappable; import org.opensearch.sql.calcite.plan.HighlightPushDown; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -4071,6 +4073,131 @@ static ChartConfig fromArguments(ArgumentMap argMap) { } } + @Override + public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { + visitChildren(node, context); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + // Resolve x-field and y-name-field names + String xFieldName = resolveFieldName(node.getXField()); + String yNameFieldName = resolveFieldName(node.getYNameField()); + + // Resolve y-data field names + List yDataFieldNames = + node.getYDataFields().stream().map(this::resolveFieldName).collect(Collectors.toList()); + + List pivotValues = node.getPivotValues() != null ? node.getPivotValues() : List.of(); + String separator = node.getSeparator(); + String format = node.getFormat(); + + // Build the pivot axis - cast to VARCHAR if needed for string comparison + RexNode yNameRef = b.field(yNameFieldName); + RelDataType yNameType = yNameRef.getType(); + RexNode axis; + if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { + if (!SqlTypeUtil.isAtomic(yNameType)) { + throw new IllegalArgumentException( + "xyseries y-name-field must be a scalar type, got: " + yNameType.getSqlTypeName()); + } + RelDataType varchar = + rx.getTypeFactory() + .createTypeWithNullability( + rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), true); + axis = rx.makeCast(varchar, yNameRef, true); + } else { + axis = yNameRef; + } + + // Build aggregate calls - MAX for each y-data field + List aggCalls = + yDataFieldNames.stream() + .map(name -> b.max(b.field(name)).as(name)) + .collect(Collectors.toList()); + + // Build pivot value entries: alias -> [literal(value)] + // LinkedHashMap preserves insertion order for deterministic column ordering + LinkedHashMap> pivotValueMap = new LinkedHashMap<>(); + for (String val : pivotValues) { + pivotValueMap.put(val, ImmutableList.of(b.literal(val))); + } + + // Execute pivot: decomposes into GROUP BY x-field with FILTER-based aggregation + // Produces columns: x-field, {val1}_{agg1}, {val1}_{agg2}, {val2}_{agg1}, ... + b.pivot( + b.groupKey(b.field(xFieldName)), + aggCalls, + ImmutableList.of(axis), + pivotValueMap.entrySet()); + + // Pivot produces value-first column ordering: val1_agg1, val1_agg2, val2_agg1, ... + // Reorder to agg-first and apply custom column naming: agg1: val1, agg1: val2, ... + List reorderProjections = new ArrayList<>(); + List reorderNames = new ArrayList<>(); + + reorderProjections.add(b.field(xFieldName)); + reorderNames.add(xFieldName); + + for (String aggName : yDataFieldNames) { + for (String pivotVal : pivotValues) { + // Reference pivot output column by its generated name: {value}_{agg} + String pivotColName = pivotVal + "_" + aggName; + try { + reorderProjections.add(b.field(pivotColName)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "xyseries: expected pivot output column '" + pivotColName + "' not found", e); + } + boolean singleDataField = yDataFieldNames.size() == 1; + reorderNames.add(generateColumnName(aggName, pivotVal, separator, format, singleDataField)); + } + } + // Fail fast with a clear message if the naming scheme produced collisions + // (e.g. a format template that omits $VAL$ or $AGG$ with multiple series). + Set seenNames = new HashSet<>(); + for (String name : reorderNames) { + if (!seenNames.add(name)) { + throw new IllegalArgumentException( + "xyseries produced duplicate output column name '" + + name + + "'. Use a format template containing both $AGG$ and $VAL$ so column names" + + " are unique."); + } + } + b.project(reorderProjections, reorderNames, true); + + // Order by x-field + b.sort(b.field(0)); + + return b.peek(); + } + + private String resolveFieldName(UnresolvedExpression expr) { + if (expr instanceof Field) { + return ((Field) expr).getField().toString(); + } + if (expr instanceof Alias) { + return ((Alias) expr).getName(); + } + return expr.toString(); + } + + private String generateColumnName( + String yDataFieldName, + String pivotValue, + String separator, + String format, + boolean singleDataField) { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + if (singleDataField) { + return pivotValue; + } + return yDataFieldName + separator + pivotValue; + } + @Override public RelNode visitTrendline(Trendline node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/docs/user/ppl/cmd/xyseries.md b/docs/user/ppl/cmd/xyseries.md new file mode 100644 index 00000000000..a81ea7d604c --- /dev/null +++ b/docs/user/ppl/cmd/xyseries.md @@ -0,0 +1,155 @@ +# xyseries + +## Description + +The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. + +## Syntax + +```syntax +xyseries [sep=] [format=] in (, , ...) [, , ...] +``` + +## Parameters + +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | The field used as the row key in the output. Results are grouped and sorted by this field. | N/A | +| `` | Required | The field whose values are used to generate output column names. Only the values listed in the `in` clause are pivoted into columns. | N/A | +| `in (, , ...)` | Required | Explicit list of pivot values to select from ``. Each value generates one output column per ``. Values must be quoted strings. | N/A | +| `` | Required (at least one) | One or more fields containing the data to pivot. If multiple fields are specified, separate them with commas. | N/A | +| `sep` | Optional | Separator between the `` name and the pivot value in output column names. Ignored if `format` is specified. | `": "` | +| `format` | Optional | Naming template for output column names. Use `$AGG$` as a placeholder for the `` name and `$VAL$` as a placeholder for the pivot value. When specified, overrides `sep`. | N/A | + +## Notes + +The following considerations apply when using the `xyseries` command: + +* The `xyseries` command is typically used after a `stats` command that groups results by both the `` and ``. +* Output column names follow the pattern `` by default (for example, `host_cnt: 200`). Use the `format` option to customize this pattern. +* When a pivot value has no matching data for a given `` row, the output cell is `null`. +* The `` values are compared as strings. Non-string fields are cast to string automatically. +* Results are sorted by `` in ascending order. +* This command requires the Calcite engine to be enabled (`plugins.calcite.enabled: true`). + +## Example 1: Basic xyseries with a single data field + +This example pivots HTTP response codes into columns for a count of hosts per URL: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | +|--------+---------------+---------------+---------------| +| /page1 | 3 | 1 | null | +| /page2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+ +``` + +## Example 2: Multiple data fields + +This example pivots multiple aggregated fields at once: + +```ppl +source=weblogs +| stats count(host) as host_cnt, count(method) as method_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt, method_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | method_cnt: 200 | method_cnt: 404 | method_cnt: 500 | +|--------+---------------+---------------+---------------+------------------+------------------+------------------| +| /page1 | 3 | 1 | null | 3 | 1 | null | +| /page2 | 5 | null | 2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +``` + +## Example 3: Custom separator + +This example uses a custom separator between the data field name and pivot value in column names: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries sep="-" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | host_cnt-200 | host_cnt-404 | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 4: Format template + +This example uses a format template to customize output column names. `$VAL$` is replaced with the pivot value and `$AGG$` is replaced with the data field name: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries format="$VAL$_$AGG$" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | 200_host_cnt | 404_host_cnt | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 5: Partial pivot values + +When only a subset of values is specified in the `in` clause, rows with unmatched `` values produce `null` for the corresponding `` rows: + +```ppl +source=accounts +| stats avg(balance) as avg_balance by gender, state +| xyseries state gender in ("F") avg_balance +``` + +The query returns the following results: + +```text +fetched rows / total rows = 7/7 ++-------+-----------------+ +| state | avg_balance: F | +|-------+-----------------| +| IL | null | +| IN | 48086.0 | +| MD | null | +| PA | 40540.0 | +| TN | null | +| VA | 32838.0 | +| WA | null | ++-------+-----------------+ +``` + +## Limitations + +The `xyseries` command has the following limitations: + +* Pivot values must be explicitly provided in the `in` clause. Dynamic pivot (deriving column names from data at runtime) is not supported. +* This command is only available when the Calcite engine is enabled. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 3eed4181c61..067d5f524fd 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -89,7 +89,8 @@ source=accounts | [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | | [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | | [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| - +| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | + - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** - [Aggregation Functions](functions/aggregations.md) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index b78a71e534c..04dc2b0e74b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -3028,6 +3028,48 @@ public void testHighlightOsdObjectFormatExplain() throws IOException { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void testXyseriesExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesMultipleDataFieldsExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance, count() as cnt by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance, cnt", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_multiple_data_fields.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesWithFormatExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state | xyseries" + + " format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_with_format.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void testExplainConsecutiveSortsAfterAggIssue5125() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 6b5ac0d4302..a32dd9fb990 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; import java.io.IOException; +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -567,4 +568,78 @@ public void testUnionUnsupportedInV2() throws IOException { } verifyQuery(result); } + + @Test + public void testXyseriesCommand() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandMultipleDataFields() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance, count() as cnt by" + + " gender, state | xyseries state gender in (\"F\", \"M\") avg_balance," + + " cnt", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithSep() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries sep=\"-\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithFormat() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\")" + + " avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index 0029921c1fc..86e98c1523f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -237,4 +237,28 @@ public void testCrossClusterConvertWithAlias() throws IOException { disableCalcite(); } + + @Test + public void testCrossClusterXyseries() throws IOException { + enableCalcite(); + + JSONObject result = + executeQuery( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in ('F', 'M') avg_balance", + TEST_INDEX_BANK_REMOTE)); + verifyColumn(result, columnName("state"), columnName("F"), columnName("M")); + verifyDataRows( + result, + rows("IL", null, 39225.0), + rows("IN", 48086.0, null), + rows("MD", null, 4180.0), + rows("PA", 40540.0, null), + rows("TN", null, 5686.0), + rows("VA", 32838.0, null), + rows("WA", null, 16418.0)); + + disableCalcite(); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml new file mode 100644 index 00000000000..610d9aa1410 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F=[$1], M=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml new file mode 100644 index 00000000000..a8419925e99 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$3], cnt: F=[$2], cnt: M=[$4]) + LogicalAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + LogicalProject(avg_balance=[$2], cnt=[$3], state=[$1], $f4=[IS TRUE(=($0, 'F'))], $f5=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)], cnt=[COUNT()]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}], avg_balance: M=[$t3], cnt: F=[$t2], cnt: M=[$t4]) + EnumerableAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=['F'], expr#5=[=($t0, $t4)], expr#6=[IS TRUE($t5)], expr#7=['M'], expr#8=[=($t0, $t7)], expr#9=[IS TRUE($t8)], avg_balance=[$t2], cnt=[$t3], state=[$t1], $f4=[$t6], $f5=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml new file mode 100644 index 00000000000..eaf89c53c2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F_avg_balance=[$1], M_avg_balance=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index b26751ad61b..fb072ae134f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -60,6 +60,8 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +XYSERIES: 'XYSERIES'; +SEP: 'SEP'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index eeaed6daf52..a23a314537d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -99,6 +99,7 @@ commands | fieldformatCommand | nomvCommand | graphLookupCommand + | xyseriesCommand | unionCommand | timewrapCommand ; @@ -153,6 +154,7 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | XYSERIES | TIMEWRAP | MAKERESULTS ; @@ -764,6 +766,19 @@ graphLookupArgs | (FILTER EQUAL LT_PRTHS logicalExpression RT_PRTHS) ; +xyseriesCommand + : XYSERIES xyseriesOption* xField = fieldExpression yNameField = fieldExpression IN LT_PRTHS xyseriesPivotValues RT_PRTHS yDataFields = fieldList + ; + +xyseriesOption + : SEP EQUAL sep = stringLiteral + | FORMAT EQUAL format = stringLiteral + ; + +xyseriesPivotValues + : stringLiteral (COMMA stringLiteral)* + ; + // clauses fromClause : SOURCE EQUAL tableOrSubqueryClause @@ -1856,4 +1871,5 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + | SEP ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e87264909c8..efdbf26a205 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -129,6 +129,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -1837,4 +1838,37 @@ public UnresolvedPlan visitGraphLookupCommand(OpenSearchPPLParser.GraphLookupCom .filter(filter) .build(); } + + /** Xyseries command. */ + @Override + public UnresolvedPlan visitXyseriesCommand(OpenSearchPPLParser.XyseriesCommandContext ctx) { + UnresolvedExpression xField = internalVisitExpression(ctx.xField); + UnresolvedExpression yNameField = internalVisitExpression(ctx.yNameField); + + // Parse pivot values from IN (...) clause + List pivotValues = + ctx.xyseriesPivotValues().stringLiteral().stream() + .map(s -> StringUtils.unquoteText(s.getText())) + .distinct() + .collect(Collectors.toList()); + + // Parse y-data fields + List yDataFields = + ctx.yDataFields.fieldExpression().stream() + .map(this::internalVisitExpression) + .collect(Collectors.toList()); + + // Parse options + String separator = ": "; + String format = null; + for (OpenSearchPPLParser.XyseriesOptionContext optCtx : ctx.xyseriesOption()) { + if (optCtx.SEP() != null) { + separator = StringUtils.unquoteText(optCtx.sep.getText()); + } else if (optCtx.FORMAT() != null) { + format = StringUtils.unquoteText(optCtx.format.getText()); + } + } + + return new Xyseries(xField, yNameField, pivotValues, yDataFields, separator, format); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5f45a96f8c..0650b4d1e4c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -83,6 +83,7 @@ import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.exception.SemanticCheckException; @@ -1827,6 +1828,78 @@ public void testMalformedPipeProducesSyntaxError() { plan("source=t | invalidCmd |"); } + // Xyseries tests + + @Test + public void testXyseriesCommandBasic() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries url response in (\"200\", \"404\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandMultipleDataFields() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt"), field("method_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries url response in (\"200\", \"404\") host_cnt, method_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSep() { + Xyseries expected = + new Xyseries( + field("url"), field("response"), List.of("200"), List.of(field("host_cnt")), "-", null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries sep=\"-\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200"), + List.of(field("host_cnt")), + ": ", + "$VAL$+$AGG$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries format=\"$VAL$+$AGG$\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSepAndFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + "-", + "$AGG$_$VAL$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries sep=\"-\" format=\"$AGG$_$VAL$\" url response in (\"200\", \"404\")" + + " host_cnt", + expected); + } + @Test public void testUnionWithSubsearches() { plan("| union [search source=t1 | where age > 30] " + "[search source=t2 | where age < 20]");