diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index a588c639185..2e3c62e8153 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -80,6 +80,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.SubqueryAlias; @@ -682,6 +683,12 @@ public LogicalPlan visitFlatten(Flatten node, AnalysisContext context) { "FLATTEN is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true"); } + @Override + public LogicalPlan visitReverse(Reverse node, AnalysisContext context) { + throw new UnsupportedOperationException( + "REVERSE is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true"); + } + @Override public LogicalPlan visitPaginate(Paginate paginate, AnalysisContext context) { LogicalPlan child = paginate.getChild().get(0).accept(this, context); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 5e67f3fba0e..f9eadced7fc 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -69,6 +69,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.RelationSubquery; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; @@ -244,6 +245,10 @@ public T visitSort(Sort node, C context) { return visitChildren(node, context); } + public T visitReverse(Reverse node, C context) { + return visitChildren(node, context); + } + public T visitLambdaFunction(LambdaFunction node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Reverse.java b/core/src/main/java/org/opensearch/sql/ast/tree/Reverse.java new file mode 100644 index 00000000000..7d48787e44e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Reverse.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** AST node represent Reverse operation. */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Reverse extends UnresolvedPlan { + + private UnresolvedPlan child; + + @Override + public Reverse attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitReverse(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 8db31f1eb88..7c936b189fe 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -355,6 +355,28 @@ public RelNode visitHead(Head node, CalcitePlanContext context) { return context.relBuilder.peek(); } + private static final String REVERSE_ROW_NUM = "__reverse_row_num__"; + + @Override + public RelNode visitReverse( + org.opensearch.sql.ast.tree.Reverse node, CalcitePlanContext context) { + visitChildren(node, context); + // Add ROW_NUMBER() column + RexNode rowNumber = + context + .relBuilder + .aggregateCall(SqlStdOperatorTable.ROW_NUMBER) + .over() + .rowsTo(RexWindowBounds.CURRENT_ROW) + .as(REVERSE_ROW_NUM); + context.relBuilder.projectPlus(rowNumber); + // Sort by row number descending + context.relBuilder.sort(context.relBuilder.desc(context.relBuilder.field(REVERSE_ROW_NUM))); + // Remove row number column + context.relBuilder.projectExcept(context.relBuilder.field(REVERSE_ROW_NUM)); + return context.relBuilder.peek(); + } + @Override public RelNode visitParse(Parse node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/docs/user/ppl/cmd/reverse.rst b/docs/user/ppl/cmd/reverse.rst new file mode 100644 index 00000000000..2efe833855f --- /dev/null +++ b/docs/user/ppl/cmd/reverse.rst @@ -0,0 +1,120 @@ +============= +reverse +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``reverse`` command to reverse the display order of search results. The same results are returned, but in reverse order. + +Version +======= +3.2.0 + +Syntax +============ +reverse + + +* No parameters: The reverse command takes no arguments or options. + +Note +===== +The `reverse` command processes the entire dataset. If applied directly to millions of records, it will consume significant memory resources on the coordinating node. Users should only apply the `reverse` command to smaller datasets, typically after aggregation operations. + +Example 1: Basic reverse operation +================================== + +The example shows reversing the order of all documents. + +PPL query:: + + os> source=accounts | fields account_number, age | reverse; + fetched rows / total rows = 4/4 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +----------------+-----+ + + +Example 2: Reverse with sort +============================ + +The example shows reversing results after sorting by age in ascending order, effectively giving descending order. + +PPL query:: + + os> source=accounts | sort age | fields account_number, age | reverse; + fetched rows / total rows = 4/4 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + | 13 | 28 | + +----------------+-----+ + + +Example 3: Reverse with head +============================ + +The example shows using reverse with head to get the last 2 records from the original order. + +PPL query:: + + os> source=accounts | reverse | head 2 | fields account_number, age; + fetched rows / total rows = 2/2 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + +----------------+-----+ + + +Example 4: Double reverse +========================= + +The example shows that applying reverse twice returns to the original order. + +PPL query:: + + os> source=accounts | reverse | reverse | fields account_number, age; + fetched rows / total rows = 4/4 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 13 | 28 | + | 1 | 32 | + | 18 | 33 | + | 6 | 36 | + +----------------+-----+ + + +Example 5: Reverse with complex pipeline +======================================= + +The example shows reverse working with filtering and field selection. + +PPL query:: + + os> source=accounts | where age > 30 | fields account_number, age | reverse; + fetched rows / total rows = 3/3 + +----------------+-----+ + | account_number | age | + |----------------+-----| + | 6 | 36 | + | 18 | 33 | + | 1 | 32 | + +----------------+-----+ diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index d1477d9b3b0..59df104d63e 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -104,6 +104,8 @@ The query start with search command and then flowing a set of command delimited - `subquery (aka subsearch) command `_ + - `reverse command `_ + - `top command `_ - `trendline command `_ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 7ab4eab104b..b59c5550dc9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -120,4 +120,31 @@ public void testFilterScriptPushDownExplain() throws Exception { public void testFilterFunctionScriptPushDownExplain() throws Exception { super.testFilterFunctionScriptPushDownExplain(); } + + @Test + public void testExplainWithReverse() throws IOException { + String result = + executeWithReplace( + "explain source=opensearch-sql_test_index_account | sort age | reverse | head 5"); + + // Verify that the plan contains a LogicalSort with fetch (from head 5) + assertTrue(result.contains("LogicalSort") && result.contains("fetch=[5]")); + + // Verify that reverse added a ROW_NUMBER and another sort (descending) + assertTrue(result.contains("ROW_NUMBER()")); + assertTrue(result.contains("dir0=[DESC]")); + } + + /** + * Executes the PPL query and returns the result as a string with windows-style line breaks + * replaced with Unix-style ones. + * + * @param ppl the PPL query to execute + * @return the result of the query as a string with line breaks replaced + * @throws IOException if an error occurs during query execution + */ + private String executeWithReplace(String ppl) throws IOException { + var result = executeQueryToString(ppl); + return result.replace("\\r\\n", "\\n"); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java new file mode 100644 index 00000000000..5ff41bcb3f5 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteReverseCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.BANK); + } + + @Test + public void testReverse() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | fields account_number | reverse", TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder( + result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1)); + } + + @Test + public void testReverseWithFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields account_number, firstname | reverse", TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint"), schema("firstname", "string")); + verifyDataRowsInOrder( + result, + rows(32, "Dillard"), + rows(25, "Virginia"), + rows(20, "Elinor"), + rows(18, "Dale"), + rows(13, "Nanette"), + rows(6, "Hattie"), + rows(1, "Amber JOHnny")); + } + + @Test + public void testReverseWithSort() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | fields account_number | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder( + result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1)); + } + + @Test + public void testDoubleReverse() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields account_number | reverse | reverse", TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder( + result, rows(1), rows(6), rows(13), rows(18), rows(20), rows(25), rows(32)); + } + + @Test + public void testReverseWithHead() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | fields account_number | reverse | head 3", TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder(result, rows(32), rows(25), rows(20)); + } + + @Test + public void testReverseWithComplexPipeline() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where account_number > 18 | fields account_number | reverse | head 2", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder(result, rows(32), rows(25)); + } + + @Test + public void testReverseWithMultipleSorts() throws IOException { + // Use the existing BANK data but with a simpler, more predictable query + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | fields account_number | reverse | head 3", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder(result, rows(32), rows(25), rows(20)); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index dfd655177ab..037e07df7dc 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -69,6 +69,7 @@ commands | appendcolCommand | expandCommand | flattenCommand + | reverseCommand ; commandName @@ -99,6 +100,7 @@ commandName | FLATTEN | TRENDLINE | EXPLAIN + | REVERSE ; searchCommand @@ -141,6 +143,10 @@ sortCommand : SORT sortbyClause ; +reverseCommand + : REVERSE + ; + evalCommand : EVAL evalClause (COMMA evalClause)* ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 2c5907b19a7..8676089bdf7 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -75,6 +75,7 @@ import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; @@ -375,6 +376,12 @@ public UnresolvedPlan visitSortCommand(SortCommandContext ctx) { .collect(Collectors.toList())); } + /** Reverse command. */ + @Override + public UnresolvedPlan visitReverseCommand(OpenSearchPPLParser.ReverseCommandContext ctx) { + return new Reverse(); + } + /** Eval command. */ @Override public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 11e3504665c..5d1319f861e 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -70,6 +70,7 @@ import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; +import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; @@ -351,6 +352,12 @@ public String visitHead(Head node, String context) { return StringUtils.format("%s | head %d", child, size); } + @Override + public String visitReverse(Reverse node, String context) { + String child = node.getChild().get(0).accept(this, context); + return StringUtils.format("%s | reverse", child); + } + @Override public String visitParse(Parse node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java new file mode 100644 index 00000000000..179fb3bc830 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java @@ -0,0 +1,181 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +public class CalcitePPLReverseTest extends CalcitePPLAbstractTest { + public CalcitePPLReverseTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testReverseParserSuccess() { + String ppl = "source=EMP | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7900; ENAME=JAMES; JOB=CLERK; MGR=7698; HIREDATE=1981-12-03; SAL=950.00;" + + " COMM=null; DEPTNO=30\n" + + "EMPNO=7876; ENAME=ADAMS; JOB=CLERK; MGR=7788; HIREDATE=1987-05-23; SAL=1100.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7844; ENAME=TURNER; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-08; SAL=1500.00;" + + " COMM=0.00; DEPTNO=30\n" + + "EMPNO=7839; ENAME=KING; JOB=PRESIDENT; MGR=null; HIREDATE=1981-11-17; SAL=5000.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7788; ENAME=SCOTT; JOB=ANALYST; MGR=7566; HIREDATE=1987-04-19; SAL=3000.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7782; ENAME=CLARK; JOB=MANAGER; MGR=7839; HIREDATE=1981-06-09; SAL=2450.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7698; ENAME=BLAKE; JOB=MANAGER; MGR=7839; HIREDATE=1981-01-05; SAL=2850.00;" + + " COMM=null; DEPTNO=30\n" + + "EMPNO=7654; ENAME=MARTIN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-09-28; SAL=1250.00;" + + " COMM=1400.00; DEPTNO=30\n" + + "EMPNO=7566; ENAME=JONES; JOB=MANAGER; MGR=7839; HIREDATE=1981-02-04; SAL=2975.00;" + + " COMM=null; DEPTNO=20\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30\n" + + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00;" + + " COMM=null; DEPTNO=20\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "" + + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY 9 DESC NULLS FIRST) `t0`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseWithSortParserSuccess() { + String ppl = "source=EMP | sort ENAME | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" + + " LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "" + + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `ENAME`) `t0`\n" + + "ORDER BY `__reverse_row_num__` DESC NULLS FIRST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testDoubleReverseParserSuccess() { + String ppl = "source=EMP | reverse | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY 9 DESC NULLS FIRST) `t0`\n" + + "ORDER BY 9 DESC NULLS FIRST) `t2`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseWithHeadParserSuccess() { + String ppl = "source=EMP | reverse | head 2"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC], fetch=[2])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "EMPNO=7934; ENAME=MILLER; JOB=CLERK; MGR=7782; HIREDATE=1982-01-23; SAL=1300.00;" + + " COMM=null; DEPTNO=10\n" + + "EMPNO=7902; ENAME=FORD; JOB=ANALYST; MGR=7566; HIREDATE=1981-12-03; SAL=3000.00;" + + " COMM=null; DEPTNO=20\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY 9 DESC NULLS FIRST\n" + + "LIMIT 2) `t0`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test(expected = Exception.class) + public void testReverseWithNumberShouldFail() { + String ppl = "source=EMP | reverse 2"; + getRelNode(ppl); + } + + @Test(expected = Exception.class) + public void testReverseWithFieldShouldFail() { + String ppl = "source=EMP | reverse EMPNO"; + getRelNode(ppl); + } + + @Test(expected = Exception.class) + public void testReverseWithStringShouldFail() { + String ppl = "source=EMP | reverse \"desc\""; + getRelNode(ppl); + } + + @Test(expected = Exception.class) + public void testReverseWithExpressionShouldFail() { + String ppl = "source=EMP | reverse EMPNO + 1"; + getRelNode(ppl); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index f044fa5886a..2d4d4411b26 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -131,6 +131,11 @@ public void testHeadCommandWithNumber() { assertEquals("source=t | head 3", anonymize("source=t | head 3")); } + @Test + public void testReverseCommand() { + assertEquals("source=t | reverse", anonymize("source=t | reverse")); + } + // todo, sort order is ignored, it doesn't impact the log analysis. @Test public void testSortCommandWithOptions() {