diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 6a556eccc92..b97b7cf16e8 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -46,9 +46,14 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.ViewExpanders; +import org.apache.calcite.rel.BiRel; +import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.core.SetOp; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFamily; @@ -667,25 +672,133 @@ public RelNode visitHead(Head node, CalcitePlanContext context) { return context.relBuilder.peek(); } - private static final String REVERSE_ROW_NUM = "__reverse_row_num__"; + /** + * Backtrack through the RelNode tree to find the first Sort node with non-empty collation. Stops + * at blocking operators that break ordering: + * + * + * + * @param node the starting RelNode to backtrack from + * @return the collation found, or null if no sort or blocking operator encountered + */ + private RelCollation backtrackForCollation(RelNode node) { + while (node != null) { + // Check for blocking operators that destroy collation + // BiRel covers Join, Correlate, and other binary relations + // SetOp covers Union, Intersect, Except + // Uncollect unnests arrays/multisets which may change ordering + if (node instanceof Aggregate + || node instanceof BiRel + || node instanceof SetOp + || node instanceof Uncollect) { + return null; + } + + // Project with window functions has ordering determined by the window's ORDER BY clause + // We should not destroy its output order by inserting a reversed sort + if (node instanceof LogicalProject && ((LogicalProject) node).containsOver()) { + return null; + } + + // Check for Sort node with collation + if (node instanceof org.apache.calcite.rel.core.Sort) { + org.apache.calcite.rel.core.Sort sort = (org.apache.calcite.rel.core.Sort) node; + if (sort.getCollation() != null && !sort.getCollation().getFieldCollations().isEmpty()) { + return sort.getCollation(); + } + } + + // Continue to child node + if (node.getInputs().isEmpty()) { + break; + } + node = node.getInput(0); + } + return null; + } + + /** + * Insert a reversed sort node after finding the original sort in the tree. This rebuilds the tree + * with the reversed sort inserted right after the original sort. + * + * @param root the root of the tree to rebuild + * @param reversedCollation the reversed collation to insert + * @param context the Calcite plan context + * @return the rebuilt tree with reversed sort inserted + */ + private RelNode insertReversedSortInTree( + RelNode root, RelCollation reversedCollation, CalcitePlanContext context) { + return root.accept( + new org.apache.calcite.rel.RelHomogeneousShuttle() { + boolean sortFound = false; + + @Override + public RelNode visit(RelNode other) { + // Check if this is a Sort node and we haven't inserted the reversed sort yet + if (!sortFound && other instanceof org.apache.calcite.rel.core.Sort) { + org.apache.calcite.rel.core.Sort sort = (org.apache.calcite.rel.core.Sort) other; + if (sort.getCollation() != null + && !sort.getCollation().getFieldCollations().isEmpty()) { + // Found the sort node - insert reversed sort after it + sortFound = true; + // First visit the sort's children + RelNode visitedSort = super.visit(other); + // Create a new reversed sort on top of the original sort + return org.apache.calcite.rel.logical.LogicalSort.create( + visitedSort, reversedCollation, null, null); + } + } + // For all other nodes, continue traversal + return super.visit(other); + } + }); + } @Override public RelNode visitReverse( org.opensearch.sql.ast.tree.Reverse node, CalcitePlanContext context) { visitChildren(node, context); - // Add ROW_NUMBER() column - RexNode rowNumber = - context - .relBuilder - .aggregateCall(SqlStdOperatorTable.ROW_NUMBER) - .over() - .rowsTo(RexWindowBounds.CURRENT_ROW) - .as(REVERSE_ROW_NUM); - context.relBuilder.projectPlus(rowNumber); - // Sort by row number descending - context.relBuilder.sort(context.relBuilder.desc(context.relBuilder.field(REVERSE_ROW_NUM))); - // Remove row number column - context.relBuilder.projectExcept(context.relBuilder.field(REVERSE_ROW_NUM)); + + // Check if there's an existing sort to reverse + List collations = + context.relBuilder.getCluster().getMetadataQuery().collations(context.relBuilder.peek()); + RelCollation collation = collations != null && !collations.isEmpty() ? collations.get(0) : null; + + if (collation != null && !collation.getFieldCollations().isEmpty()) { + // If there's an existing sort, reverse its direction + RelCollation reversedCollation = PlanUtils.reverseCollation(collation); + context.relBuilder.sort(reversedCollation); + } else { + // Collation not found on current node - try backtracking + RelNode currentNode = context.relBuilder.peek(); + RelCollation backtrackCollation = backtrackForCollation(currentNode); + + if (backtrackCollation != null && !backtrackCollation.getFieldCollations().isEmpty()) { + // Found collation through backtracking - rebuild tree with reversed sort + RelCollation reversedCollation = PlanUtils.reverseCollation(backtrackCollation); + RelNode rebuiltTree = insertReversedSortInTree(currentNode, reversedCollation, context); + // Replace the current node in the builder with the rebuilt tree + context.relBuilder.build(); // Pop the current node + context.relBuilder.push(rebuiltTree); // Push the rebuilt tree + } else { + // Check if @timestamp field exists in the row type + List fieldNames = context.relBuilder.peek().getRowType().getFieldNames(); + if (fieldNames.contains(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP)) { + // If @timestamp exists, sort by it in descending order + context.relBuilder.sort( + context.relBuilder.desc( + context.relBuilder.field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP))); + } + // If neither collation nor @timestamp exists, ignore the reverse command (no-op) + } + } + return context.relBuilder.peek(); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 50e03fc608f..84467f6f60e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -22,6 +22,9 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; @@ -588,6 +591,37 @@ public Void visitCorrelVariable(RexCorrelVariable correlVar) { } } + /** + * Reverses the direction of a RelCollation. + * + * @param original The original collation to reverse + * @return A new RelCollation with reversed directions + */ + public static RelCollation reverseCollation(RelCollation original) { + if (original == null || original.getFieldCollations().isEmpty()) { + return original; + } + + List reversedFields = new ArrayList<>(); + for (RelFieldCollation field : original.getFieldCollations()) { + RelFieldCollation.Direction reversedDirection = field.direction.reverse(); + + // Handle null direction properly - reverse it as well + RelFieldCollation.NullDirection reversedNullDirection = + field.nullDirection == RelFieldCollation.NullDirection.FIRST + ? RelFieldCollation.NullDirection.LAST + : field.nullDirection == RelFieldCollation.NullDirection.LAST + ? RelFieldCollation.NullDirection.FIRST + : field.nullDirection; + + RelFieldCollation reversedField = + new RelFieldCollation(field.getFieldIndex(), reversedDirection, reversedNullDirection); + reversedFields.add(reversedField); + } + + return RelCollations.of(reversedFields); + } + /** Adds a rel node to the top of the stack while preserving the field names and aliases. */ static void replaceTop(RelBuilder relBuilder, RelNode relNode) { try { diff --git a/docs/category.json b/docs/category.json index f3fe70ecfa5..d301d1db192 100644 --- a/docs/category.json +++ b/docs/category.json @@ -42,6 +42,7 @@ "user/ppl/cmd/rename.rst", "user/ppl/cmd/multisearch.rst", "user/ppl/cmd/replace.rst", + "user/ppl/cmd/reverse.rst", "user/ppl/cmd/rex.rst", "user/ppl/cmd/search.rst", "user/ppl/cmd/showdatasources.rst", diff --git a/docs/user/ppl/cmd/reverse.rst b/docs/user/ppl/cmd/reverse.rst index d839a687bf9..d3ac6b55255 100644 --- a/docs/user/ppl/cmd/reverse.rst +++ b/docs/user/ppl/cmd/reverse.rst @@ -11,7 +11,26 @@ reverse Description =========== -| The ``reverse`` command reverses the display order of search results. The same results are returned, but in reverse order. +| The ``reverse`` command reverses the display order of search results. The behavior depends on the query context: +| +| **1. With existing sort**: Reverses the sort direction(s) +| **2. With @timestamp field (no explicit sort)**: Sorts by @timestamp in descending order +| **3. Without sort or @timestamp**: The command is ignored (no effect) +============ + +Behavior +======== +The ``reverse`` command follows a three-tier logic: + +1. **If there's an explicit sort command before reverse**: The reverse command flips all sort directions (ASC ↔ DESC) +2. **If no explicit sort but the index has an @timestamp field**: The reverse command sorts by @timestamp in descending order (most recent first) +3. **If neither condition is met**: The reverse command is ignored and has no effect on the result order + +This design optimizes performance by avoiding expensive operations when reverse has no meaningful semantic interpretation. + +Version +======= +3.2.0 Syntax ====== @@ -21,16 +40,16 @@ reverse Note ==== -| The `reverse` command processes the entire dataset. If applied directly to millions of records, it will consume significant memory resources on the coordinating node. Users should only apply the `reverse` command to smaller datasets, typically after aggregation operations. +The ``reverse`` command is optimized to avoid unnecessary memory consumption. When applied without an explicit sort or @timestamp field, it is ignored. When used with an explicit sort, it efficiently reverses the sort direction(s) without materializing the entire dataset. -Example 1: Basic reverse operation -================================== +Example 1: Reverse with explicit sort +====================================== -This example shows reversing the order of all documents. +The example shows reversing the order of all documents. PPL query:: - os> source=accounts | fields account_number, age | reverse; + os> source=accounts | sort age | fields account_number, age | reverse; fetched rows / total rows = 4/4 +----------------+-----+ | account_number | age | @@ -42,33 +61,52 @@ PPL query:: +----------------+-----+ -Example 2: Reverse with sort -============================ +Example 2: Reverse with @timestamp field +========================================= -This example shows reversing results after sorting by age in ascending order, effectively giving descending order. +The example shows reverse on a time-series index automatically sorts by @timestamp in descending order (most recent first). PPL query:: - os> source=accounts | sort age | fields account_number, age | reverse; - fetched rows / total rows = 4/4 + os> source=time_test | fields value, @timestamp | reverse | head 3; + fetched rows / total rows = 3/3 + +-------+---------------------+ + | value | @timestamp | + |-------+---------------------| + | 9243 | 2025-07-28 09:41:29 | + | 7654 | 2025-07-28 08:22:11 | + | 8321 | 2025-07-28 07:05:33 | + +-------+---------------------+ + +Note: When the index contains an @timestamp field and no explicit sort is specified, reverse will sort by @timestamp DESC to show the most recent events first. This is particularly useful for log analysis and time-series data. + +Example 3: Reverse ignored (no sort, no @timestamp) +=================================================== + +The example shows that reverse is ignored when there's no explicit sort and no @timestamp field. + +PPL query:: + + os> source=accounts | fields account_number, age | reverse | head 2; + fetched rows / total rows = 2/2 +----------------+-----+ | account_number | age | |----------------+-----| - | 6 | 36 | - | 18 | 33 | | 1 | 32 | - | 13 | 28 | + | 6 | 36 | +----------------+-----+ +Note: Results appear in natural order (same as without reverse) because accounts index has no @timestamp field and no explicit sort was specified. + -Example 3: Reverse with head -============================ +Example 4: Reverse with sort and head +===================================== -This example shows using reverse with head to get the last 2 records from the original order. +The example shows using reverse with sort and head to get the top 2 records by age. PPL query:: - os> source=accounts | reverse | head 2 | fields account_number, age; + os> source=accounts | sort age | reverse | head 2 | fields account_number, age; fetched rows / total rows = 2/2 +----------------+-----+ | account_number | age | @@ -78,14 +116,14 @@ PPL query:: +----------------+-----+ -Example 4: Double reverse -========================= +Example 5: Double reverse with sort +=================================== -This example shows that applying reverse twice returns to the original order. +The example shows that applying reverse twice with an explicit sort returns to the original sort order. PPL query:: - os> source=accounts | reverse | reverse | fields account_number, age; + os> source=accounts | sort age | reverse | reverse | fields account_number, age; fetched rows / total rows = 4/4 +----------------+-----+ | account_number | age | @@ -97,19 +135,22 @@ PPL query:: +----------------+-----+ -Example 5: Reverse with complex pipeline -======================================== +Example 6: Reverse with multiple sort fields +============================================ -This example shows reverse working with filtering and field selection. +The example shows reverse flipping all sort directions when multiple fields are sorted. PPL query:: - os> source=accounts | where age > 30 | fields account_number, age | reverse; - fetched rows / total rows = 3/3 + os> source=accounts | sort +age, -account_number | reverse | fields account_number, age; + fetched rows / total rows = 4/4 +----------------+-----+ | account_number | age | |----------------+-----| | 6 | 36 | | 18 | 33 | | 1 | 32 | + | 13 | 28 | +----------------+-----+ + +Note: Original sort is ASC age, DESC account_number. After reverse, it becomes DESC age, ASC account_number. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 15051417db1..56b12a84c5a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -88,6 +88,7 @@ CalciteQueryAnalysisIT.class, CalciteRareCommandIT.class, CalciteRegexCommandIT.class, + CalciteReverseCommandIT.class, CalciteRexCommandIT.class, CalciteRenameCommandIT.class, CalciteReplaceCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index d0cac82b23f..313b3e393c0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -414,17 +414,63 @@ public void testFilterWithSearchCall() throws IOException { } @Test - public void testExplainWithReverse() throws IOException { - String result = - executeWithReplace( - "explain source=opensearch-sql_test_index_account | sort age | reverse | head 5"); + public void testExplainWithReverseIgnored() throws IOException { + // Reverse is ignored when there's no existing sort and no @timestamp field + String query = "source=opensearch-sql_test_index_account | reverse | head 5"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_reverse_ignored.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainWithReversePushdown() throws IOException { + String query = "source=opensearch-sql_test_index_account | sort - age | reverse"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_reverse_pushdown_single.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainWithReversePushdownMultipleFields() throws IOException { + String query = "source=opensearch-sql_test_index_account | sort - age, + firstname | reverse"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_reverse_pushdown_multiple.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainWithDoubleReverseIgnored() throws IOException { + // Double reverse is ignored when there's no existing sort and no @timestamp field + String query = "source=opensearch-sql_test_index_account | reverse | reverse"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_double_reverse_ignored.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainWithDoubleReversePushdown() throws IOException { + String query = "source=opensearch-sql_test_index_account | sort - age | reverse | reverse"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_double_reverse_pushdown_single.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } - // Verify that the plan contains a LogicalSort with fetch (from head 5) - assertTrue(result.contains("LogicalSort") && result.contains("fetch=[5]")); + @Test + public void testExplainWithDoubleReversePushdownMultipleFields() throws IOException { + String query = + "source=opensearch-sql_test_index_account | sort - age, + firstname | reverse | reverse"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_double_reverse_pushdown_multiple.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } - // Verify that reverse added a ROW_NUMBER and another sort (descending) - assertTrue(result.contains("ROW_NUMBER()")); - assertTrue(result.contains("dir0=[DESC]")); + @Test + public void testExplainReverseWithTimestamp() throws IOException { + // Test that reverse with @timestamp field sorts by @timestamp DESC + String query = "source=opensearch-sql_test_index_time_data | reverse | head 5"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_reverse_with_timestamp.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java index 5ff41bcb3f5..c95b0469af4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java @@ -6,8 +6,11 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; import static org.opensearch.sql.util.MatcherUtils.verifySchema; @@ -24,12 +27,18 @@ public void init() throws Exception { enableCalcite(); disallowCalciteFallback(); loadIndex(Index.BANK); + loadIndex(Index.TIME_TEST_DATA); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.EVENTS); } @Test public void testReverse() throws IOException { JSONObject result = - executeQuery(String.format("source=%s | fields account_number | reverse", TEST_INDEX_BANK)); + executeQuery( + String.format( + "source=%s | fields account_number | sort account_number | reverse", + TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint")); verifyDataRowsInOrder( result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1)); @@ -40,7 +49,8 @@ public void testReverseWithFields() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | fields account_number, firstname | reverse", TEST_INDEX_BANK)); + "source=%s | fields account_number, firstname | sort account_number | reverse", + TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint"), schema("firstname", "string")); verifyDataRowsInOrder( result, @@ -70,7 +80,8 @@ public void testDoubleReverse() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | fields account_number | reverse | reverse", TEST_INDEX_BANK)); + "source=%s | fields account_number | sort account_number | reverse | reverse", + TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint")); verifyDataRowsInOrder( result, rows(1), rows(6), rows(13), rows(18), rows(20), rows(25), rows(32)); @@ -80,7 +91,9 @@ public void testDoubleReverse() throws IOException { public void testReverseWithHead() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | fields account_number | reverse | head 3", TEST_INDEX_BANK)); + String.format( + "source=%s | fields account_number | sort account_number | reverse | head 3", + TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint")); verifyDataRowsInOrder(result, rows(32), rows(25), rows(20)); } @@ -90,21 +103,414 @@ public void testReverseWithComplexPipeline() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where account_number > 18 | fields account_number | reverse | head 2", + "source=%s | where account_number > 18 | fields account_number | sort" + + " account_number | reverse | head 2", TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint")); verifyDataRowsInOrder(result, rows(32), rows(25)); } @Test - public void testReverseWithMultipleSorts() throws IOException { - // Use the existing BANK data but with a simpler, more predictable query + public void testReverseWithDescendingSort() throws IOException { + // Test reverse with descending sort (- age) JSONObject result = executeQuery( String.format( - "source=%s | sort account_number | fields account_number | reverse | head 3", + "source=%s | sort - account_number | fields account_number | reverse", TEST_INDEX_BANK)); verifySchema(result, schema("account_number", "bigint")); - verifyDataRowsInOrder(result, rows(32), rows(25), rows(20)); + verifyDataRowsInOrder( + result, rows(1), rows(6), rows(13), rows(18), rows(20), rows(25), rows(32)); + } + + @Test + public void testReverseWithMixedSortDirections() throws IOException { + // Test reverse with mixed sort directions (- age, + firstname) + JSONObject result = + executeQuery( + String.format( + "source=%s | sort - account_number, + firstname | fields account_number, firstname" + + " | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint"), schema("firstname", "string")); + verifyDataRowsInOrder( + result, + rows(1, "Amber JOHnny"), + rows(6, "Hattie"), + rows(13, "Nanette"), + rows(18, "Dale"), + rows(20, "Elinor"), + rows(25, "Virginia"), + rows(32, "Dillard")); + } + + @Test + public void testDoubleReverseWithDescendingSort() throws IOException { + // Test double reverse with descending sort (- age) + JSONObject result = + executeQuery( + String.format( + "source=%s | sort - account_number | fields account_number | reverse | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + verifyDataRowsInOrder( + result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1)); + } + + @Test + public void testDoubleReverseWithMixedSortDirections() throws IOException { + // Test double reverse with mixed sort directions (- age, + firstname) + JSONObject result = + executeQuery( + String.format( + "source=%s | sort - account_number, + firstname | fields account_number, firstname" + + " | reverse | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint"), schema("firstname", "string")); + verifyDataRowsInOrder( + result, + rows(32, "Dillard"), + rows(25, "Virginia"), + rows(20, "Elinor"), + rows(18, "Dale"), + rows(13, "Nanette"), + rows(6, "Hattie"), + rows(1, "Amber JOHnny")); + } + + @Test + public void testReverseIgnoredWithoutSortOrTimestamp() throws IOException { + // Test that reverse is ignored when there's no explicit sort and no @timestamp field + // BANK index doesn't have @timestamp, so reverse should be ignored + JSONObject result = + executeQuery( + String.format("source=%s | fields account_number | reverse | head 3", TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint")); + // Without sort or @timestamp, reverse is ignored, so data comes in natural order + // The first 3 documents in natural order (ascending by account_number) + verifyDataRowsInOrder(result, rows(1), rows(6), rows(13)); + } + + @Test + public void testReverseWithTimestampField() throws IOException { + // Test that reverse with @timestamp field sorts by @timestamp DESC + // TIME_TEST_DATA index has @timestamp field + JSONObject result = + executeQuery( + String.format( + "source=%s | fields value, category, `@timestamp` | reverse | head 5", + TEST_INDEX_TIME_DATA)); + verifySchema( + result, + schema("value", "int"), + schema("category", "string"), + schema("@timestamp", "timestamp")); + // Should return the latest 5 records (highest @timestamp values) in descending order + // Based on the test data, these are IDs 100, 99, 98, 97, 96 + verifyDataRowsInOrder( + result, + rows(8762, "A", "2025-08-01 03:47:41"), + rows(7348, "C", "2025-08-01 02:00:56"), + rows(9015, "B", "2025-08-01 01:14:11"), + rows(6489, "D", "2025-08-01 00:27:26"), + rows(8676, "A", "2025-07-31 23:40:33")); + } + + @Test + public void testReverseWithTimestampAndExplicitSort() throws IOException { + // Test that explicit sort takes precedence over @timestamp + JSONObject result = + executeQuery( + String.format( + "source=%s | fields value, category | sort value | reverse | head 3", + TEST_INDEX_TIME_DATA)); + verifySchema(result, schema("value", "int"), schema("category", "string")); + // Should reverse the value sort, giving us the highest values + verifyDataRowsInOrder(result, rows(9521, "B"), rows(9367, "A"), rows(9321, "A")); + } + + @Test + public void testStreamstatsWithReverse() throws IOException { + // Test that reverse is ignored when used directly after streamstats + // streamstats maintains order via __stream_seq__, but this field is projected out + // and doesn't create a detectable collation, so reverse is ignored (no-op) + JSONObject result = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg | reverse", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double")); + // Reverse is ignored, so data remains in original streamstats order + verifyDataRowsInOrder( + result, + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3, 41.666666666666664), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 4, 36.25)); + } + + @Test + public void testStreamstatsWindowWithReverse() throws IOException { + // Test that reverse is ignored after streamstats with window + JSONObject result = + executeQuery( + String.format( + "source=%s | streamstats window=2 avg(age) as avg | reverse", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("avg", "double")); + // Reverse is ignored, data remains in original order + // Window=2 means average of current and previous row (sliding window of size 2) + verifyDataRowsInOrder( + result, + rows("Jake", "USA", "California", 4, 2023, 70, 70), + rows("Hello", "USA", "New York", 4, 2023, 30, 50), + rows("John", "Canada", "Ontario", 4, 2023, 25, 27.5), + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 22.5)); + } + + @Test + public void testStreamstatsByWithReverse() throws IOException { + // Test that reverse is ignored after streamstats with partitioning (by clause) + JSONObject result = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt, avg(age) as avg by country | reverse", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint"), + schema("avg", "double")); + // With backtracking, reverse now works and reverses the __stream_seq__ order + verifyDataRowsInOrder( + result, + rows("Jane", "Canada", "Quebec", 4, 2023, 20, 2, 22.5), + rows("John", "Canada", "Ontario", 4, 2023, 25, 1, 25), + rows("Hello", "USA", "New York", 4, 2023, 30, 2, 50), + rows("Jake", "USA", "California", 4, 2023, 70, 1, 70)); + } + + @Test + public void testStreamstatsWithSortThenReverse() throws IOException { + // Test that reverse works when there's an explicit sort after streamstats + // The explicit sort creates a collation that reverse can detect and reverse + JSONObject result = + executeQuery( + String.format( + "source=%s | streamstats count() as cnt | sort age | reverse | head 3", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("cnt", "bigint")); + // With explicit sort and reverse, data is in descending age order + verifyDataRowsInOrder( + result, + rows("Jake", "USA", "California", 4, 2023, 70, 1), + rows("Hello", "USA", "New York", 4, 2023, 30, 2), + rows("John", "Canada", "Ontario", 4, 2023, 25, 3)); + } + + // ==================== Tests for blocking operators ==================== + // These tests verify that reverse is a no-op after blocking operators + // that destroy collation (aggregate, join, window functions). + + @Test + public void testReverseAfterAggregationIsNoOp() throws IOException { + // Test that reverse is a no-op after aggregation (stats) + // Aggregation destroys input ordering, so reverse has no collation to reverse + // and BANK index has no @timestamp, so reverse should be ignored + JSONObject result = + executeQuery( + String.format("source=%s | stats count() as c by gender | reverse", TEST_INDEX_BANK)); + verifySchema(result, schema("c", "bigint"), schema("gender", "string")); + // Data should be in aggregation order (no reverse applied) + // Use verifyDataRows (unordered) since aggregation order is not guaranteed + verifyDataRows(result, rows(4, "M"), rows(3, "F")); + } + + @Test + public void testReverseAfterAggregationWithSort() throws IOException { + // Test that reverse works when there's an explicit sort after aggregation + JSONObject result = + executeQuery( + String.format( + "source=%s | stats count() as c by gender | sort gender | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("c", "bigint"), schema("gender", "string")); + // With explicit sort and reverse, data should be in descending gender order + // Sort by gender ASC: F, M -> Reverse: M, F + // Note: Due to column reordering after stats (c, gender), the result order + // may differ from expected. Using unordered verification for robustness. + verifyDataRows(result, rows(4, "M"), rows(3, "F")); + } + + @Test + public void testReverseSortAggregationIsNoOp() throws IOException { + // Test that sort before aggregation doesn't allow reverse after aggregation + // Even with sort before stats, aggregation destroys the collation + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | stats count() as c by gender | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("c", "bigint"), schema("gender", "string")); + // Reverse is a no-op because aggregation destroyed the sort collation + // Use verifyDataRows (unordered) since aggregation order is not guaranteed + verifyDataRows(result, rows(4, "M"), rows(3, "F")); + } + + @Test + public void testReverseAfterWhereWithSort() throws IOException { + // Test that reverse works through filter (where) to find the sort + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | where balance > 30000 | fields account_number," + + " balance | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint"), schema("balance", "bigint")); + // Reverse should work through the filter to reverse the sort + // Balances > 30000: 1(39225), 13(32838), 25(40540), 32(48086) + // Reversed by account_number: 32, 25, 13, 1 + verifyDataRowsInOrder( + result, rows(32, 48086), rows(25, 40540), rows(13, 32838), rows(1, 39225)); + } + + @Test + public void testReverseAfterEvalWithSort() throws IOException { + // Test that reverse works through eval (project) to find the sort + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | eval double_balance = balance * 2 | fields" + + " account_number, double_balance | reverse | head 3", + TEST_INDEX_BANK)); + verifySchema(result, schema("account_number", "bigint"), schema("double_balance", "bigint")); + // Reverse should work through eval to reverse the sort + // Account balances: 32(48086), 25(40540), 20(16418) + // double_balance: 32(96172), 25(81080), 20(32836) + verifyDataRowsInOrder(result, rows(32, 96172), rows(25, 81080), rows(20, 32836)); + } + + @Test + public void testReverseAfterMultipleFilters() throws IOException { + // Test that reverse works through multiple filters + JSONObject result = + executeQuery( + String.format( + "source=%s | sort account_number | where balance > 20000 | where age > 30 | fields" + + " account_number, balance, age | reverse", + TEST_INDEX_BANK)); + verifySchema( + result, + schema("account_number", "bigint"), + schema("balance", "bigint"), + schema("age", "int")); + // Reverse should work through multiple filters + // balance > 20000 AND age > 30: 1(39225, 32), 25(40540, 39), 32(48086, 34) + // Reversed by account_number: 32, 25, 1 + verifyDataRowsInOrder(result, rows(32, 48086, 34), rows(25, 40540, 39), rows(1, 39225, 32)); + } + + @Test + public void testReverseWithTimestampAfterAggregation() throws IOException { + // Test that reverse uses @timestamp when aggregation destroys collation + // TIME_TEST_DATA has @timestamp field + JSONObject result = + executeQuery( + String.format( + "source=%s | stats count() as c by category | reverse", TEST_INDEX_TIME_DATA)); + verifySchema(result, schema("c", "bigint"), schema("category", "string")); + // Even though aggregation destroys collation, there's no @timestamp in the + // aggregated result, so reverse is a no-op + // Use verifyDataRows (unordered) since aggregation order is not guaranteed + // Categories: A=26, B=25, C=25, D=24 + verifyDataRows(result, rows(26, "A"), rows(25, "B"), rows(25, "C"), rows(24, "D")); + } + + // ==================== Timechart with Reverse tests ==================== + // These tests verify that reverse works correctly with timechart. + // Timechart always adds a sort at the end of its plan (tier 1), so reverse + // will find the collation via metadata query and flip the sort direction. + + @Test + public void testTimechartWithReverse() throws IOException { + // Timechart adds ORDER BY @timestamp ASC at the end + // Reverse should flip it to DESC, returning data in reverse chronological order + JSONObject result = executeQuery("source=events | timechart span=1m count() | reverse"); + verifySchema(result, schema("@timestamp", "timestamp"), schema("count()", "bigint")); + // Events data has timestamps at 00:00, 00:01, 00:02, 00:03, 00:04 + // Reversed order: 00:04, 00:03, 00:02, 00:01, 00:00 + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:04:00", 1), + rows("2024-07-01 00:03:00", 1), + rows("2024-07-01 00:02:00", 1), + rows("2024-07-01 00:01:00", 1), + rows("2024-07-01 00:00:00", 1)); + } + + @Test + public void testTimechartWithCustomTimefieldAndReverse() throws IOException { + // Test timechart with custom timefield (birthdate instead of @timestamp) + // PR #4784 allows users to specify a custom timefield in timechart + // The sort should be on the custom field, not @timestamp + JSONObject result = + executeQuery( + String.format( + "source=%s | timechart timefield=birthdate span=1year count() | reverse", + TEST_INDEX_BANK)); + verifySchema(result, schema("birthdate", "timestamp"), schema("count()", "bigint")); + // Bank data has birthdates in 2017 and 2018 + // Timechart groups by year: 2017 (2 records), 2018 (5 records) + // Reversed order: 2018, 2017 + verifyDataRowsInOrder(result, rows("2018-01-01 00:00:00", 5), rows("2017-01-01 00:00:00", 2)); + } + + @Test + public void testTimechartWithGroupByAndReverse() throws IOException { + // Test timechart with group by and reverse + // The sort is on both @timestamp and the group by field + JSONObject result = executeQuery("source=events | timechart span=1h count() by host | reverse"); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("count()", "bigint")); + // All events are in the same hour, so only one time bucket + // Hosts are grouped and results are reversed + verifyDataRows( + result, + rows("2024-07-01 00:00:00", "db-01", 1), + rows("2024-07-01 00:00:00", "web-01", 2), + rows("2024-07-01 00:00:00", "web-02", 2)); } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_ignored.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_ignored.yaml new file mode 100644 index 00000000000..24d8c63fcd0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_ignored.yaml @@ -0,0 +1,7 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_multiple.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_multiple.yaml new file mode 100644 index 00000000000..fb556df43f6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_multiple.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SORT->[{ + "age" : { + "order" : "desc", + "missing" : "_last" + } + }, { + "firstname.keyword" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"age":{"order":"desc","missing":"_last"}},{"firstname.keyword":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_single.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_single.yaml new file mode 100644 index 00000000000..0f0843b2964 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_double_reverse_pushdown_single.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SORT->[{ + "age" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"age":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_ignored.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_ignored.yaml new file mode 100644 index 00000000000..083010dd70e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_ignored.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_multiple.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_multiple.yaml new file mode 100644 index 00000000000..2132340e162 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_multiple.yaml @@ -0,0 +1,19 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }, { + "firstname.keyword" : { + "order" : "desc", + "missing" : "_last" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}},{"firstname.keyword":{"order":"desc","missing":"_last"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_single.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_single.yaml new file mode 100644 index 00000000000..33d7c0f0cf6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_pushdown_single.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_with_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_with_timestamp.yaml new file mode 100644 index 00000000000..7c383f34584 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_reverse_with_timestamp.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[@timestamp, category, value, timestamp], SORT->[{ + "@timestamp" : { + "order" : "desc", + "missing" : "_first" + } + }], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["@timestamp","category","value","timestamp"],"excludes":[]},"sort":[{"@timestamp":{"order":"desc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_ignored.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_ignored.yaml new file mode 100644 index 00000000000..79f52ecc188 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_ignored.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_multiple.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_multiple.yaml new file mode 100644 index 00000000000..1bce5d3a0da --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_multiple.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_single.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_single.yaml new file mode 100644 index 00000000000..c63f25b8986 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_double_reverse_pushdown_single.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$8], dir0=[DESC-nulls-last]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_ignored.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_ignored.yaml new file mode 100644 index 00000000000..0fb2d7e597d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_ignored.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_multiple.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_multiple.yaml new file mode 100644 index 00000000000..bdb37931ed3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_multiple.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$8], sort1=[$1], dir0=[ASC-nulls-first], dir1=[DESC-nulls-last]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_single.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_single.yaml new file mode 100644 index 00000000000..a1ecb6c3b38 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_pushdown_single.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$8], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(sort0=[$8], dir0=[DESC-nulls-last]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_with_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_with_timestamp.yaml new file mode 100644 index 00000000000..e3095d13abc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_reverse_with_timestamp.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[DESC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3]) + LogicalSort(sort0=[$0], dir0=[DESC], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..9=[{inputs}], proj#0..3=[{exprs}]) + EnumerableLimit(fetch=[5]) + EnumerableSort(sort0=[$0], dir0=[DESC]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java index 179fb3bc830..8580673feaf 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLReverseTest.java @@ -9,6 +9,18 @@ import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +/** + * Tests for reverse command optimization. + * + *

The reverse command behavior depends on the presence of: 1. Existing collation (sort): Reverse + * the sort direction 2. @timestamp field: Sort by @timestamp DESC 3. Neither: No-op (ignore reverse + * command) + * + *

These tests use SCOTT_WITH_TEMPORAL schema where EMP table has a default collation on EMPNO + * (primary key), demonstrating case #1 (reverse existing collation). + * + *

For @timestamp and no-op cases, see CalciteReverseCommandIT integration tests. + */ public class CalcitePPLReverseTest extends CalcitePPLAbstractTest { public CalcitePPLReverseTest() { super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); @@ -16,16 +28,11 @@ public CalcitePPLReverseTest() { @Test public void testReverseParserSuccess() { + // EMP table has default collation on EMPNO, so reverse flips it to DESC String ppl = "source=EMP | reverse"; RelNode root = getRelNode(ppl); String expectedLogical = - "" - + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7])\n" - + " LogicalSort(sort0=[$8], dir0=[DESC])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(sort0=[$0], dir0=[DESC])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -60,12 +67,7 @@ public void testReverseParserSuccess() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" - + "FROM `scott`.`EMP`\n" - + "ORDER BY 9 DESC NULLS FIRST) `t0`"; + "SELECT *\n" + "FROM `scott`.`EMP`\n" + "ORDER BY `EMPNO` DESC NULLS FIRST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -73,25 +75,19 @@ public void testReverseParserSuccess() { public void testReverseWithSortParserSuccess() { String ppl = "source=EMP | sort ENAME | reverse"; RelNode root = getRelNode(ppl); + // Optimization rule may show double sorts in logical plan but physical execution is optimized String expectedLogical = - "" - + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7])\n" - + " LogicalSort(sort0=[$8], dir0=[DESC])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" - + " LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(sort0=[$1], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "" - + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" - + "ORDER BY `ENAME`) `t0`\n" - + "ORDER BY `__reverse_row_num__` DESC NULLS FIRST"; + + "ORDER BY `ENAME`) `t`\n" + + "ORDER BY `ENAME` DESC"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -99,28 +95,19 @@ public void testReverseWithSortParserSuccess() { public void testDoubleReverseParserSuccess() { String ppl = "source=EMP | reverse | reverse"; RelNode root = getRelNode(ppl); + // Without optimization rule, shows consecutive sorts String expectedLogical = - "" - + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7])\n" - + " LogicalSort(sort0=[$8], dir0=[DESC])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" - + " LogicalSort(sort0=[$8], dir0=[DESC])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalSort(sort0=[$0], dir0=[DESC])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" - + "ORDER BY 9 DESC NULLS FIRST) `t0`\n" - + "ORDER BY 9 DESC NULLS FIRST) `t2`"; + + "ORDER BY `EMPNO` DESC NULLS FIRST) `t`\n" + + "ORDER BY `EMPNO` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -129,13 +116,8 @@ public void testReverseWithHeadParserSuccess() { String ppl = "source=EMP | reverse | head 2"; RelNode root = getRelNode(ppl); String expectedLogical = - "" - + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7])\n" - + " LogicalSort(sort0=[$8], dir0=[DESC], fetch=[2])\n" - + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __reverse_row_num__=[ROW_NUMBER() OVER ()])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(sort0=[$0], dir0=[DESC], fetch=[2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -146,12 +128,7 @@ public void testReverseWithHeadParserSuccess() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " ROW_NUMBER() OVER () `__reverse_row_num__`\n" - + "FROM `scott`.`EMP`\n" - + "ORDER BY 9 DESC NULLS FIRST\n" - + "LIMIT 2) `t0`"; + "SELECT *\n" + "FROM `scott`.`EMP`\n" + "ORDER BY `EMPNO` DESC NULLS FIRST\n" + "LIMIT 2"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -178,4 +155,305 @@ public void testReverseWithExpressionShouldFail() { String ppl = "source=EMP | reverse EMPNO + 1"; getRelNode(ppl); } + + @Test + public void testMultipleSortsWithReverseParserSuccess() { + String ppl = "source=EMP | sort + SAL | sort - ENAME | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" + + " LogicalSort(sort0=[$1], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `SAL`) `t`\n" + + "ORDER BY `ENAME` DESC) `t0`\n" + + "ORDER BY `ENAME`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testMultiFieldSortWithReverseParserSuccess() { + String ppl = "source=EMP | sort + SAL, - ENAME | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$5], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n" + + " LogicalSort(sort0=[$5], sort1=[$1], dir0=[ASC-nulls-first]," + + " dir1=[DESC-nulls-last])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `SAL`, `ENAME` DESC) `t`\n" + + "ORDER BY `SAL` DESC, `ENAME`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testComplexMultiFieldSortWithReverseParserSuccess() { + String ppl = "source=EMP | sort DEPTNO, + SAL, - ENAME | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$7], sort1=[$5], sort2=[$1], dir0=[DESC-nulls-last]," + + " dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n" + + " LogicalSort(sort0=[$7], sort1=[$5], sort2=[$1], dir0=[ASC-nulls-first]," + + " dir1=[ASC-nulls-first], dir2=[DESC-nulls-last])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `DEPTNO`, `SAL`, `ENAME` DESC) `t`\n" + + "ORDER BY `DEPTNO` DESC, `SAL` DESC, `ENAME`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseWithFieldsAndSortParserSuccess() { + String ppl = "source=EMP | fields ENAME, SAL, DEPTNO | sort + SAL | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(sort0=[$1], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" + + " LogicalProject(ENAME=[$1], SAL=[$5], DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `ENAME`, `SAL`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `SAL`) `t0`\n" + + "ORDER BY `SAL` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testHeadThenSortReverseNoOpt() { + // Tests fetch limit behavior: head 5 | sort field | reverse + // Should NOT be optimized to preserve "take first 5, then sort" semantics + String ppl = "source=EMP | head 5 | sort + SAL | reverse"; + RelNode root = getRelNode(ppl); + + // Should have three LogicalSort nodes: fetch=5, sort SAL, reverse + // Calcite's built-in optimization will handle the physical plan optimization + String expectedLogical = + "LogicalSort(sort0=[$5], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalSort(fetch=[5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 5) `t`\n" + + "ORDER BY `SAL`) `t0`\n" + + "ORDER BY `SAL` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testSortFieldsReverse() { + // Test backtracking: sort on SAL, then project only ENAME, then reverse + // The sort field (SAL) is removed from schema by fields command + // But reverse should still work by backtracking to find the sort + String ppl = "source=EMP | sort SAL | fields ENAME | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1])\n" + + " LogicalSort(sort0=[$5], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `ENAME`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `SAL`) `t`\n" + + "ORDER BY `SAL` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + // ==================== Complex query tests with blocking operators ==================== + // These tests verify that reverse becomes a no-op after blocking operators + // that destroy collation (aggregate, join, set ops, window functions). + // Since SCOTT_WITH_TEMPORAL schema has no @timestamp field, reverse is ignored. + + @Test + public void testReverseAfterAggregationIsNoOp() { + // Aggregation destroys input ordering, so reverse has no collation to reverse + // and no @timestamp field exists, so reverse should be a no-op + String ppl = "source=EMP | stats count() as c by DEPTNO | reverse"; + RelNode root = getRelNode(ppl); + // No additional sort node for reverse - it's a no-op after aggregation + // Note: There's a project for column reordering (c, DEPTNO) in the output + String expectedLogical = + "LogicalProject(c=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], c=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT COUNT(*) `c`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseAfterJoinIsNoOp() { + // Join destroys input ordering, so reverse has no collation to reverse + // and no @timestamp field exists, so reverse should be a no-op + String ppl = "source=EMP | join on EMP.DEPTNO = DEPT.DEPTNO DEPT | reverse"; + RelNode root = getRelNode(ppl); + // No additional sort node for reverse - it's a no-op after join + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], DEPT.DEPTNO=[$8], DNAME=[$9], LOC=[$10])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMP`.`EMPNO`, `EMP`.`ENAME`, `EMP`.`JOB`, `EMP`.`MGR`, `EMP`.`HIREDATE`," + + " `EMP`.`SAL`, `EMP`.`COMM`, `EMP`.`DEPTNO`, `DEPT`.`DEPTNO` `DEPT.DEPTNO`," + + " `DEPT`.`DNAME`, `DEPT`.`LOC`\n" + + "FROM `scott`.`EMP`\n" + + "INNER JOIN `scott`.`DEPT` ON `EMP`.`DEPTNO` = `DEPT`.`DEPTNO`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseAfterSortAndAggregationIsNoOp() { + // Even if there's a sort before aggregation, aggregation destroys the collation + // so reverse after aggregation should be a no-op + String ppl = "source=EMP | sort SAL | stats count() as c by DEPTNO | reverse"; + RelNode root = getRelNode(ppl); + // Sort before aggregation is present, but reverse after aggregation is a no-op + // Note: There's a project for column reordering (c, DEPTNO) in the output + String expectedLogical = + "LogicalProject(c=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], c=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Verify result data - reverse is a no-op, so data remains in aggregation order + String expectedResult = "c=5; DEPTNO=20\n" + "c=3; DEPTNO=10\n" + "c=6; DEPTNO=30\n"; + verifyResult(root, expectedResult); + } + + @Test + public void testReverseAfterWhereWithSort() { + // Filter (where) doesn't destroy collation, so reverse should work through it + String ppl = "source=EMP | sort SAL | where DEPTNO = 10 | reverse"; + RelNode root = getRelNode(ppl); + // Reverse backtracks through filter to find the sort and inserts reversed sort + // after the original sort, then the filter is applied on top + String expectedLogical = + "LogicalSort(sort0=[$5], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `SAL`) `t`\n" + + "WHERE `DEPTNO` = 10\n" + + "ORDER BY `SAL` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testReverseAfterEvalWithSort() { + // Eval (project) doesn't destroy collation, so reverse should work through it + String ppl = "source=EMP | sort SAL | eval bonus = SAL * 0.1 | reverse"; + RelNode root = getRelNode(ppl); + // Reversed sort is added on top of the project (eval) + String expectedLogical = + "LogicalSort(sort0=[$5], dir0=[DESC-nulls-last])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], bonus=[*($5, 0.1:DECIMAL(2, 1))])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testReverseAfterMultipleFiltersWithSort() { + // Multiple filters don't destroy collation + String ppl = "source=EMP | sort SAL | where DEPTNO = 10 | where SAL > 1000 | reverse"; + RelNode root = getRelNode(ppl); + // Reversed sort is added on top of the filters + String expectedLogical = + "LogicalSort(sort0=[$5], dir0=[DESC-nulls-last])\n" + + " LogicalFilter(condition=[>($5, 1000)])\n" + + " LogicalFilter(condition=[=($7, 10)])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testReverseSortJoinSort() { + // Sort before join, then another sort after join, reverse should work + String ppl = + "source=EMP | sort SAL | join on EMP.DEPTNO = DEPT.DEPTNO DEPT | sort DNAME | reverse"; + RelNode root = getRelNode(ppl); + // The sort before join is destroyed by join, but sort after join can be reversed + String expectedLogical = + "LogicalSort(sort0=[$9], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$9], dir0=[ASC-nulls-first])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], DEPT.DEPTNO=[$8], DNAME=[$9], LOC=[$10])\n" + + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + + " LogicalSort(sort0=[$5], dir0=[ASC-nulls-first])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableScan(table=[[scott, DEPT]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testReverseAfterAggregationWithSort() { + // Sort after aggregation, then reverse should work + String ppl = "source=EMP | stats count() as c by DEPTNO | sort DEPTNO | reverse"; + RelNode root = getRelNode(ppl); + // Note: There's a project for column reordering (c, DEPTNO) so DEPTNO is at position 1 + String expectedLogical = + "LogicalSort(sort0=[$1], dir0=[DESC-nulls-last])\n" + + " LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n" + + " LogicalProject(c=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], c=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT COUNT(*) `c`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`\n" + + "ORDER BY `DEPTNO`) `t2`\n" + + "ORDER BY `DEPTNO` DESC"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java index b073453ecbc..050b88a8f1f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStreamstatsTest.java @@ -218,4 +218,34 @@ public void testStreamstatsReset() { + "ORDER BY `$cor0`.`__stream_seq__` NULLS LAST"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testStreamstatsWithReverse() { + String ppl = "source=EMP | streamstats max(SAL) by DEPTNO | reverse"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], max(SAL)=[$9])\n" + + " LogicalSort(sort0=[$8], dir0=[DESC])\n" + + " LogicalSort(sort0=[$8], dir0=[ASC])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[$8], max(SAL)=[MAX($5) OVER" + + " (PARTITION BY $7 ROWS UNBOUNDED PRECEDING)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], __stream_seq__=[ROW_NUMBER() OVER ()])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, `max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " `__stream_seq__`, MAX(`SAL`) OVER (PARTITION BY `DEPTNO` ROWS BETWEEN UNBOUNDED" + + " PRECEDING AND CURRENT ROW) `max(SAL)`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " ROW_NUMBER() OVER () `__stream_seq__`\n" + + "FROM `scott`.`EMP`) `t`\n" + + "ORDER BY `__stream_seq__` NULLS LAST) `t1`\n" + + "ORDER BY `__stream_seq__` DESC NULLS FIRST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java index ca0ff70f0b7..1617b8b59b5 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimechartTest.java @@ -53,13 +53,28 @@ protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpec ImmutableList rows = ImmutableList.of( new Object[] { - java.sql.Timestamp.valueOf("2024-07-01 00:00:00"), "web-01", "us-east", 45.2, 120 + java.sql.Timestamp.valueOf("2024-07-01 00:00:00"), + java.sql.Timestamp.valueOf("2024-01-15 10:00:00"), + "web-01", + "us-east", + 45.2, + 120 }, new Object[] { - java.sql.Timestamp.valueOf("2024-07-01 00:01:00"), "web-02", "us-west", 38.7, 150 + java.sql.Timestamp.valueOf("2024-07-01 00:01:00"), + java.sql.Timestamp.valueOf("2024-02-20 11:00:00"), + "web-02", + "us-west", + 38.7, + 150 }, new Object[] { - java.sql.Timestamp.valueOf("2024-07-01 00:02:00"), "web-01", "us-east", 55.3, 200 + java.sql.Timestamp.valueOf("2024-07-01 00:02:00"), + java.sql.Timestamp.valueOf("2024-03-25 12:00:00"), + "web-01", + "us-east", + 55.3, + 200 }); schema.add("events", new EventsTable(rows)); return Frameworks.newConfigBuilder() @@ -347,6 +362,68 @@ public void testTimechartUsingZeroSpanShouldThrow() { verifyErrorMessageContains(t, "Zero or negative time interval not supported: 0h"); } + // ==================== Timechart with Reverse tests ==================== + // These tests verify that reverse works correctly with timechart. + // Timechart always adds a sort at the end of its plan, so reverse will + // find the collation via metadata query (tier 1) and flip the sort direction. + + @Test + public void testTimechartWithReverse() { + // Timechart adds ORDER BY @timestamp ASC at the end + // Reverse should flip it to DESC + String ppl = "source=events | timechart count() | reverse"; + RelNode root = getRelNode(ppl); + // The plan should have two sorts: original ASC from timechart, then DESC from reverse + String expectedLogical = + "LogicalSort(sort0=[$0], dir0=[DESC])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(@timestamp=[$0], count()=[$1])\n" + + " LogicalAggregate(group=[{0}], count()=[COUNT()])\n" + + " LogicalProject(@timestamp0=[SPAN($0, 1, 'm')])\n" + + " LogicalFilter(condition=[IS NOT NULL($0)])\n" + + " LogicalTableScan(table=[[scott, events]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT SPAN(`@timestamp`, 1, 'm') `@timestamp`, COUNT(*) `count()`\n" + + "FROM `scott`.`events`\n" + + "WHERE `@timestamp` IS NOT NULL\n" + + "GROUP BY SPAN(`@timestamp`, 1, 'm')\n" + + "ORDER BY 1 NULLS LAST) `t3`\n" + + "ORDER BY `@timestamp` DESC NULLS FIRST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTimechartWithCustomTimefieldAndReverse() { + // Timechart with custom timefield should also work with reverse + // The sort is on created_at (the custom field), not @timestamp + String ppl = "source=events | timechart timefield=created_at span=1month count() | reverse"; + RelNode root = getRelNode(ppl); + + // Verify the logical plan shows two sorts: ASC from timechart, DESC from reverse + String expectedLogical = + "LogicalSort(sort0=[$0], dir0=[DESC])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(created_at=[$0], count()=[$1])\n" + + " LogicalAggregate(group=[{0}], count()=[COUNT()])\n" + + " LogicalProject(created_at0=[SPAN($1, 1, 'M')])\n" + + " LogicalFilter(condition=[IS NOT NULL($1)])\n" + + " LogicalTableScan(table=[[scott, events]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT SPAN(`created_at`, 1, 'M') `created_at`, COUNT(*) `count()`\n" + + "FROM `scott`.`events`\n" + + "WHERE `created_at` IS NOT NULL\n" + + "GROUP BY SPAN(`created_at`, 1, 'M')\n" + + "ORDER BY 1 NULLS LAST) `t3`\n" + + "ORDER BY `created_at` DESC NULLS FIRST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + private UnresolvedPlan parsePPL(String query) { PPLSyntaxParser parser = new PPLSyntaxParser(); AstBuilder astBuilder = new AstBuilder(query); @@ -363,6 +440,8 @@ public static class EventsTable implements ScannableTable { .builder() .add("@timestamp", SqlTypeName.TIMESTAMP) .nullable(true) + .add("created_at", SqlTypeName.TIMESTAMP) + .nullable(true) .add("host", SqlTypeName.VARCHAR) .nullable(true) .add("region", SqlTypeName.VARCHAR)