Skip to content

feat(optimizer): Native TopNRank optimization #24138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aditi-pandit merged 2 commits into from
Feb 12, 2026
Merged

feat(optimizer): Native TopNRank optimization #24138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc06f0b
feat(native): Add TopNRowNumberNode rank function plumbing to Velox
aditi-pandit Nov 6, 2025
a43be71
feat(optimizer): Add TopN Rank pushdown
aditi-pandit Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions presto-docs/src/main/sphinx/presto_cpp/properties-session.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,3 +588,17 @@ with StringView type during global aggregation.
Native Execution only. Ratio of unused (evicted) bytes to total bytes that triggers
compaction. The value is in the range of [0, 1). Currently only applies to
approx_most_frequent aggregate with StringView type during global aggregation.

``optimizer.optimize_top_n_rank``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

* **Type:** ``boolean``
* **Default value:** ``false``

If this is true, then filter and limit queries for ``n`` rows of
``rank()`` and ``dense_rank()`` window function values are executed
with a special TopNRowNumber operator instead of the
WindowFunction operator.

The TopNRowNumber operator is more efficient than window as
it has a streaming behavior and does not need to buffer all input rows.
11 changes: 11 additions & 0 deletions presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ public final class SystemSessionProperties
public static final String ADAPTIVE_PARTIAL_AGGREGATION = "adaptive_partial_aggregation";
public static final String ADAPTIVE_PARTIAL_AGGREGATION_ROWS_REDUCTION_RATIO_THRESHOLD = "adaptive_partial_aggregation_unique_rows_ratio_threshold";
public static final String OPTIMIZE_TOP_N_ROW_NUMBER = "optimize_top_n_row_number";
public static final String OPTIMIZE_TOP_N_RANK = "optimize_top_n_rank";
public static final String OPTIMIZE_CASE_EXPRESSION_PREDICATE = "optimize_case_expression_predicate";
public static final String MAX_GROUPING_SETS = "max_grouping_sets";
public static final String LEGACY_UNNEST = "legacy_unnest";
Expand Down Expand Up @@ -1004,6 +1005,11 @@ public SystemSessionProperties(
"Use top N row number optimization",
featuresConfig.isOptimizeTopNRowNumber(),
false),
booleanProperty(
OPTIMIZE_TOP_N_RANK,
"Use top N rank and dense_rank optimization",
featuresConfig.isOptimizeTopNRank(),
false),
booleanProperty(
OPTIMIZE_CASE_EXPRESSION_PREDICATE,
"Optimize case expression predicates",
Expand Down Expand Up @@ -2694,6 +2700,11 @@ public static boolean isOptimizeTopNRowNumber(Session session)
return session.getSystemProperty(OPTIMIZE_TOP_N_ROW_NUMBER, Boolean.class);
}

public static boolean isOptimizeTopNRank(Session session)
{
return session.getSystemProperty(OPTIMIZE_TOP_N_RANK, Boolean.class);
}

public static boolean isOptimizeCaseExpressionPredicate(Session session)
{
return session.getSystemProperty(OPTIMIZE_CASE_EXPRESSION_PREDICATE, Boolean.class);
Expand Down
14 changes: 14 additions & 0 deletions presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ public class FeaturesConfig
private boolean adaptivePartialAggregationEnabled;
private double adaptivePartialAggregationRowsReductionRatioThreshold = 0.8;
private boolean optimizeTopNRowNumber = true;

private boolean optimizeTopNRank;
private boolean pushLimitThroughOuterJoin = true;
private boolean optimizeConstantGroupingKeys = true;

Expand Down Expand Up @@ -1195,13 +1197,25 @@ public boolean isOptimizeTopNRowNumber()
return optimizeTopNRowNumber;
}

public boolean isOptimizeTopNRank()
{
return optimizeTopNRank;
}

@Config("optimizer.optimize-top-n-row-number")
public FeaturesConfig setOptimizeTopNRowNumber(boolean optimizeTopNRowNumber)
{
this.optimizeTopNRowNumber = optimizeTopNRowNumber;
return this;
}

@Config("optimizer.optimize-top-n-rank")
public FeaturesConfig setOptimizeTopNRank(boolean optimizeTopNRank)
{
this.optimizeTopNRank = optimizeTopNRank;
return this;
}

public boolean isOptimizeCaseExpressionPredicate()
{
return optimizeCaseExpressionPredicate;
Expand Down
1 change: 1 addition & 0 deletions presto-main-base/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,7 @@ public Optional<PlanNode> visitTopNRowNumber(TopNRowNumberNode node, Context con
new DataOrganizationSpecification(
partitionBy,
node.getSpecification().getOrderingScheme().map(scheme -> getCanonicalOrderingScheme(scheme, context.getExpressions()))),
node.getRankingFunction(),
rowNumberVariable,
node.getMaxRowCountPerPartition(),
node.isPartial(),
Expand Down
1 change: 1 addition & 0 deletions ...rc/main/java/com/facebook/presto/sql/planner/iterative/rule/MinMaxByToWindowFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ else if (!maxByAggregations.isEmpty() && minByAggregations.isEmpty()) {
node.getStatsEquivalentPlanNode(),
node.getSource(),
dataOrganizationSpecification,
TopNRowNumberNode.RankingFunction.ROW_NUMBER,
rowNumberVariable,
1,
false,
Expand Down
1 change: 1 addition & 0 deletions ...o-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,7 @@ public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, PreferredPr
idAllocator.getNextId(),
child.getNode(),
node.getSpecification(),
node.getRankingFunction(),
node.getRowNumberVariable(),
node.getMaxRowCountPerPartition(),
true,
Expand Down
1 change: 1 addition & 0 deletions .../src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, HashComputa
node.getId(),
child.getNode(),
node.getSpecification(),
node.getRankingFunction(),
node.getRowNumberVariable(),
node.getMaxRowCountPerPartition(),
node.isPartial(),
Expand Down
1 change: 1 addition & 0 deletions ...ase/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanNodeDecorrelator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,7 @@ public Optional<DecorrelationResult> visitTopN(TopNNode node, Void context)
new DataOrganizationSpecification(
ImmutableList.copyOf(childDecorrelationResult.variablesToPropagate),
Optional.of(orderingScheme)),
TopNRowNumberNode.RankingFunction.ROW_NUMBER,
variableAllocator.newVariable("row_number", BIGINT),
toIntExact(node.getCount()),
false,
Expand Down
1 change: 1 addition & 0 deletions ...src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,7 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext<Set<Va
node.getStatsEquivalentPlanNode(),
source,
node.getSpecification(),
node.getRankingFunction(),
node.getRowNumberVariable(),
node.getMaxRowCountPerPartition(),
node.isPartial(),
Expand Down
1 change: 1 addition & 0 deletions .../src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,7 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext<Void>
node.getId(),
context.rewrite(node.getSource()),
canonicalizeAndDistinct(node.getSpecification()),
node.getRankingFunction(),
canonicalize(node.getRowNumberVariable()),
node.getMaxRowCountPerPartition(),
node.isPartial(),
Expand Down
97 changes: 77 additions & 20 deletions ...ase/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,14 @@
import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator;
import com.facebook.presto.sql.relational.RowExpressionDomainTranslator;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;

import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;

import static com.facebook.presto.SystemSessionProperties.isNativeExecutionEnabled;
import static com.facebook.presto.SystemSessionProperties.isOptimizeTopNRank;
import static com.facebook.presto.SystemSessionProperties.isOptimizeTopNRowNumber;
import static com.facebook.presto.common.predicate.Marker.Bound.BELOW;
import static com.facebook.presto.common.type.BigintType.BIGINT;
Expand Down Expand Up @@ -134,6 +137,12 @@ public PlanNode visitWindow(WindowNode node, RewriteContext<Void> context)
return replaceChildren(node, ImmutableList.of(rewrittenSource));
}

private boolean canReplaceWithTopNRowNumber(WindowNode node)
{
return (canOptimizeRowNumberFunction(node, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) ||
(isNativeExecutionEnabled(session) && canOptimizeRankFunction(node, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRank(session));
}

@Override
public PlanNode visitLimit(LimitNode node, RewriteContext<Void> context)
{
Expand All @@ -152,16 +161,22 @@ public PlanNode visitLimit(LimitNode node, RewriteContext<Void> context)
planChanged = true;
source = rowNumberNode;
}
else if (source instanceof WindowNode && canOptimizeWindowFunction((WindowNode) source, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) {
else if (source instanceof WindowNode) {
WindowNode windowNode = (WindowNode) source;
// verify that unordered row_number window functions are replaced by RowNumberNode
verify(windowNode.getOrderingScheme().isPresent());
TopNRowNumberNode topNRowNumberNode = convertToTopNRowNumber(windowNode, limit);
if (windowNode.getPartitionBy().isEmpty()) {
return topNRowNumberNode;
if (canReplaceWithTopNRowNumber(windowNode)) {
// Unordered row_number window functions are replaced by RowNumberNode and
// only rank/dense_rank with ordering schema are optimized.
verify(windowNode.getOrderingScheme().isPresent());

TopNRowNumberNode topNRowNumberNode = convertToTopNRowNumber(windowNode, limit);
planChanged = true;
// Limit can be entirely skipped for row_number without partitioning (not for rank/dense_rank).
if (windowNode.getPartitionBy().isEmpty() &&
canOptimizeRowNumberFunction(windowNode, metadata.getFunctionAndTypeManager())) {
return topNRowNumberNode;
}
source = topNRowNumberNode;
}
planChanged = true;
source = topNRowNumberNode;
}
return replaceChildren(node, ImmutableList.of(source));
}
Expand All @@ -183,15 +198,17 @@ public PlanNode visitFilter(FilterNode node, RewriteContext<Void> context)
return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt());
}
}
else if (source instanceof WindowNode && canOptimizeWindowFunction((WindowNode) source, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) {
else if (source instanceof WindowNode) {
WindowNode windowNode = (WindowNode) source;
VariableReferenceExpression rowNumberVariable = getOnlyElement(windowNode.getCreatedVariable());
OptionalInt upperBound = extractUpperBound(tupleDomain, rowNumberVariable);

if (upperBound.isPresent()) {
source = convertToTopNRowNumber(windowNode, upperBound.getAsInt());
planChanged = true;
return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt());
if (canReplaceWithTopNRowNumber(windowNode)) {
VariableReferenceExpression rowNumberVariable = getOnlyElement(windowNode.getCreatedVariable());
OptionalInt upperBound = extractUpperBound(tupleDomain, rowNumberVariable);

if (upperBound.isPresent()) {
source = convertToTopNRowNumber(windowNode, upperBound.getAsInt());
planChanged = true;
return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt());
}
}
}
return replaceChildren(node, ImmutableList.of(source));
Expand Down Expand Up @@ -275,11 +292,30 @@ private static RowNumberNode mergeLimit(RowNumberNode node, int newRowCountPerPa

private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limit)
{
String windowFunction = Iterables.getOnlyElement(windowNode.getWindowFunctions().values()).getFunctionCall().getFunctionHandle().getName();
String[] parts = windowFunction.split("\\.");
String windowFunctionName = parts[parts.length - 1];
TopNRowNumberNode.RankingFunction rankingFunction;
switch (windowFunctionName) {
case "row_number":
rankingFunction = TopNRowNumberNode.RankingFunction.ROW_NUMBER;
break;
case "rank":
rankingFunction = TopNRowNumberNode.RankingFunction.RANK;
break;
case "dense_rank":
rankingFunction = TopNRowNumberNode.RankingFunction.DENSE_RANK;
break;
default:
throw new IllegalArgumentException("Unsupported window function for TopNRowNumberNode: " + windowFunctionName);
}

return new TopNRowNumberNode(
windowNode.getSourceLocation(),
idAllocator.getNextId(),
windowNode.getSource(),
windowNode.getSpecification(),
rankingFunction,
getOnlyElement(windowNode.getCreatedVariable()),
limit,
false,
Expand All @@ -288,22 +324,43 @@ private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limi

private static boolean canReplaceWithRowNumber(WindowNode node, FunctionAndTypeManager functionAndTypeManager)
{
return canOptimizeWindowFunction(node, functionAndTypeManager) && !node.getOrderingScheme().isPresent();
return canOptimizeRowNumberFunction(node, functionAndTypeManager) && !node.getOrderingScheme().isPresent();
}

private static boolean canOptimizeWindowFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager)
private static boolean canOptimizeRowNumberFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager)
{
if (node.getWindowFunctions().size() != 1) {
return false;
}
VariableReferenceExpression rowNumberVariable = getOnlyElement(node.getWindowFunctions().keySet());
return isRowNumberMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(node.getWindowFunctions().get(rowNumberVariable).getFunctionHandle()));
return isRowNumberMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(getOnlyElement(node.getWindowFunctions().values()).getFunctionHandle()));
}

private static boolean canOptimizeRankFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager)
{
if (node.getWindowFunctions().size() != 1) {
return false;
}

// This optimization requires an ordering scheme for the rank functions.
if (!node.getOrderingScheme().isPresent()) {
return false;
}

return isRankMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(getOnlyElement(node.getWindowFunctions().values()).getFunctionHandle()));
}

private static boolean isRowNumberMetadata(FunctionAndTypeManager functionAndTypeManager, FunctionMetadata functionMetadata)
{
FunctionHandle rowNumberFunction = functionAndTypeManager.lookupFunction("row_number", ImmutableList.of());
return functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(rowNumberFunction));
}

private static boolean isRankMetadata(FunctionAndTypeManager functionAndTypeManager, FunctionMetadata functionMetadata)
{
FunctionHandle rankFunction = functionAndTypeManager.lookupFunction("rank", ImmutableList.of());
FunctionHandle denseRankFunction = functionAndTypeManager.lookupFunction("dense_rank", ImmutableList.of());
return functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(rankFunction)) ||
functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(denseRankFunction));
}
}
}
Loading
Loading