From c8a8d162a0570a390dea178e64c6d6a36c278401 Mon Sep 17 00:00:00 2001 From: Xiao Du Date: Sun, 15 Mar 2026 23:11:00 -0700 Subject: [PATCH] feat: Add session property mark_distinct_spill_enabled (#27247) Summary: per title Differential Revision: D94965529 --- .../src/main/sphinx/presto_cpp/properties-session.rst | 8 ++++++++ .../NativeWorkerSessionPropertyProvider.java | 6 ++++++ .../presto_cpp/main/SessionProperties.cpp | 8 ++++++++ .../presto_cpp/main/SessionProperties.h | 4 ++++ .../presto_cpp/main/tests/SessionPropertiesTest.cpp | 2 ++ 5 files changed, 28 insertions(+) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index fd2357cf3f19e..f0ac0af05a8aa 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -193,6 +193,14 @@ Use this threshold to manage memory usage more efficiently during `ORDER BY` ope Native Execution only. Enable row number spilling on native engine. +``native_mark_distinct_spill_enabled`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +Native Execution only. Enable mark distinct spilling on native engine. + ``native_simplified_expression_evaluation_enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 054f458460b03..1e61845edee41 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -46,6 +46,7 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_WRITER_SPILL_ENABLED = "native_writer_spill_enabled"; public static final String NATIVE_WRITER_FLUSH_THRESHOLD_BYTES = "native_writer_flush_threshold_bytes"; public static final String NATIVE_ROW_NUMBER_SPILL_ENABLED = "native_row_number_spill_enabled"; + public static final String NATIVE_MARK_DISTINCT_SPILL_ENABLED = "native_mark_distinct_spill_enabled"; public static final String NATIVE_TOPN_ROW_NUMBER_SPILL_ENABLED = "native_topn_row_number_spill_enabled"; public static final String NATIVE_SPILLER_NUM_PARTITION_BITS = "native_spiller_num_partition_bits"; public static final String NATIVE_DEBUG_VALIDATE_OUTPUT_FROM_OPERATORS = "native_debug_validate_output_from_operators"; @@ -189,6 +190,11 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "Native Execution only. Enable row number spilling on native engine", false, !nativeExecution), + booleanProperty( + NATIVE_MARK_DISTINCT_SPILL_ENABLED, + "Native Execution only. Enable mark distinct spilling on native engine", + false, + !nativeExecution), booleanProperty( NATIVE_TOPN_ROW_NUMBER_SPILL_ENABLED, "Native Execution only. Enable topN row number spilling on native engine", diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index b9a45fd40226f..ac01207a37e23 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -205,6 +205,14 @@ SessionProperties::SessionProperties() { QueryConfig::kRowNumberSpillEnabled, boolToString(c.rowNumberSpillEnabled())); + addSessionProperty( + kMarkDistinctSpillEnabled, + "Native Execution only. Enable mark distinct spilling on native engine", + BOOLEAN(), + false, + QueryConfig::kMarkDistinctSpillEnabled, + boolToString(c.markDistinctSpillEnabled())); + addSessionProperty( kSpillerNumPartitionBits, "The number of bits (N) used to calculate the spilling " diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index aa8a5a93b3b14..29d98540b05ac 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -114,6 +114,10 @@ class SessionProperties { static constexpr const char* kRowNumberSpillEnabled = "native_row_number_spill_enabled"; + /// Enable mark distinct spilling on native engine. + static constexpr const char* kMarkDistinctSpillEnabled = + "native_mark_distinct_spill_enabled"; + /// The compression algorithm type to compress the spilled data. static constexpr const char* kSpillCompressionCodec = "native_spill_compression_codec"; diff --git a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp index 546c79c866597..f46dd0367347c 100644 --- a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp @@ -57,6 +57,8 @@ TEST_F(SessionPropertiesTest, validateMapping) { core::QueryConfig::kWriterFlushThresholdBytes}, {SessionProperties::kRowNumberSpillEnabled, core::QueryConfig::kRowNumberSpillEnabled}, + {SessionProperties::kMarkDistinctSpillEnabled, + core::QueryConfig::kMarkDistinctSpillEnabled}, {SessionProperties::kSpillerNumPartitionBits, core::QueryConfig::kSpillNumPartitionBits}, {SessionProperties::kTopNRowNumberSpillEnabled,