diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index 4718c59f64059..ebd6c4153070f 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -240,3 +240,21 @@ Native Execution only. Enable window spilling on native engine. * **Default value:** ``true`` Native Execution only. Enable writer spilling on native engine. + +``native_max_output_buffer_size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``bigint`` +* **Default value:** ``33554432`` + +The maximum size in bytes for the task's buffered output. The buffer is shared among all drivers. Default is 32MB. + +``native_max_page_partitioning_buffer_size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``bigint`` +* **Default value:** ``33554432`` + +The maximum bytes to buffer per PartitionedOutput operator to avoid creating tiny SerializedPages. +For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator would buffer up to that number of +bytes / number of destinations for each destination before producing a SerializedPage. Default is 32MB. diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 2981ec7a86830..c22c7e80f06a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -366,6 +366,8 @@ public final class SystemSessionProperties public static final String NATIVE_QUERY_TRACE_NODE_IDS = "native_query_trace_node_ids"; public static final String NATIVE_QUERY_TRACE_MAX_BYTES = "native_query_trace_max_bytes"; public static final String NATIVE_QUERY_TRACE_REG_EXP = "native_query_trace_task_reg_exp"; + public static final String NATIVE_MAX_PAGE_PARTITIONING_BUFFER_SIZE = "native_max_page_partitioning_buffer_size"; + public static final String NATIVE_MAX_OUTPUT_BUFFER_SIZE = "native_max_output_buffer_size"; public static final String DEFAULT_VIEW_SECURITY_MODE = "default_view_security_mode"; public static final String JOIN_PREFILTER_BUILD_SIDE = "join_prefilter_build_side"; @@ -1805,6 +1807,17 @@ public SystemSessionProperties( "The regexp of traced task id. We only enable trace on a task if its id matches.", "", false), + longProperty(NATIVE_MAX_OUTPUT_BUFFER_SIZE, + "The maximum size in bytes for the task's buffered output. The buffer is shared among all drivers.", + 200L << 20, + false), + longProperty(NATIVE_MAX_PAGE_PARTITIONING_BUFFER_SIZE, + "The maximum bytes to buffer per PartitionedOutput operator to avoid creating tiny " + + "SerializedPages. For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator " + + "would buffer up to that number of bytes / number of destinations for each destination before " + + "producing a SerializedPage.", + 24L << 20, + false), booleanProperty( RANDOMIZE_OUTER_JOIN_NULL_KEY, "(Deprecated) Randomize null join key for outer join", diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index 8af2b944b443c..cf63b5a75c395 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -299,11 +299,32 @@ SessionProperties::SessionProperties() { kQueryTraceTaskRegExp, "The regexp of traced task id. We only enable trace on a task if its id" " matches.", - BIGINT(), + VARCHAR(), false, QueryConfig::kQueryTraceTaskRegExp, c.queryTraceTaskRegExp()); + addSessionProperty( + kMaxOutputBufferSize, + "The maximum size in bytes for the task's buffered output. The buffer is" + " shared among all drivers.", + BIGINT(), + false, + QueryConfig::kMaxOutputBufferSize, + std::to_string(c.maxOutputBufferSize())); + + addSessionProperty( + kMaxPartitionedOutputBufferSize, + "The maximum bytes to buffer per PartitionedOutput operator to avoid" + "creating tiny SerializedPages. For " + "PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator" + "would buffer up to that number of bytes / number of destinations for " + "each destination before producing a SerializedPage.", + BIGINT(), + false, + QueryConfig::kMaxPartitionedOutputBufferSize, + std::to_string(c.maxPartitionedOutputBufferSize())); + // If `legacy_timestamp` is true, the coordinator expects timestamp // conversions without a timezone to be converted to the user's // session_timezone. diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index 86ad349918c50..c9a5db98d3326 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -205,6 +205,19 @@ class SessionProperties { static constexpr const char* kQueryTraceTaskRegExp = "native_query_trace_task_reg_exp"; + /// The maximum size in bytes for the task's buffered output. The buffer is + /// shared among all drivers. + static constexpr const char* kMaxOutputBufferSize = + "native_max_output_buffer_size"; + + /// The maximum bytes to buffer per PartitionedOutput operator to avoid + /// creating tiny SerializedPages. For + /// PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator + /// would buffer up to that number of bytes / number of destinations for each + /// destination before producing a SerializedPage. + static constexpr const char* kMaxPartitionedOutputBufferSize = + "native_max_page_partitioning_buffer_size"; + SessionProperties(); const std::unordered_map>&