From 9925abb96175331039e1e66f734bf8c47be29fac Mon Sep 17 00:00:00 2001 From: Jialiang Tan Date: Mon, 9 Jun 2025 12:25:09 -0700 Subject: [PATCH] [native] Add remaining from spill header to http shuffle API Summary: To support PartitionedOutput spilling, we need an additional flag telling the receiving end if the remaining buffer will be from spill, for the receiving end to make proper load decisions. Differential Revision: D76286914 --- .../presto_protocol/core/presto_protocol-json-cpp.mustache | 1 + .../presto_protocol/core/presto_protocol-json-hpp.mustache | 1 + .../presto_cpp/presto_protocol/core/presto_protocol_core.cpp | 2 ++ .../presto_cpp/presto_protocol/core/presto_protocol_core.h | 1 + 4 files changed, 5 insertions(+) diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-cpp.mustache b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-cpp.mustache index e0381ece05bec..9af6aae623b4a 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-cpp.mustache +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-cpp.mustache @@ -87,6 +87,7 @@ const char* const PRESTO_PAGE_NEXT_TOKEN_HEADER = "X-Presto-Page-End-Sequence-Id const char* const PRESTO_BUFFER_COMPLETE_HEADER = "X-Presto-Buffer-Complete"; const char* const PRESTO_GET_DATA_SIZE_HEADER = "X-Presto-Get-Data-Size"; const char* const PRESTO_BUFFER_REMAINING_BYTES_HEADER = "X-Presto-Buffer-Remaining-Bytes"; +const char* const PRESTO_BUFFER_REMAINING_FROM_SPILL_HEADER = "X-Presto-Buffer-Remaining-From-Spill"; const char* const PRESTO_MAX_WAIT_DEFAULT = "2s"; const char* const PRESTO_MAX_SIZE_DEFAULT = "4096 B"; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-hpp.mustache b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-hpp.mustache index 2aa21b068d207..c58fddc1034b0 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-hpp.mustache +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol-json-hpp.mustache @@ -61,6 +61,7 @@ extern const char* const PRESTO_PAGE_NEXT_TOKEN_HEADER; extern const char* const PRESTO_BUFFER_COMPLETE_HEADER; extern const char* const PRESTO_GET_DATA_SIZE_HEADER; extern const char* const PRESTO_BUFFER_REMAINING_BYTES_HEADER; +extern const char* const PRESTO_BUFFER_REMAINING_FROM_SPILL_HEADER; extern const char* const PRESTO_MAX_WAIT_DEFAULT; extern const char* const PRESTO_MAX_SIZE_DEFAULT; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 3e989030598cd..904db4d5b0c56 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -87,6 +87,8 @@ const char* const PRESTO_BUFFER_COMPLETE_HEADER = "X-Presto-Buffer-Complete"; const char* const PRESTO_GET_DATA_SIZE_HEADER = "X-Presto-Get-Data-Size"; const char* const PRESTO_BUFFER_REMAINING_BYTES_HEADER = "X-Presto-Buffer-Remaining-Bytes"; +const char* const PRESTO_BUFFER_REMAINING_FROM_SPILL_HEADER = + "X-Presto-Buffer-Remaining-From-Spill"; const char* const PRESTO_MAX_WAIT_DEFAULT = "2s"; const char* const PRESTO_MAX_SIZE_DEFAULT = "4096 B"; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index bf9d31035b964..e32223bc91e1d 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -58,6 +58,7 @@ extern const char* const PRESTO_PAGE_NEXT_TOKEN_HEADER; extern const char* const PRESTO_BUFFER_COMPLETE_HEADER; extern const char* const PRESTO_GET_DATA_SIZE_HEADER; extern const char* const PRESTO_BUFFER_REMAINING_BYTES_HEADER; +extern const char* const PRESTO_BUFFER_REMAINING_FROM_SPILL_HEADER; extern const char* const PRESTO_MAX_WAIT_DEFAULT; extern const char* const PRESTO_MAX_SIZE_DEFAULT;