Skip to content

Commit 410bb8c

Browse files
prashantgolashprashantgolash
authored
[Coordinator throttling] Endpoint on Java worker reporting nodestats (prestodb#25687)
Summary: Similar to cpp worker added the endpoint for java. We won't be using the worker-load as going forward we will be focussing on cpp worker only Differential Revision: D79471792
1 parent 1cf0190 commit 410bb8c

File tree

9 files changed

+503
-59
lines changed

9 files changed

+503
-59
lines changed

‎presto-main/src/main/java/com/facebook/presto/server/ServerInfoResource.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.facebook.presto.execution.resourceGroups.ResourceGroupManager;
2020
import com.facebook.presto.metadata.StaticCatalogStore;
2121
import com.facebook.presto.spi.NodeState;
22+
import com.facebook.presto.spi.NodeStats;
2223
import jakarta.annotation.security.RolesAllowed;
2324
import jakarta.inject.Inject;
2425
import jakarta.ws.rs.Consumes;
@@ -128,6 +129,16 @@ else if (!nodeResourceStatusProvider.hasResources() || !resourceGroupManager.isC
128129
}
129130
}
130131

132+
@GET
133+
@Path("stats")
134+
@Produces({APPLICATION_JSON, APPLICATION_THRIFT_BINARY, APPLICATION_THRIFT_COMPACT, APPLICATION_THRIFT_FB_COMPACT})
135+
@RolesAllowed(ADMIN)
136+
public NodeStats getServerStats()
137+
{
138+
NodeStats stats = new NodeStats(getServerState(), null);
139+
return stats;
140+
}
141+
131142
@GET
132143
@Path("coordinator")
133144
@Produces(TEXT_PLAIN)

‎presto-native-execution/presto_cpp/main/PrestoServer.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,14 @@ void PrestoServer::run() {
354354
json infoStateJson = convertNodeState(server->nodeState());
355355
http::sendOkResponse(downstream, infoStateJson);
356356
});
357+
httpServer_->registerGet(
358+
"/v1/info/stats",
359+
[server = this](
360+
proxygen::HTTPMessage* /*message*/,
361+
const std::vector<std::unique_ptr<folly::IOBuf>>& /*body*/,
362+
proxygen::ResponseHandler* downstream) {
363+
server->reportNodeStats(downstream);
364+
});
357365
httpServer_->registerPut(
358366
"/v1/info/state",
359367
[server = this](
@@ -1743,4 +1751,16 @@ void PrestoServer::createTaskManager() {
17431751
driverExecutor_.get(), httpSrvCpuExecutor_.get(), spillerExecutor_.get());
17441752
}
17451753

1754+
void PrestoServer::reportNodeStats(proxygen::ResponseHandler* downstream) {
1755+
protocol::NodeStats nodeStats;
1756+
1757+
auto loadMetrics = std::make_shared<protocol::NodeLoadMetrics>();
1758+
loadMetrics->cpuOverload = cpuOverloaded_;
1759+
loadMetrics->memoryOverload = memOverloaded_;
1760+
1761+
nodeStats.loadMetrics = loadMetrics;
1762+
nodeStats.nodeState = convertNodeState(this->nodeState());
1763+
1764+
http::sendOkResponse(downstream, json(nodeStats));
1765+
}
17461766
} // namespace facebook::presto

‎presto-native-execution/presto_cpp/main/PrestoServer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@ class PrestoServer {
208208

209209
void reportNodeStatus(proxygen::ResponseHandler* downstream);
210210

211+
void reportNodeStats(proxygen::ResponseHandler* downstream);
212+
211213
void handleGracefulShutdown(
212214
const std::vector<std::unique_ptr<folly::IOBuf>>& body,
213215
proxygen::ResponseHandler* downstream);

‎presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp‎

Lines changed: 142 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -7034,6 +7034,148 @@ void from_json(const json& j, MergeJoinNode& p) {
70347034
} // namespace facebook::presto::protocol
70357035
namespace facebook::presto::protocol {
70367036

7037+
void to_json(json& j, const NodeLoadMetrics& p) {
7038+
j = json::object();
7039+
to_json_key(
7040+
j,
7041+
"cpuUsedPercent",
7042+
p.cpuUsedPercent,
7043+
"NodeLoadMetrics",
7044+
"double",
7045+
"cpuUsedPercent");
7046+
to_json_key(
7047+
j,
7048+
"memoryUsedInBytes",
7049+
p.memoryUsedInBytes,
7050+
"NodeLoadMetrics",
7051+
"double",
7052+
"memoryUsedInBytes");
7053+
to_json_key(
7054+
j,
7055+
"numQueuedDrivers",
7056+
p.numQueuedDrivers,
7057+
"NodeLoadMetrics",
7058+
"int",
7059+
"numQueuedDrivers");
7060+
to_json_key(
7061+
j,
7062+
"cpuOverload",
7063+
p.cpuOverload,
7064+
"NodeLoadMetrics",
7065+
"bool",
7066+
"cpuOverload");
7067+
to_json_key(
7068+
j,
7069+
"memoryOverload",
7070+
p.memoryOverload,
7071+
"NodeLoadMetrics",
7072+
"bool",
7073+
"memoryOverload");
7074+
}
7075+
7076+
void from_json(const json& j, NodeLoadMetrics& p) {
7077+
from_json_key(
7078+
j,
7079+
"cpuUsedPercent",
7080+
p.cpuUsedPercent,
7081+
"NodeLoadMetrics",
7082+
"double",
7083+
"cpuUsedPercent");
7084+
from_json_key(
7085+
j,
7086+
"memoryUsedInBytes",
7087+
p.memoryUsedInBytes,
7088+
"NodeLoadMetrics",
7089+
"double",
7090+
"memoryUsedInBytes");
7091+
from_json_key(
7092+
j,
7093+
"numQueuedDrivers",
7094+
p.numQueuedDrivers,
7095+
"NodeLoadMetrics",
7096+
"int",
7097+
"numQueuedDrivers");
7098+
from_json_key(
7099+
j,
7100+
"cpuOverload",
7101+
p.cpuOverload,
7102+
"NodeLoadMetrics",
7103+
"bool",
7104+
"cpuOverload");
7105+
from_json_key(
7106+
j,
7107+
"memoryOverload",
7108+
p.memoryOverload,
7109+
"NodeLoadMetrics",
7110+
"bool",
7111+
"memoryOverload");
7112+
}
7113+
} // namespace facebook::presto::protocol
7114+
namespace facebook::presto::protocol {
7115+
// Loosly copied this here from NLOHMANN_JSON_SERIALIZE_ENUM()
7116+
7117+
// NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays
7118+
static const std::pair<NodeState, json> NodeState_enum_table[] =
7119+
{ // NOLINT: cert-err58-cpp
7120+
{NodeState::ACTIVE, "ACTIVE"},
7121+
{NodeState::INACTIVE, "INACTIVE"},
7122+
{NodeState::SHUTTING_DOWN, "SHUTTING_DOWN"}};
7123+
void to_json(json& j, const NodeState& e) {
7124+
static_assert(std::is_enum<NodeState>::value, "NodeState must be an enum!");
7125+
const auto* it = std::find_if(
7126+
std::begin(NodeState_enum_table),
7127+
std::end(NodeState_enum_table),
7128+
[e](const std::pair<NodeState, json>& ej_pair) -> bool {
7129+
return ej_pair.first == e;
7130+
});
7131+
j = ((it != std::end(NodeState_enum_table))
7132+
? it
7133+
: std::begin(NodeState_enum_table))
7134+
->second;
7135+
}
7136+
void from_json(const json& j, NodeState& e) {
7137+
static_assert(std::is_enum<NodeState>::value, "NodeState must be an enum!");
7138+
const auto* it = std::find_if(
7139+
std::begin(NodeState_enum_table),
7140+
std::end(NodeState_enum_table),
7141+
[&j](const std::pair<NodeState, json>& ej_pair) -> bool {
7142+
return ej_pair.second == j;
7143+
});
7144+
e = ((it != std::end(NodeState_enum_table))
7145+
? it
7146+
: std::begin(NodeState_enum_table))
7147+
->first;
7148+
}
7149+
} // namespace facebook::presto::protocol
7150+
namespace facebook::presto::protocol {
7151+
7152+
void to_json(json& j, const NodeStats& p) {
7153+
j = json::object();
7154+
to_json_key(
7155+
j, "nodeState", p.nodeState, "NodeStats", "NodeState", "nodeState");
7156+
to_json_key(
7157+
j,
7158+
"loadMetrics",
7159+
p.loadMetrics,
7160+
"NodeStats",
7161+
"NodeLoadMetrics",
7162+
"loadMetrics");
7163+
}
7164+
7165+
void from_json(const json& j, NodeStats& p) {
7166+
from_json_key(
7167+
j, "nodeState", p.nodeState, "NodeStats", "NodeState", "nodeState");
7168+
from_json_key(
7169+
j,
7170+
"loadMetrics",
7171+
p.loadMetrics,
7172+
"NodeStats",
7173+
"NodeLoadMetrics",
7174+
"loadMetrics");
7175+
}
7176+
} // namespace facebook::presto::protocol
7177+
namespace facebook::presto::protocol {
7178+
70377179
void to_json(json& j, const NodeVersion& p) {
70387180
j = json::object();
70397181
to_json_key(j, "version", p.version, "NodeVersion", "String", "version");
@@ -11569,39 +11711,3 @@ void from_json(const json& j, WindowNode& p) {
1156911711
"preSortedOrderPrefix");
1157011712
}
1157111713
} // namespace facebook::presto::protocol
11572-
namespace facebook::presto::protocol {
11573-
// Loosly copied this here from NLOHMANN_JSON_SERIALIZE_ENUM()
11574-
11575-
// NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays
11576-
static const std::pair<NodeState, json> NodeState_enum_table[] =
11577-
{ // NOLINT: cert-err58-cpp
11578-
{NodeState::ACTIVE, "ACTIVE"},
11579-
{NodeState::INACTIVE, "INACTIVE"},
11580-
{NodeState::SHUTTING_DOWN, "SHUTTING_DOWN"}};
11581-
void to_json(json& j, const NodeState& e) {
11582-
static_assert(std::is_enum<NodeState>::value, "NodeState must be an enum!");
11583-
const auto* it = std::find_if(
11584-
std::begin(NodeState_enum_table),
11585-
std::end(NodeState_enum_table),
11586-
[e](const std::pair<NodeState, json>& ej_pair) -> bool {
11587-
return ej_pair.first == e;
11588-
});
11589-
j = ((it != std::end(NodeState_enum_table))
11590-
? it
11591-
: std::begin(NodeState_enum_table))
11592-
->second;
11593-
}
11594-
void from_json(const json& j, NodeState& e) {
11595-
static_assert(std::is_enum<NodeState>::value, "NodeState must be an enum!");
11596-
const auto* it = std::find_if(
11597-
std::begin(NodeState_enum_table),
11598-
std::end(NodeState_enum_table),
11599-
[&j](const std::pair<NodeState, json>& ej_pair) -> bool {
11600-
return ej_pair.second == j;
11601-
});
11602-
e = ((it != std::end(NodeState_enum_table))
11603-
? it
11604-
: std::begin(NodeState_enum_table))
11605-
->first;
11606-
}
11607-
} // namespace facebook::presto::protocol

‎presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1761,6 +1761,30 @@ void to_json(json& j, const MergeJoinNode& p);
17611761
void from_json(const json& j, MergeJoinNode& p);
17621762
} // namespace facebook::presto::protocol
17631763
namespace facebook::presto::protocol {
1764+
struct NodeLoadMetrics {
1765+
double cpuUsedPercent = {};
1766+
double memoryUsedInBytes = {};
1767+
int numQueuedDrivers = {};
1768+
bool cpuOverload = {};
1769+
bool memoryOverload = {};
1770+
};
1771+
void to_json(json& j, const NodeLoadMetrics& p);
1772+
void from_json(const json& j, NodeLoadMetrics& p);
1773+
} // namespace facebook::presto::protocol
1774+
namespace facebook::presto::protocol {
1775+
enum class NodeState { ACTIVE, INACTIVE, SHUTTING_DOWN };
1776+
extern void to_json(json& j, const NodeState& e);
1777+
extern void from_json(const json& j, NodeState& e);
1778+
} // namespace facebook::presto::protocol
1779+
namespace facebook::presto::protocol {
1780+
struct NodeStats {
1781+
NodeState nodeState = {};
1782+
std::shared_ptr<NodeLoadMetrics> loadMetrics = {};
1783+
};
1784+
void to_json(json& j, const NodeStats& p);
1785+
void from_json(const json& j, NodeStats& p);
1786+
} // namespace facebook::presto::protocol
1787+
namespace facebook::presto::protocol {
17641788
struct NodeVersion {
17651789
String version = {};
17661790
};
@@ -2527,8 +2551,3 @@ struct WindowNode : public PlanNode {
25272551
void to_json(json& j, const WindowNode& p);
25282552
void from_json(const json& j, WindowNode& p);
25292553
} // namespace facebook::presto::protocol
2530-
namespace facebook::presto::protocol {
2531-
enum class NodeState { ACTIVE, INACTIVE, SHUTTING_DOWN };
2532-
extern void to_json(json& j, const NodeState& e);
2533-
extern void from_json(const json& j, NodeState& e);
2534-
} // namespace facebook::presto::protocol

‎presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,6 @@ ExtraFields:
4242
RemoteTransactionHandle:
4343
Optional<String>: dummy
4444

45-
AddToOutput:
46-
- NodeState
47-
4845
AbstractClasses:
4946
ColumnHandle:
5047
super: JsonEncodedSubclass
@@ -345,3 +342,5 @@ JavaClasses:
345342
- presto-spi/src/main/java/com/facebook/presto/spi/plan/DeleteNode.java
346343
- presto-spi/src/main/java/com/facebook/presto/spi/plan/BaseInputDistribution.java
347344
- presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInFunctionKind.java
345+
- presto-spi/src/main/java/com/facebook/presto/spi/NodeStats.java
346+
- presto-spi/src/main/java/com/facebook/presto/spi/NodeLoadMetrics.java

0 commit comments

Comments
 (0)