Skip to content

[native] Add config for stats reporting and counters. #21745

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
amitkdutta merged 1 commit into from
Jan 23, 2024
Merged

[native] Add config for stats reporting and counters. #21745

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions presto-native-execution/presto_cpp/main/PrestoServer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ void PrestoServer::run() {
baseVeloxQueryConfig->initialize(
fmt::format("{}/velox.properties", configDirectoryPath_));

if (systemConfig->enableRuntimeMetricsCollection()) {
enableRuntimeMetricReporting();
}

httpPort = systemConfig->httpServerHttpPort();
if (systemConfig->httpServerHttpsEnabled()) {
httpsPort = systemConfig->httpServerHttpsPort();
Expand Down Expand Up @@ -987,6 +991,12 @@ std::string PrestoServer::getBaseSpillDirectory() const {
return SystemConfig::instance()->spillerSpillPath().value_or("");
}

void PrestoServer::enableRuntimeMetricReporting() {
// This flag must be set to register the counters.
facebook::velox::BaseStatsReporter::registered = true;
registerStatsCounters();
}

void PrestoServer::populateMemAndCPUInfo() {
auto systemConfig = SystemConfig::instance();
const int64_t nodeMemoryGb = systemConfig->systemMemoryGb();
Expand Down
7 changes: 5 additions & 2 deletions presto-native-execution/presto_cpp/main/PrestoServer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,6 @@ class PrestoServer {

virtual void registerMemoryArbitrators();

virtual void registerStatsCounters();

/// Invoked after creating global (singleton) config objects (SystemConfig and
/// NodeConfig) and before loading their properties from the file.
/// In the implementation any extra config properties can be registered.
Expand All @@ -143,6 +141,9 @@ class PrestoServer {
/// Invoked to get the spill directory.
virtual std::string getBaseSpillDirectory() const;

/// Invoked to enable stats reporting and register counters.
virtual void enableRuntimeMetricReporting();

/// Invoked to get the list of filters passed to the http server.
std::vector<std::unique_ptr<proxygen::RequestHandlerFactory>>
getHttpServerFilters();
Expand All @@ -151,6 +152,8 @@ class PrestoServer {

void initializeThreadPools();

void registerStatsCounters();

protected:
void addServerPeriodicTasks();

Expand Down
5 changes: 5 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Configs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ SystemConfig::SystemConfig() {
BOOL_PROP(kCacheVeloxTtlEnabled, false),
STR_PROP(kCacheVeloxTtlThreshold, "2d"),
STR_PROP(kCacheVeloxTtlCheckInterval, "1h"),
BOOL_PROP(kEnableRuntimeMetricsCollection, false),
};
}

Expand Down Expand Up @@ -591,6 +592,10 @@ std::chrono::duration<double> SystemConfig::cacheVeloxTtlCheckInterval() const {
optionalProperty(kCacheVeloxTtlCheckInterval).value());
}

bool SystemConfig::enableRuntimeMetricsCollection() const {
return optionalProperty<bool>(kEnableRuntimeMetricsCollection).value();
}

NodeConfig::NodeConfig() {
registeredProps_ =
std::unordered_map<std::string, folly::Optional<std::string>>{
Expand Down
5 changes: 5 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Configs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,9 @@ class SystemConfig : public ConfigBase {
"cache.velox.ttl-check-interval"};
static constexpr std::string_view kUseMmapAllocator{"use-mmap-allocator"};

static constexpr std::string_view kEnableRuntimeMetricsCollection{
"runtime-metrics-collection-enabled"};

/// Specifies the memory arbitrator kind. If it is empty, then there is no
/// memory arbitration.
static constexpr std::string_view kMemoryArbitratorKind{
Expand Down Expand Up @@ -655,6 +658,8 @@ class SystemConfig : public ConfigBase {
std::chrono::duration<double> cacheVeloxTtlThreshold() const;

std::chrono::duration<double> cacheVeloxTtlCheckInterval() const;

bool enableRuntimeMetricsCollection() const;
};

/// Provides access to node properties defined in node.properties file.
Expand Down