Skip to content

suicide immediately when resource hard limit is met #1649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 31, 2024
Merged

suicide immediately when resource hard limit is met #1649

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c591eb7
suicide immediately when resource hard limit is met
henryzhx8 Jul 31, 2024
f0cff81
polish
henryzhx8 Jul 31, 2024
d94663d
polish
henryzhx8 Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 28 additions & 5 deletions core/monitor/Monitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ using namespace std;
using namespace sls_logs;

DEFINE_FLAG_BOOL(logtail_dump_monitor_info, "enable to dump Logtail monitor info (CPU, mem)", false);
DEFINE_FLAG_INT32(check_resource_interval, "", 5);
DECLARE_FLAG_BOOL(send_prefer_real_ip);
DECLARE_FLAG_BOOL(check_profile_region);

Expand Down Expand Up @@ -140,7 +141,7 @@ void LogtailMonitor::Stop() {

void LogtailMonitor::Monitor() {
LOG_INFO(sLogger, ("profiling", "started"));
int32_t lastMonitorTime = time(NULL);
int32_t lastMonitorTime = time(NULL), lastCheckHardLimitTime = time(nullptr);
CpuStat curCpuStat;
{
unique_lock<mutex> lock(mThreadRunningMux);
Expand Down Expand Up @@ -172,6 +173,17 @@ void LogtailMonitor::Monitor() {
}
#endif

if ((monitorTime - lastCheckHardLimitTime) < INT32_FLAG(check_resource_interval))
continue;
lastCheckHardLimitTime = monitorTime;

if (CheckHardCpuLimit() || CheckHardMemLimit()) {
LOG_ERROR(sLogger,
("Resource used by program exceeds hard limit",
"prepare restart Logtail")("cpu_usage", mCpuStat.mCpuUsage)("mem_rss", mMemStat.mRss));
Suicide();
}

// Update statistics and send to logtail_status_profile regularly.
// If CPU or memory limit triggered, send to logtail_suicide_profile.
if ((monitorTime - lastMonitorTime) < INT32_FLAG(monitor_interval))
Expand All @@ -191,9 +203,9 @@ void LogtailMonitor::Monitor() {
// Returning true means too much violations, so we have to prepare to restart
// logtail to release resource.
// Mainly for controlling memory because we have no idea to descrease memory usage.
if (CheckCpuLimit() || CheckMemLimit()) {
if (CheckSoftCpuLimit() || CheckSoftMemLimit()) {
LOG_ERROR(sLogger,
("Resource used by program exceeds upper limit",
("Resource used by program exceeds upper limit for some time",
"prepare restart Logtail")("cpu_usage", mCpuStat.mCpuUsage)("mem_rss", mMemStat.mRss));
Suicide();
}
Expand Down Expand Up @@ -439,7 +451,7 @@ void LogtailMonitor::CalCpuStat(const CpuStat& curCpu, CpuStat& savedCpu) {
#endif
}

bool LogtailMonitor::CheckCpuLimit() {
bool LogtailMonitor::CheckSoftCpuLimit() {
float cpuUsageLimit = AppConfig::GetInstance()->IsResourceAutoScale()
? AppConfig::GetInstance()->GetScaledCpuUsageUpLimit()
: AppConfig::GetInstance()->GetCpuUsageUpLimit();
Expand All @@ -451,7 +463,7 @@ bool LogtailMonitor::CheckCpuLimit() {
return false;
}

bool LogtailMonitor::CheckMemLimit() {
bool LogtailMonitor::CheckSoftMemLimit() {
if (mMemStat.mRss > AppConfig::GetInstance()->GetMemUsageUpLimit()) {
if (++mMemStat.mViolateNum > INT32_FLAG(mem_limit_num))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

间隔变了，这里次数没变？

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

间隔变了，这里次数没变？

return true;
Expand All @@ -460,6 +472,17 @@ bool LogtailMonitor::CheckMemLimit() {
return false;
}

bool LogtailMonitor::CheckHardCpuLimit() {
float cpuUsageLimit = AppConfig::GetInstance()->IsResourceAutoScale()
? AppConfig::GetInstance()->GetScaledCpuUsageUpLimit()
: AppConfig::GetInstance()->GetCpuUsageUpLimit();
return mCpuStat.mCpuUsage > 10 * cpuUsageLimit;
}

bool LogtailMonitor::CheckHardMemLimit() {
return mMemStat.mRss > 10 * AppConfig::GetInstance()->GetMemUsageUpLimit();
}

void LogtailMonitor::DumpToLocal(const sls_logs::LogGroup& logGroup) {
string dumpStr = "\n####logtail status####\n";
for (int32_t logIdx = 0; logIdx < logGroup.logs_size(); ++logIdx) {
Expand Down
11 changes: 7 additions & 4 deletions core/monitor/Monitor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,15 @@ class LogtailMonitor : public MetricStore {
// set @curCpu to @savedCpu after calculation.
void CalCpuStat(const CpuStat& curCpu, CpuStat& savedCpu);

// CheckCpuLimit checks if current cpu usage exceeds limit.
// CheckSoftCpuLimit checks if current cpu usage exceeds limit.
// @return true if the cpu usage exceeds limit continuously.
bool CheckCpuLimit();
// CheckMemLimit checks if the memory usage exceeds limit.
bool CheckSoftCpuLimit();
// CheckSoftMemLimit checks if the memory usage exceeds limit.
// @return true if the memory usage exceeds limit continuously.
bool CheckMemLimit();
bool CheckSoftMemLimit();

bool CheckHardCpuLimit();
bool CheckHardMemLimit();

// SendStatusProfile collects status profile and send them to server.
// @suicide indicates if the target LogStore is logtail_suicide_profile.
Expand Down
Loading