Skip to content

[Enhancement] Extend 'show usage resource groups;' with mem_pool metrics (backport #66690) #67412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mergify wants to merge 1 commit into from
Closed

[Enhancement] Extend 'show usage resource groups;' with mem_pool metrics (backport #66690) #67412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions be/src/agent/resource_group_usage_recorder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,26 @@ std::vector<TResourceGroupUsage> ResourceGroupUsageRecorder::get_resource_group_
if (it == group_to_usage.end()) {
TResourceGroupUsage group_usage;
group_usage.__set_group_id(wg.id());
group_usage.__set_group_version(wg.version());
group_usage.__set_mem_used_bytes(wg.mem_consumption_bytes());
group_usage.__set_num_running_queries(wg.num_running_queries());
group_usage.__set_mem_limit_bytes(wg.mem_limit_bytes());
group_usage.__set_mem_pool(wg.mem_pool());
group_usage.__set_mem_pool_mem_limit_bytes(wg.parent_memory_limit_bytes().value_or(0));
group_usage.__set_mem_pool_mem_used_bytes(wg.parent_memory_usage_bytes().value_or(0));
group_to_usage.emplace(wg.id(), std::move(group_usage));

curr_group_to_cpu_runtime_ns.emplace(wg.id(), wg.cpu_runtime_ns());
} else {
TResourceGroupUsage& group_usage = it->second;
group_usage.__set_mem_used_bytes(group_usage.mem_used_bytes + wg.mem_consumption_bytes());
group_usage.__set_num_running_queries(group_usage.num_running_queries + wg.num_running_queries());

if (wg.version() >= group_usage.group_version) {
group_usage.__set_group_version(wg.version());
group_usage.__set_mem_pool(wg.mem_pool());
group_usage.__set_mem_limit_bytes(wg.mem_limit_bytes());
group_usage.__set_mem_pool_mem_limit_bytes(wg.parent_memory_limit_bytes().value_or(0));
group_usage.__set_mem_pool_mem_used_bytes(wg.parent_memory_usage_bytes().value_or(0));
}
curr_group_to_cpu_runtime_ns[wg.id()] += wg.cpu_runtime_ns();
}
});
Expand Down
9 changes: 9 additions & 0 deletions be/src/exec/workgroup/work_group.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,15 @@ class WorkGroup : public std::enable_shared_from_this<WorkGroup> {
int64_t num_total_queries() const { return _num_total_queries; }
int64_t concurrency_overflow_count() const { return _concurrency_overflow_count; }
int64_t bigquery_count() const { return _bigquery_count; }
std::optional<int64_t> parent_memory_limit_bytes() const {
return _mem_tracker != nullptr && _mem_tracker->parent() ? std::make_optional(_mem_tracker->parent()->limit())
: std::nullopt;
}
std::optional<int64_t> parent_memory_usage_bytes() const {
return _mem_tracker != nullptr && _mem_tracker->parent()
? std::make_optional(_mem_tracker->parent()->consumption())
: std::nullopt;
}

int64_t big_query_mem_limit() const { return _big_query_mem_limit; }
bool use_big_query_mem_limit() const {
Expand Down
3 changes: 3 additions & 0 deletions be/test/agent/resource_group_usage_recorder_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ TEST(ResourceGroupUsageRecorderTest, test_get_resource_group_usages) {
ASSERT_EQ(group_usages.size(), 1);
ASSERT_EQ(group_usages[0].group_id, default_wg->id());
ASSERT_EQ(group_usages[0].cpu_core_used_permille, num_cores * 1000);
ASSERT_EQ(group_usages[0].mem_pool, workgroup::WorkGroup::DEFAULT_MEM_POOL);
ASSERT_EQ(group_usages[0].mem_limit_bytes, default_wg->mem_limit_bytes());
ASSERT_EQ(group_usages[0].mem_pool_mem_limit_bytes, default_wg->mem_limit_bytes());
}

} // namespace starrocks
13 changes: 13 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/qe/ShowResultMetaFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -653,12 +653,25 @@ public ShowResultSetMetaData visitShowDataStatement(ShowDataStmt statement, Void
@Override
public ShowResultSetMetaData visitShowResourceGroupUsageStatement(ShowResourceGroupUsageStmt statement, Void context) {
return ShowResultSetMetaData.builder()
<<<<<<< HEAD
.addColumn(new Column("Name", ScalarType.createVarchar(64)))
.addColumn(new Column("Id", ScalarType.createVarchar(64)))
.addColumn(new Column("Backend", ScalarType.createVarchar(64)))
.addColumn(new Column("BEInUseCpuCores", ScalarType.createVarchar(64)))
.addColumn(new Column("BEInUseMemBytes", ScalarType.createVarchar(64)))
.addColumn(new Column("BERunningQueries", ScalarType.createVarchar(64)))
=======
.addColumn(new Column("Name", TypeFactory.createVarcharType(64)))
.addColumn(new Column("Id", TypeFactory.createVarcharType(64)))
.addColumn(new Column("Backend", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEInUseCpuCores", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEInUseMemBytes", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BERunningQueries", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEMemLimitBytes", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEMemPool", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEMemPoolInUseMemBytes", TypeFactory.createVarcharType(64)))
.addColumn(new Column("BEMemPoolMemLimitBytes", TypeFactory.createVarcharType(64)))
>>>>>>> f4a8df573a ([Enhancement] Extend 'show usage resource groups;' with mem_pool metrics (#66690))
.build();
}

Expand Down
13 changes: 13 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowResourceGroupUsageStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,21 @@ public class ShowResourceGroupUsageStmt extends ShowStmt {
item -> Double.toString(item.usage.getCpuCoreUsagePermille() / 1000.0D)),
Pair.create(new Column("BEInUseMemBytes", ScalarType.createVarchar(64)),
item -> Long.toString(item.usage.getMemUsageBytes())),
<<<<<<< HEAD
Pair.create(new Column("BERunningQueries", ScalarType.createVarchar(64)),
item -> Integer.toString(item.usage.getNumRunningQueries()))
=======
Pair.create(new Column("BERunningQueries", TypeFactory.createVarcharType(64)),
item -> Integer.toString(item.usage.getNumRunningQueries())),
Pair.create(new Column("BEMemLimitBytes", TypeFactory.createVarcharType(64)),
item -> Long.toString(item.usage.getMemLimitBytes())),
Pair.create(new Column("BEMemPool", TypeFactory.createVarcharType(64)),
item -> item.usage.getMemPool()),
Pair.create(new Column("BEMemPoolInUseMemBytes", TypeFactory.createVarcharType(64)),
item -> Long.toString(item.usage.getMemPoolMemUsageBytes())),
Pair.create(new Column("BEMemPoolMemLimitBytes", TypeFactory.createVarcharType(64)),
item -> Long.toString(item.usage.getMemPoolMemLimitBytes()))
>>>>>>> f4a8df573a ([Enhancement] Extend 'show usage resource groups;' with mem_pool metrics (#66690))
);

private static final List<Function<ShowItem, String>> COLUMN_SUPPLIERS = META_DATA.stream()
Expand Down
39 changes: 36 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/system/ComputeNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -771,17 +771,33 @@ public static class ResourceGroupUsage {
private final int cpuCoreUsagePermille;
private final long memUsageBytes;
private final int numRunningQueries;

private ResourceGroupUsage(ResourceGroup group, int cpuCoreUsagePermille, long memUsageBytes, int numRunningQueries) {
private final long memLimitBytes;
private final String memPool;
private final long memPoolMemUsageBytes;
private final long memPoolMemLimitBytes;

private ResourceGroupUsage(ResourceGroup group,
int cpuCoreUsagePermille,
long memUsageBytes,
int numRunningQueries,
long memLimitBytes,
String memPool,
long memPoolMemUsageBytes,
long memPoolMemLimitBytes) {
this.group = group;
this.cpuCoreUsagePermille = cpuCoreUsagePermille;
this.memUsageBytes = memUsageBytes;
this.numRunningQueries = numRunningQueries;
this.memLimitBytes = memLimitBytes;
this.memPool = memPool;
this.memPoolMemUsageBytes = memPoolMemUsageBytes;
this.memPoolMemLimitBytes = memPoolMemLimitBytes;
}

private static ResourceGroupUsage fromThrift(TResourceGroupUsage tUsage, ResourceGroup group) {
return new ResourceGroupUsage(group, tUsage.getCpu_core_used_permille(), tUsage.getMem_used_bytes(),
tUsage.getNum_running_queries());
tUsage.getNum_running_queries(), tUsage.getMem_limit_bytes(), tUsage.getMem_pool(),
tUsage.getMem_pool_mem_used_bytes(), tUsage.getMem_pool_mem_limit_bytes());
}

public boolean isCpuCoreUsagePermilleEffective() {
Expand All @@ -803,5 +819,22 @@ public long getMemUsageBytes() {
public int getNumRunningQueries() {
return numRunningQueries;
}

public long getMemLimitBytes() {
return memLimitBytes;
}

public String getMemPool() {
return memPool;
}

public long getMemPoolMemUsageBytes() {
return memPoolMemUsageBytes;
}

public long getMemPoolMemLimitBytes() {
return memPoolMemLimitBytes;
}

}
}
6 changes: 5 additions & 1 deletion fe/fe-core/src/test/java/com/starrocks/qe/ShowStmtMetaTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -843,13 +843,17 @@ public void testShowRoutineLoadTaskStmt() {
public void testShowResourceGroupUsageStmt() {
ShowResourceGroupUsageStmt stmt = new ShowResourceGroupUsageStmt("test_group", null);
ShowResultSetMetaData metaData = new ShowResultMetaFactory().getMetadata(stmt);
Assertions.assertEquals(6, metaData.getColumnCount());
Assertions.assertEquals(10, metaData.getColumnCount());
Assertions.assertEquals("Name", metaData.getColumn(0).getName());
Assertions.assertEquals("Id", metaData.getColumn(1).getName());
Assertions.assertEquals("Backend", metaData.getColumn(2).getName());
Assertions.assertEquals("BEInUseCpuCores", metaData.getColumn(3).getName());
Assertions.assertEquals("BEInUseMemBytes", metaData.getColumn(4).getName());
Assertions.assertEquals("BERunningQueries", metaData.getColumn(5).getName());
Assertions.assertEquals("BEMemLimitBytes", metaData.getColumn(6).getName());
Assertions.assertEquals("BEMemPool", metaData.getColumn(7).getName());
Assertions.assertEquals("BEMemPoolInUseMemBytes", metaData.getColumn(8).getName());
Assertions.assertEquals("BEMemPoolMemLimitBytes", metaData.getColumn(9).getName());
}

@Test
Expand Down
Loading
Loading