Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/scheduler/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
nodeGPUMemoryPercentage := prometheus.NewDesc(
"hami_node_gpu_memory_allocated_ratio",
"GPU Memory Allocated Percentage on a certain GPU",
[]string{"node", "device_uuid", "device_index"}, nil,
[]string{"node", "device_uuid", "device_index", "device_type"}, nil,
)
nodeGPUMigInstance := prometheus.NewDesc(
"hami_node_gpu_mig_instance_info",
Expand Down Expand Up @@ -190,7 +190,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
legacyMemoryPercentage = prometheus.NewDesc(
Comment thread
archlitchi marked this conversation as resolved.
Resolved
"nodeGPUMemoryPercentage",
"GPU Memory Allocated Percentage on a certain GPU",
[]string{"nodeid", "deviceuuid", "deviceidx"}, nil,
[]string{"nodeid", "deviceuuid", "deviceidx", "devicetype"}, nil,
)
legacyMigInstance = prometheus.NewDesc(
"nodeGPUMigInstance",
Expand Down Expand Up @@ -261,7 +261,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
}

if devs.Device.Totalmem > 0 {
if err := sendMetric(ch, nodeGPUMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index)); err != nil {
if err := sendMetric(ch, nodeGPUMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodeGPUMemoryPercentage metric: %v", err)
}
}
Expand All @@ -274,7 +274,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
sendLegacyMetric(ch, legacyCoreAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedcores), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type)
if devs.Device.Totalmem > 0 {
sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index))
sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/scheduler/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,10 @@ hami_gpu_core_limit_ratio{device_index="1",device_type="test-device",device_uuid
hami_gpu_core_limit_ratio{device_index="2",device_type="NVIDIA",device_uuid="normal-memory",node="node-1"} 2
# HELP hami_node_gpu_memory_allocated_ratio GPU Memory Allocated Percentage on a certain GPU
# TYPE hami_node_gpu_memory_allocated_ratio gauge
hami_node_gpu_memory_allocated_ratio{device_index="2",device_uuid="normal-memory",node="node-1"} 0.25
hami_node_gpu_memory_allocated_ratio{device_index="2",device_type="NVIDIA",device_uuid="normal-memory",node="node-1"} 0.25
# HELP nodeGPUMemoryPercentage GPU Memory Allocated Percentage on a certain GPU
# TYPE nodeGPUMemoryPercentage gauge
nodeGPUMemoryPercentage{deviceidx="2",deviceuuid="normal-memory",nodeid="node-1"} 0.25
nodeGPUMemoryPercentage{deviceidx="2",devicetype="NVIDIA",deviceuuid="normal-memory",nodeid="node-1"} 0.25
`

if err := promtestutil.CollectAndCompare(
Expand Down
Loading