From 7fb437d9df3223592dbdf44189a1f4de8fea8654 Mon Sep 17 00:00:00 2001 From: Prabal Poddar Date: Wed, 5 Aug 2026 03:11:53 +0530 Subject: [PATCH 1/2] fix(metrics): add device_type label to node memory ratio metrics (#2370) Signed-off-by: Prabal Poddar --- cmd/scheduler/metrics.go | 8 ++++---- cmd/scheduler/metrics_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index 00e0764bed..831b037b3f 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -137,7 +137,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric nodeGPUMemoryPercentage := prometheus.NewDesc( "hami_node_gpu_memory_allocated_ratio", "GPU Memory Allocated Percentage on a certain GPU", - []string{"node", "device_uuid", "device_index"}, nil, + []string{"node", "device_uuid", "device_index", "device_type"}, nil, ) nodeGPUMigInstance := prometheus.NewDesc( "hami_node_gpu_mig_instance_info", @@ -190,7 +190,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric legacyMemoryPercentage = prometheus.NewDesc( "nodeGPUMemoryPercentage", "GPU Memory Allocated Percentage on a certain GPU", - []string{"nodeid", "deviceuuid", "deviceidx"}, nil, + []string{"nodeid", "deviceuuid", "deviceidx", "devicetype"}, nil, ) legacyMigInstance = prometheus.NewDesc( "nodeGPUMigInstance", @@ -261,7 +261,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric } if devs.Device.Totalmem > 0 { - if err := sendMetric(ch, nodeGPUMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index)); err != nil { + if err := sendMetric(ch, nodeGPUMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodeGPUMemoryPercentage metric: %v", err) } } @@ -274,7 +274,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric sendLegacyMetric(ch, legacyCoreAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedcores), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type) if devs.Device.Totalmem > 0 { - sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index)) + sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) } } } diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 0c7a1f9adf..dc9d0e0c55 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -361,10 +361,10 @@ hami_gpu_core_limit_ratio{device_index="1",device_type="test-device",device_uuid hami_gpu_core_limit_ratio{device_index="2",device_type="NVIDIA",device_uuid="normal-memory",node="node-1"} 2 # HELP hami_node_gpu_memory_allocated_ratio GPU Memory Allocated Percentage on a certain GPU # TYPE hami_node_gpu_memory_allocated_ratio gauge -hami_node_gpu_memory_allocated_ratio{device_index="2",device_uuid="normal-memory",node="node-1"} 0.25 +hami_node_gpu_memory_allocated_ratio{device_index="2",device_type="NVIDIA",device_uuid="normal-memory",node="node-1"} 0.25 # HELP nodeGPUMemoryPercentage GPU Memory Allocated Percentage on a certain GPU # TYPE nodeGPUMemoryPercentage gauge -nodeGPUMemoryPercentage{deviceidx="2",deviceuuid="normal-memory",nodeid="node-1"} 0.25 +nodeGPUMemoryPercentage{deviceidx="2",devicetype="NVIDIA",deviceuuid="normal-memory",nodeid="node-1"} 0.25 ` if err := promtestutil.CollectAndCompare( From 7a9cec21340d3a4253a7b988c74b3731966e5d3d Mon Sep 17 00:00:00 2001 From: Prabal Poddar Date: Tue, 11 Aug 2026 14:00:59 +0530 Subject: [PATCH 2/2] revert(metrics): preserve legacy mode label contract for nodeGPUMemoryPercentage Signed-off-by: Prabal Poddar --- cmd/scheduler/metrics.go | 4 ++-- cmd/scheduler/metrics_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index 831b037b3f..e8ddfb6ae9 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -190,7 +190,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric legacyMemoryPercentage = prometheus.NewDesc( "nodeGPUMemoryPercentage", "GPU Memory Allocated Percentage on a certain GPU", - []string{"nodeid", "deviceuuid", "deviceidx", "devicetype"}, nil, + []string{"nodeid", "deviceuuid", "deviceidx"}, nil, ) legacyMigInstance = prometheus.NewDesc( "nodeGPUMigInstance", @@ -274,7 +274,7 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric sendLegacyMetric(ch, legacyCoreAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedcores), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type) if devs.Device.Totalmem > 0 { - sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) + sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index)) } } } diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index dc9d0e0c55..e696e84111 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -364,7 +364,7 @@ hami_gpu_core_limit_ratio{device_index="2",device_type="NVIDIA",device_uuid="nor hami_node_gpu_memory_allocated_ratio{device_index="2",device_type="NVIDIA",device_uuid="normal-memory",node="node-1"} 0.25 # HELP nodeGPUMemoryPercentage GPU Memory Allocated Percentage on a certain GPU # TYPE nodeGPUMemoryPercentage gauge -nodeGPUMemoryPercentage{deviceidx="2",devicetype="NVIDIA",deviceuuid="normal-memory",nodeid="node-1"} 0.25 +nodeGPUMemoryPercentage{deviceidx="2",deviceuuid="normal-memory",nodeid="node-1"} 0.25 ` if err := promtestutil.CollectAndCompare(