From 693649f5df65f07977ffd1f90e65a86281f98e65 Mon Sep 17 00:00:00 2001 From: manoj-1407 Date: Mon, 10 Aug 2026 07:07:04 +0000 Subject: [PATCH] feat: add hami_gpu_device_health metric to scheduler DeviceUsage.Health is used internally to skip unhealthy devices in Fit() but was never emitted as a Prometheus metric. Operators have no way to alert on an unhealthy GPU without inspecting node annotations manually. Add hami_gpu_device_health gauge (1=healthy, 0=unhealthy) in collectNodeMetrics, with label set {node, device_uuid, device_index, device_type} consistent with hami_gpu_memory_limit_bytes. Signed-off-by: G. Manoj Kumar Signed-off-by: manoj-1407 --- cmd/scheduler/metrics.go | 15 ++++++++++ cmd/scheduler/metrics_test.go | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index 00e0764bed..e8741bc27a 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -144,6 +144,11 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric "Realized MIG instance identity and scheduler placement", []string{"node", "device_uuid", "device_index", "mig_uuid", "profile", "gpu_instance_id", "compute_instance_id", "placement_start", "placement_size"}, nil, ) + nodeGPUDeviceHealthDesc := prometheus.NewDesc( + "hami_gpu_device_health", + "GPU device health status (1=healthy, 0=unhealthy)", + []string{"node", "device_uuid", "device_index", "device_type"}, nil, + ) // Legacy metric descriptors (only created when legacy mode is enabled) var ( @@ -266,6 +271,16 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric } } + healthVal := float64(0) + if devs.Device.Health { + healthVal = 1 + } + if err := sendMetric(ch, nodeGPUDeviceHealthDesc, prometheus.GaugeValue, + healthVal, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type, + ); err != nil { + klog.V(4).Infof("Failed to send nodeGPUDeviceHealthDesc metric: %v", err) + } + if legacy { sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) sendLegacyMetric(ch, legacyCoreLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalcore), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 3af0d82d67..772aacd113 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -477,3 +477,57 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ } }) } + +func TestCollectNodeMetricsDeviceHealth(t *testing.T) { + nodeUsage := map[string]*schedulerpkg.NodeUsage{ + "node-1": { + Devices: policy.DeviceUsageList{ + DeviceLists: []*policy.DeviceListsScore{ + { + Device: &device.DeviceUsage{ + ID: "GPU-healthy-0", + Index: 0, + Totalmem: 8192, + Totalcore: 100, + Type: "NVIDIA", + Health: true, + }, + }, + { + Device: &device.DeviceUsage{ + ID: "GPU-unhealthy-1", + Index: 1, + Totalmem: 8192, + Totalcore: 100, + Type: "NVIDIA", + Health: false, + }, + }, + }, + }, + }, + } + + collector := ClusterManagerCollector{ + ClusterManager: &ClusterManager{LegacyMetrics: false}, + metricsProvider: &fakeMetricsProvider{ + nodeUsage: nodeUsage, + quotaManager: device.NewQuotaManager(), + podManager: device.NewPodManager(), + }, + } + + want := ` +# HELP hami_gpu_device_health GPU device health status (1=healthy, 0=unhealthy) +# TYPE hami_gpu_device_health gauge +hami_gpu_device_health{device_index="0",device_type="NVIDIA",device_uuid="GPU-healthy-0",node="node-1"} 1 +hami_gpu_device_health{device_index="1",device_type="NVIDIA",device_uuid="GPU-unhealthy-1",node="node-1"} 0 +` + if err := promtestutil.CollectAndCompare( + collector, + strings.NewReader(want), + "hami_gpu_device_health", + ); err != nil { + t.Fatalf("unexpected collecting result:\n%s", err) + } +}