Skip to content

feat: add hami_gpu_device_health metric to scheduler #2540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
manoj-1407 wants to merge 1 commit into from
+69 −0
Closed

feat: add hami_gpu_device_health metric to scheduler #2540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions cmd/scheduler/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
"Realized MIG instance identity and scheduler placement",
[]string{"node", "device_uuid", "device_index", "mig_uuid", "profile", "gpu_instance_id", "compute_instance_id", "placement_start", "placement_size"}, nil,
)
nodeGPUDeviceHealthDesc := prometheus.NewDesc(

@mesutoezdil mesutoezdil Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

name says gpu but this loop runs for every backend, not only gpu. same pattern as the other metrics near it. just checking this is on purpose.

@manoj-1407 manoj-1407 Aug 11, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, intentional — same pattern as hami_gpu_memory_limit_bytes, hami_gpu_core_limit_ratio, and the other descriptors in that loop. The gpu prefix follows the existing naming convention in this file rather than reflecting the backend type at runtime. The loop is backend-agnostic by design.

"hami_gpu_device_health",
"GPU device health status (1=healthy, 0=unhealthy)",
[]string{"node", "device_uuid", "device_index", "device_type"}, nil,
)

// Legacy metric descriptors (only created when legacy mode is enabled)
var (
Expand Down Expand Up @@ -266,6 +271,16 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
}
}

healthVal := float64(0)

@mesutoezdil mesutoezdil Aug 11, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this health value live at scrape time, or a cached value from an older check? matters for alert delay if someone pages off this metric.

@manoj-1407 manoj-1407 Aug 11, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's the value from the most recent CheckHealth() call, written into DeviceUsage.Health by the device plugin and cached in nodeManager. so it reflects last known state, not a live poll at scrape time — lag depends on the device plugin's check interval. happy to add a note to the metric description if that's useful.

if devs.Device.Health {
healthVal = 1
}
if err := sendMetric(ch, nodeGPUDeviceHealthDesc, prometheus.GaugeValue,
healthVal, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type,
); err != nil {
klog.V(4).Infof("Failed to send nodeGPUDeviceHealthDesc metric: %v", err)
}

if legacy {
sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyCoreLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalcore), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
Expand Down
54 changes: 54 additions & 0 deletions cmd/scheduler/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,3 +477,57 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/
}
})
}

func TestCollectNodeMetricsDeviceHealth(t *testing.T) {
nodeUsage := map[string]*schedulerpkg.NodeUsage{
"node-1": {
Devices: policy.DeviceUsageList{
DeviceLists: []*policy.DeviceListsScore{
{
Device: &device.DeviceUsage{
ID: "GPU-healthy-0",
Index: 0,
Totalmem: 8192,
Totalcore: 100,
Type: "NVIDIA",
Health: true,
},
},
{
Device: &device.DeviceUsage{
ID: "GPU-unhealthy-1",
Index: 1,
Totalmem: 8192,
Totalcore: 100,
Type: "NVIDIA",
Health: false,
},
},
},
},
},
}

collector := ClusterManagerCollector{
ClusterManager: &ClusterManager{LegacyMetrics: false},
metricsProvider: &fakeMetricsProvider{
nodeUsage: nodeUsage,
quotaManager: device.NewQuotaManager(),
podManager: device.NewPodManager(),
},
}

want := `
# HELP hami_gpu_device_health GPU device health status (1=healthy, 0=unhealthy)
# TYPE hami_gpu_device_health gauge
hami_gpu_device_health{device_index="0",device_type="NVIDIA",device_uuid="GPU-healthy-0",node="node-1"} 1
hami_gpu_device_health{device_index="1",device_type="NVIDIA",device_uuid="GPU-unhealthy-1",node="node-1"} 0
`
if err := promtestutil.CollectAndCompare(
collector,
strings.NewReader(want),
"hami_gpu_device_health",
); err != nil {
t.Fatalf("unexpected collecting result:\n%s", err)
}
}
Loading