Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions cmd/scheduler/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
"Realized MIG instance identity and scheduler placement",
[]string{"node", "device_uuid", "device_index", "mig_uuid", "profile", "gpu_instance_id", "compute_instance_id", "placement_start", "placement_size"}, nil,
)
nodeGPUDeviceHealthDesc := prometheus.NewDesc(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

name says gpu but this loop runs for every backend. on purpose?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah it's intentional — Health is set on all device types, the name reflects what the metric measures rather than which backend it's coming from

"hami_gpu_device_health",
"GPU device health status (1=healthy, 0=unhealthy)",
[]string{"node", "device_uuid", "device_index", "device_type"}, nil,
)

// Legacy metric descriptors (only created when legacy mode is enabled)
var (
Expand Down Expand Up @@ -266,6 +271,16 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
}
}

healthVal := float64(0)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is health live at scrape time or a cached value?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's a cached value from the scheduler's in-memory device list, same as the other metrics in collectNodeMetrics

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if a node has zero devices in devicelists, does this metric emit anything for that node at all?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nothing gets emitted in that case, the loop just doesn't run — same behavior as the existing MIG metric

if devs.Device.Health {
healthVal = 1
}
if err := sendMetric(ch, nodeGPUDeviceHealthDesc, prometheus.GaugeValue,
healthVal, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type,
); err != nil {
klog.V(4).Infof("Failed to send nodeGPUDeviceHealthDesc metric: %v", err)
}

if legacy {
sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyCoreLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalcore), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
Expand Down
54 changes: 54 additions & 0 deletions cmd/scheduler/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,3 +477,57 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/
}
})
}

func TestCollectNodeMetricsDeviceHealth(t *testing.T) {
nodeUsage := map[string]*schedulerpkg.NodeUsage{
"node-1": {
Devices: policy.DeviceUsageList{
DeviceLists: []*policy.DeviceListsScore{
{
Device: &device.DeviceUsage{
ID: "GPU-healthy-0",
Index: 0,
Totalmem: 8192,
Totalcore: 100,
Type: "NVIDIA",
Health: true,
},
},
{
Device: &device.DeviceUsage{
ID: "GPU-unhealthy-1",
Index: 1,
Totalmem: 8192,
Totalcore: 100,
Type: "NVIDIA",
Health: false,
},
},
},
},
},
}

collector := ClusterManagerCollector{
ClusterManager: &ClusterManager{LegacyMetrics: false},
metricsProvider: &fakeMetricsProvider{
nodeUsage: nodeUsage,
quotaManager: device.NewQuotaManager(),
podManager: device.NewPodManager(),
},
}

want := `
# HELP hami_gpu_device_health GPU device health status (1=healthy, 0=unhealthy)
# TYPE hami_gpu_device_health gauge
hami_gpu_device_health{device_index="0",device_type="NVIDIA",device_uuid="GPU-healthy-0",node="node-1"} 1
hami_gpu_device_health{device_index="1",device_type="NVIDIA",device_uuid="GPU-unhealthy-1",node="node-1"} 0
`
if err := promtestutil.CollectAndCompare(
collector,
strings.NewReader(want),
"hami_gpu_device_health",
); err != nil {
t.Fatalf("unexpected collecting result:\n%s", err)
}
}
Loading