Skip to content

fix(metrics): normalize AMD core allocation ratio to percentage #2527

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hami-robot merged 2 commits into from
Aug 10, 2026
Merged

fix(metrics): normalize AMD core allocation ratio to percentage #2527

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33eedd3
fix(metrics): normalize AMD container core allocation ratio to percen…
yxxhero Aug 10, 2026
4967a28
refactor(metrics): split Collect into node/quota/container collectors
yxxhero Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 98 additions & 59 deletions cmd/scheduler/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,26 @@ func normalizeAMDCoreMetrics(deviceType string, total, allocated int32) (float64
return normalizedCoreLimit, math.Ceil(float64(allocated) / float64(total) * normalizedCoreLimit)
}

// findNodeDeviceUsage looks up a device by UUID across every node's usage and
// returns its total core capacity and type. ok is false when no node advertises
// the device, in which case the caller falls back to emitting raw values.
func findNodeDeviceUsage(nu *map[string]*schedulerpkg.NodeUsage, uuid string) (totalcore int32, deviceType string, ok bool) {
for _, ni := range *nu {
for _, dls := range ni.Devices.DeviceLists {
if dls.Device != nil && dls.Device.ID == uuid {
return dls.Device.Totalcore, dls.Device.Type, true
}
}
}
return 0, "", false
}

// mibToBytes converts a memory quantity expressed in mebibytes (MiB), the unit
// used throughout HAMi's device accounting, to bytes for byte-oriented metrics.
func mibToBytes(mib int32) float64 {
return float64(mib) * 1024 * 1024
}

// Describe is implemented with DescribeByCollect. That's possible because the
// Collect method will always return the same metrics with the same descriptors.
func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
Expand All @@ -72,8 +92,18 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
klog.V(3).Info("Starting to collect metrics for scheduler")
legacy := cc.ClusterManager.LegacyMetrics
// A single snapshot is shared by the node- and container-level collectors so
// they observe a consistent cluster state within one scrape.
nu := cc.metricsProvider.InspectAllNodesUsage()
cc.collectNodeMetrics(ch, nu, legacy)
cc.collectQuotaMetrics(ch, legacy)
cc.collectContainerMetrics(ch, nu, legacy)
}

// New metric descriptors
// collectNodeMetrics emits node-level GPU metrics (memory/core limits and
// allocations, sharing counts, MIG instances and overview) for every device on
// every node. AMD core values are normalized to a percentage.
func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric, nu *map[string]*schedulerpkg.NodeUsage, legacy bool) {
nodevGPUMemoryLimitDesc := prometheus.NewDesc(
"hami_gpu_memory_limit_bytes",
"Device memory limit for a certain GPU",
Expand Down Expand Up @@ -125,9 +155,6 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
legacyOverview *prometheus.Desc
legacyMemoryPercentage *prometheus.Desc
legacyMigInstance *prometheus.Desc
legacyAllocatedMemory *prometheus.Desc
legacyAllocatedCore *prometheus.Desc
legacyQuotaUsed *prometheus.Desc
)
if legacy {
legacyMemoryLimitDesc = prometheus.NewDesc(
Expand Down Expand Up @@ -170,24 +197,8 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
"GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps",
[]string{"nodeid", "deviceuuid", "deviceidx", "migname"}, nil,
)
legacyAllocatedMemory = prometheus.NewDesc(
"vGPUMemoryAllocated",
"vGPU memory allocated from a container",
[]string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
)
legacyAllocatedCore = prometheus.NewDesc(
"vGPUCoreAllocated",
"vGPU core allocated from a container",
[]string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
)
legacyQuotaUsed = prometheus.NewDesc(
"QuotaUsed",
"resourcequota usage for a certain device",
[]string{"quotanamespace", "quotaName", "limit"}, nil,
)
}

nu := cc.metricsProvider.InspectAllNodesUsage()
for nodeID, val := range *nu {
for _, devs := range val.Devices.DeviceLists {
coreLimit, coreAllocated := normalizeAMDCoreMetrics(devs.Device.Type, devs.Device.Totalcore, devs.Device.Usedcores)
Expand All @@ -207,13 +218,13 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
}
}

if err := sendMetric(ch, nodevGPUMemoryLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
if err := sendMetric(ch, nodevGPUMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodevGPUMemoryLimitDesc metric: %v", err)
}
if err := sendMetric(ch, nodevGPUCoreLimitDesc, prometheus.GaugeValue, coreLimit, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodevGPUCoreLimitDesc metric: %v", err)
}
if err := sendMetric(ch, nodevGPUMemoryAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type); err != nil {
if err := sendMetric(ch, nodevGPUMemoryAllocatedDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodevGPUMemoryAllocatedDesc metric: %v", err)
}
if err := sendMetric(ch, nodevGPUSharedNumDesc, prometheus.GaugeValue, float64(devs.Device.Used), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
Expand All @@ -222,7 +233,7 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
if err := sendMetric(ch, nodeGPUCoreAllocatedDesc, prometheus.GaugeValue, coreAllocated, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodeGPUCoreAllocatedDesc metric: %v", err)
}
if err := sendMetric(ch, nodeGPUOverview, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type); err != nil {
if err := sendMetric(ch, nodeGPUOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type); err != nil {
klog.V(4).Infof("Failed to send nodeGPUOverview metric: %v", err)
}

Expand All @@ -233,34 +244,35 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
}

if legacy {
sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyCoreLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalcore), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyMemoryAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type)
sendLegacyMetric(ch, legacyMemoryAllocatedDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type)
sendLegacyMetric(ch, legacySharedNumDesc, prometheus.GaugeValue, float64(devs.Device.Used), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyCoreAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedcores), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type)
sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type)
sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type)
if devs.Device.Totalmem > 0 {
sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index))
}
}
}
}
}

ctrvGPUdeviceAllocatedMemoryDesc := prometheus.NewDesc(
"hami_vgpu_memory_allocated_bytes",
"vGPU memory allocated from a container",
[]string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil,
)
ctrvGPUdeviceAllocatedCoreDesc := prometheus.NewDesc(
"hami_vgpu_core_allocated_ratio",
"vGPU core allocated from a container",
[]string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil,
)
// collectQuotaMetrics emits per-namespace resource quota usage.
func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metric, legacy bool) {
quotaUsedDesc := prometheus.NewDesc(
"hami_resource_quota_used",
"resourcequota usage for a certain device",
[]string{"namespace", "quota_name", "limit"}, nil,
)
var legacyQuotaUsed *prometheus.Desc
if legacy {
legacyQuotaUsed = prometheus.NewDesc(
"QuotaUsed",
"resourcequota usage for a certain device",
[]string{"quotanamespace", "quotaName", "limit"}, nil,
)
}
for ns, val := range cc.metricsProvider.GetQuotaManager().GetResourceQuota() {
for quotaname, q := range *val {
if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil {
Expand All @@ -271,6 +283,38 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
}
}
}
}

// collectContainerMetrics emits per-container vGPU metrics for all scheduled
// pods. AMD core allocations are normalized to a percentage via
// normalizeAMDCoreMetrics (issue #2518); legacy metrics keep raw values.
func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, nu *map[string]*schedulerpkg.NodeUsage, legacy bool) {
ctrvGPUdeviceAllocatedMemoryDesc := prometheus.NewDesc(
"hami_vgpu_memory_allocated_bytes",
"vGPU memory allocated from a container",
[]string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil,
)
ctrvGPUdeviceAllocatedCoreDesc := prometheus.NewDesc(
"hami_vgpu_core_allocated_ratio",
"vGPU core allocated from a container",
[]string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil,
)
var (
legacyAllocatedMemory *prometheus.Desc
legacyAllocatedCore *prometheus.Desc
)
if legacy {
legacyAllocatedMemory = prometheus.NewDesc(
"vGPUMemoryAllocated",
"vGPU memory allocated from a container",
[]string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
)
legacyAllocatedCore = prometheus.NewDesc(
"vGPUCoreAllocated",
"vGPU core allocated from a container",
[]string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil,
)
}
schedpods, _ := cc.metricsProvider.GetPodManager().GetScheduledPods()
for _, val := range schedpods {
for _, podSingleDevice := range val.Devices {
Expand All @@ -289,35 +333,30 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
val.Namespace, val.Name, ctridx, val.NodeID)
continue
}
if err := sendMetric(ch, ctrvGPUdeviceAllocatedMemoryDesc, prometheus.GaugeValue, float64(ctrdevval.Usedmem)*float64(1024)*float64(1024), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID); err != nil {
// Resolve the matching node device's total core capacity and type so
// AMD physical compute-unit (CU) counts in Usedcores can be normalized
// to the percentage unit used by hami_vgpu_core_allocated_ratio (#2518).
totalcore, deviceType, found := findNodeDeviceUsage(nu, ctrdevval.UUID)
klog.V(4).InfoS("Resolved device for container metric",
"deviceUUID", ctrdevval.UUID,
"totalCore", totalcore,
"deviceType", deviceType,
"found", found,
"nodeID", val.NodeID,
)
containerLabels := []string{val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID}
usedMemBytes := mibToBytes(ctrdevval.Usedmem)
if err := sendMetric(ch, ctrvGPUdeviceAllocatedMemoryDesc, prometheus.GaugeValue, usedMemBytes, containerLabels...); err != nil {
klog.V(4).Infof("Failed to send ctrvGPUdeviceAllocatedMemoryDesc metric: %v", err)
}
if err := sendMetric(ch, ctrvGPUdeviceAllocatedCoreDesc, prometheus.GaugeValue, float64(ctrdevval.Usedcores), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID); err != nil {
_, ctrCoreAllocated := normalizeAMDCoreMetrics(deviceType, totalcore, ctrdevval.Usedcores)
if err := sendMetric(ch, ctrvGPUdeviceAllocatedCoreDesc, prometheus.GaugeValue, ctrCoreAllocated, containerLabels...); err != nil {
klog.V(4).Infof("Failed to send ctrvGPUdeviceAllocatedCoreDesc metric: %v", err)
}
if legacy {
sendLegacyMetric(ch, legacyAllocatedMemory, prometheus.GaugeValue, float64(ctrdevval.Usedmem)*float64(1024)*float64(1024), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
sendLegacyMetric(ch, legacyAllocatedCore, prometheus.GaugeValue, float64(ctrdevval.Usedcores), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID)
sendLegacyMetric(ch, legacyAllocatedMemory, prometheus.GaugeValue, usedMemBytes, containerLabels...)
sendLegacyMetric(ch, legacyAllocatedCore, prometheus.GaugeValue, float64(ctrdevval.Usedcores), containerLabels...)
}
var totaldev int32
found := false
for _, ni := range *nu {
for _, nodedev := range ni.Devices.DeviceLists {
if strings.Compare(nodedev.Device.ID, ctrdevval.UUID) == 0 {
totaldev = nodedev.Device.Totalmem
found = true
break
}
}
if found {
break
}
}
klog.V(4).InfoS("Total memory for device",
"deviceUUID", ctrdevval.UUID,
"totalMemory", totaldev,
"nodeID", val.NodeID,
)
}
}
}
Expand Down
Loading
Loading