diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index d652582253..5412256225 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -62,6 +62,26 @@ func normalizeAMDCoreMetrics(deviceType string, total, allocated int32) (float64 return normalizedCoreLimit, math.Ceil(float64(allocated) / float64(total) * normalizedCoreLimit) } +// findNodeDeviceUsage looks up a device by UUID across every node's usage and +// returns its total core capacity and type. ok is false when no node advertises +// the device, in which case the caller falls back to emitting raw values. +func findNodeDeviceUsage(nu *map[string]*schedulerpkg.NodeUsage, uuid string) (totalcore int32, deviceType string, ok bool) { + for _, ni := range *nu { + for _, dls := range ni.Devices.DeviceLists { + if dls.Device != nil && dls.Device.ID == uuid { + return dls.Device.Totalcore, dls.Device.Type, true + } + } + } + return 0, "", false +} + +// mibToBytes converts a memory quantity expressed in mebibytes (MiB), the unit +// used throughout HAMi's device accounting, to bytes for byte-oriented metrics. +func mibToBytes(mib int32) float64 { + return float64(mib) * 1024 * 1024 +} + // Describe is implemented with DescribeByCollect. That's possible because the // Collect method will always return the same metrics with the same descriptors. func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) { @@ -72,8 +92,18 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) { func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { klog.V(3).Info("Starting to collect metrics for scheduler") legacy := cc.ClusterManager.LegacyMetrics + // A single snapshot is shared by the node- and container-level collectors so + // they observe a consistent cluster state within one scrape. + nu := cc.metricsProvider.InspectAllNodesUsage() + cc.collectNodeMetrics(ch, nu, legacy) + cc.collectQuotaMetrics(ch, legacy) + cc.collectContainerMetrics(ch, nu, legacy) +} - // New metric descriptors +// collectNodeMetrics emits node-level GPU metrics (memory/core limits and +// allocations, sharing counts, MIG instances and overview) for every device on +// every node. AMD core values are normalized to a percentage. +func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric, nu *map[string]*schedulerpkg.NodeUsage, legacy bool) { nodevGPUMemoryLimitDesc := prometheus.NewDesc( "hami_gpu_memory_limit_bytes", "Device memory limit for a certain GPU", @@ -125,9 +155,6 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { legacyOverview *prometheus.Desc legacyMemoryPercentage *prometheus.Desc legacyMigInstance *prometheus.Desc - legacyAllocatedMemory *prometheus.Desc - legacyAllocatedCore *prometheus.Desc - legacyQuotaUsed *prometheus.Desc ) if legacy { legacyMemoryLimitDesc = prometheus.NewDesc( @@ -170,24 +197,8 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { "GPU Sharing mode. 0 for hami-core, 1 for mig, 2 for mps", []string{"nodeid", "deviceuuid", "deviceidx", "migname"}, nil, ) - legacyAllocatedMemory = prometheus.NewDesc( - "vGPUMemoryAllocated", - "vGPU memory allocated from a container", - []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil, - ) - legacyAllocatedCore = prometheus.NewDesc( - "vGPUCoreAllocated", - "vGPU core allocated from a container", - []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil, - ) - legacyQuotaUsed = prometheus.NewDesc( - "QuotaUsed", - "resourcequota usage for a certain device", - []string{"quotanamespace", "quotaName", "limit"}, nil, - ) } - nu := cc.metricsProvider.InspectAllNodesUsage() for nodeID, val := range *nu { for _, devs := range val.Devices.DeviceLists { coreLimit, coreAllocated := normalizeAMDCoreMetrics(devs.Device.Type, devs.Device.Totalcore, devs.Device.Usedcores) @@ -207,13 +218,13 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { } } - if err := sendMetric(ch, nodevGPUMemoryLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { + if err := sendMetric(ch, nodevGPUMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodevGPUMemoryLimitDesc metric: %v", err) } if err := sendMetric(ch, nodevGPUCoreLimitDesc, prometheus.GaugeValue, coreLimit, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodevGPUCoreLimitDesc metric: %v", err) } - if err := sendMetric(ch, nodevGPUMemoryAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type); err != nil { + if err := sendMetric(ch, nodevGPUMemoryAllocatedDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodevGPUMemoryAllocatedDesc metric: %v", err) } if err := sendMetric(ch, nodevGPUSharedNumDesc, prometheus.GaugeValue, float64(devs.Device.Used), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { @@ -222,7 +233,7 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { if err := sendMetric(ch, nodeGPUCoreAllocatedDesc, prometheus.GaugeValue, coreAllocated, nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodeGPUCoreAllocatedDesc metric: %v", err) } - if err := sendMetric(ch, nodeGPUOverview, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type); err != nil { + if err := sendMetric(ch, nodeGPUOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type); err != nil { klog.V(4).Infof("Failed to send nodeGPUOverview metric: %v", err) } @@ -233,34 +244,35 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { } if legacy { - sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) + sendLegacyMetric(ch, legacyMemoryLimitDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) sendLegacyMetric(ch, legacyCoreLimitDesc, prometheus.GaugeValue, float64(devs.Device.Totalcore), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) - sendLegacyMetric(ch, legacyMemoryAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type) + sendLegacyMetric(ch, legacyMemoryAllocatedDesc, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), devs.Device.Type) sendLegacyMetric(ch, legacySharedNumDesc, prometheus.GaugeValue, float64(devs.Device.Used), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) sendLegacyMetric(ch, legacyCoreAllocatedDesc, prometheus.GaugeValue, float64(devs.Device.Usedcores), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), devs.Device.Type) - sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, float64(devs.Device.Usedmem)*float64(1024)*float64(1024), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type) + sendLegacyMetric(ch, legacyOverview, prometheus.GaugeValue, mibToBytes(devs.Device.Usedmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index), fmt.Sprint(devs.Device.Totalcore), fmt.Sprint(devs.Device.Totalmem), devs.Device.Type) if devs.Device.Totalmem > 0 { sendLegacyMetric(ch, legacyMemoryPercentage, prometheus.GaugeValue, float64(devs.Device.Usedmem)/float64(devs.Device.Totalmem), nodeID, devs.Device.ID, fmt.Sprint(devs.Device.Index)) } } } } +} - ctrvGPUdeviceAllocatedMemoryDesc := prometheus.NewDesc( - "hami_vgpu_memory_allocated_bytes", - "vGPU memory allocated from a container", - []string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil, - ) - ctrvGPUdeviceAllocatedCoreDesc := prometheus.NewDesc( - "hami_vgpu_core_allocated_ratio", - "vGPU core allocated from a container", - []string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil, - ) +// collectQuotaMetrics emits per-namespace resource quota usage. +func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metric, legacy bool) { quotaUsedDesc := prometheus.NewDesc( "hami_resource_quota_used", "resourcequota usage for a certain device", []string{"namespace", "quota_name", "limit"}, nil, ) + var legacyQuotaUsed *prometheus.Desc + if legacy { + legacyQuotaUsed = prometheus.NewDesc( + "QuotaUsed", + "resourcequota usage for a certain device", + []string{"quotanamespace", "quotaName", "limit"}, nil, + ) + } for ns, val := range cc.metricsProvider.GetQuotaManager().GetResourceQuota() { for quotaname, q := range *val { if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil { @@ -271,6 +283,38 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { } } } +} + +// collectContainerMetrics emits per-container vGPU metrics for all scheduled +// pods. AMD core allocations are normalized to a percentage via +// normalizeAMDCoreMetrics (issue #2518); legacy metrics keep raw values. +func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, nu *map[string]*schedulerpkg.NodeUsage, legacy bool) { + ctrvGPUdeviceAllocatedMemoryDesc := prometheus.NewDesc( + "hami_vgpu_memory_allocated_bytes", + "vGPU memory allocated from a container", + []string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil, + ) + ctrvGPUdeviceAllocatedCoreDesc := prometheus.NewDesc( + "hami_vgpu_core_allocated_ratio", + "vGPU core allocated from a container", + []string{"namespace", "node", "pod", "container_index", "device_uuid"}, nil, + ) + var ( + legacyAllocatedMemory *prometheus.Desc + legacyAllocatedCore *prometheus.Desc + ) + if legacy { + legacyAllocatedMemory = prometheus.NewDesc( + "vGPUMemoryAllocated", + "vGPU memory allocated from a container", + []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil, + ) + legacyAllocatedCore = prometheus.NewDesc( + "vGPUCoreAllocated", + "vGPU core allocated from a container", + []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil, + ) + } schedpods, _ := cc.metricsProvider.GetPodManager().GetScheduledPods() for _, val := range schedpods { for _, podSingleDevice := range val.Devices { @@ -289,35 +333,30 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { val.Namespace, val.Name, ctridx, val.NodeID) continue } - if err := sendMetric(ch, ctrvGPUdeviceAllocatedMemoryDesc, prometheus.GaugeValue, float64(ctrdevval.Usedmem)*float64(1024)*float64(1024), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID); err != nil { + // Resolve the matching node device's total core capacity and type so + // AMD physical compute-unit (CU) counts in Usedcores can be normalized + // to the percentage unit used by hami_vgpu_core_allocated_ratio (#2518). + totalcore, deviceType, found := findNodeDeviceUsage(nu, ctrdevval.UUID) + klog.V(4).InfoS("Resolved device for container metric", + "deviceUUID", ctrdevval.UUID, + "totalCore", totalcore, + "deviceType", deviceType, + "found", found, + "nodeID", val.NodeID, + ) + containerLabels := []string{val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID} + usedMemBytes := mibToBytes(ctrdevval.Usedmem) + if err := sendMetric(ch, ctrvGPUdeviceAllocatedMemoryDesc, prometheus.GaugeValue, usedMemBytes, containerLabels...); err != nil { klog.V(4).Infof("Failed to send ctrvGPUdeviceAllocatedMemoryDesc metric: %v", err) } - if err := sendMetric(ch, ctrvGPUdeviceAllocatedCoreDesc, prometheus.GaugeValue, float64(ctrdevval.Usedcores), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID); err != nil { + _, ctrCoreAllocated := normalizeAMDCoreMetrics(deviceType, totalcore, ctrdevval.Usedcores) + if err := sendMetric(ch, ctrvGPUdeviceAllocatedCoreDesc, prometheus.GaugeValue, ctrCoreAllocated, containerLabels...); err != nil { klog.V(4).Infof("Failed to send ctrvGPUdeviceAllocatedCoreDesc metric: %v", err) } if legacy { - sendLegacyMetric(ch, legacyAllocatedMemory, prometheus.GaugeValue, float64(ctrdevval.Usedmem)*float64(1024)*float64(1024), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID) - sendLegacyMetric(ch, legacyAllocatedCore, prometheus.GaugeValue, float64(ctrdevval.Usedcores), val.Namespace, val.NodeID, val.Name, fmt.Sprint(ctridx), ctrdevval.UUID) + sendLegacyMetric(ch, legacyAllocatedMemory, prometheus.GaugeValue, usedMemBytes, containerLabels...) + sendLegacyMetric(ch, legacyAllocatedCore, prometheus.GaugeValue, float64(ctrdevval.Usedcores), containerLabels...) } - var totaldev int32 - found := false - for _, ni := range *nu { - for _, nodedev := range ni.Devices.DeviceLists { - if strings.Compare(nodedev.Device.ID, ctrdevval.UUID) == 0 { - totaldev = nodedev.Device.Totalmem - found = true - break - } - } - if found { - break - } - } - klog.V(4).InfoS("Total memory for device", - "deviceUUID", ctrdevval.UUID, - "totalMemory", totaldev, - "nodeID", val.NodeID, - ) } } } diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 60f19e98b0..3fcff7813c 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -142,6 +142,168 @@ func TestSchedulerDescribeCollectSync(t *testing.T) { } } +func TestMibToBytes(t *testing.T) { + tests := []struct { + name string + mib int32 + want float64 + }{ + {name: "zero", mib: 0, want: 0}, + {name: "one mebibyte", mib: 1, want: 1024 * 1024}, + {name: "typical device memory", mib: 16384, want: 16384 * 1024 * 1024}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := mibToBytes(tc.mib); got != tc.want { + t.Errorf("mibToBytes(%d) = %v, want %v", tc.mib, got, tc.want) + } + }) + } +} + +func TestFindNodeDeviceUsage(t *testing.T) { + nodeUsage := map[string]*schedulerpkg.NodeUsage{ + "node-1": { + Devices: policy.DeviceUsageList{ + DeviceLists: []*policy.DeviceListsScore{ + {Device: &device.DeviceUsage{ID: "AMD-1", Totalcore: 64, Type: "AMDGPU"}}, + {Device: &device.DeviceUsage{ID: "NVIDIA-1", Totalcore: 100, Type: "NVIDIA"}}, + }, + }, + }, + "node-2": { + Devices: policy.DeviceUsageList{ + DeviceLists: []*policy.DeviceListsScore{ + {Device: &device.DeviceUsage{ID: "AMD-2", Totalcore: 304, Type: "AMDGPU"}}, + }, + }, + }, + } + + tests := []struct { + name string + uuid string + wantTotalcore int32 + wantDeviceType string + wantOk bool + }{ + {name: "found on first node", uuid: "AMD-1", wantTotalcore: 64, wantDeviceType: "AMDGPU", wantOk: true}, + {name: "found on second node", uuid: "AMD-2", wantTotalcore: 304, wantDeviceType: "AMDGPU", wantOk: true}, + {name: "non-AMD device", uuid: "NVIDIA-1", wantTotalcore: 100, wantDeviceType: "NVIDIA", wantOk: true}, + {name: "unknown device returns not ok", uuid: "missing", wantTotalcore: 0, wantDeviceType: "", wantOk: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotTotalcore, gotDeviceType, gotOk := findNodeDeviceUsage(&nodeUsage, tc.uuid) + if gotTotalcore != tc.wantTotalcore || gotDeviceType != tc.wantDeviceType || gotOk != tc.wantOk { + t.Errorf("findNodeDeviceUsage(%q) = (%d, %q, %v), want (%d, %q, %v)", + tc.uuid, gotTotalcore, gotDeviceType, gotOk, tc.wantTotalcore, tc.wantDeviceType, tc.wantOk) + } + }) + } +} + +func TestAMDCoreAllocatedRatioNormalization(t *testing.T) { + // Regression test for issue #2518: AMD stores physical compute-unit (CU) + // counts in Usedcores, so a 50% request on a 64-CU device is stored as 32 CUs. + // Both the container-level hami_vgpu_core_allocated_ratio and the node-level + // hami_gpu_core_allocated_ratio must be normalized to 50, while legacy metrics + // keep the raw CU count (32) for backward compatibility. + nodeUsage := map[string]*schedulerpkg.NodeUsage{ + "node-1": { + Devices: policy.DeviceUsageList{ + DeviceLists: []*policy.DeviceListsScore{ + { + Device: &device.DeviceUsage{ + ID: "AMD-1", + Index: 0, + Count: 1, + Totalmem: 192000, + Totalcore: 64, + Usedcores: 32, + Type: "AMDGPU", + Mode: "hami-core", + }, + }, + }, + }, + }, + } + + pm := device.NewPodManager() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amd-pod", + Namespace: "default", + UID: k8stypes.UID("uid-amd"), + }, + } + // A 50% core request on a 64-CU device is stored as 32 physical CUs. + podDevices := device.PodDevices{ + "AMDGPU": device.PodSingleDevice{ + device.ContainerDevices{ + { + UUID: "AMD-1", + Type: "AMDGPU", + Usedmem: 96000, + Usedcores: 32, + }, + }, + }, + } + pm.AddPod(pod, "node-1", podDevices) + + newCollector := ClusterManagerCollector{ + ClusterManager: &ClusterManager{LegacyMetrics: false}, + metricsProvider: &fakeMetricsProvider{ + nodeUsage: nodeUsage, + quotaManager: device.NewQuotaManager(), + podManager: pm, + }, + } + + // Non-legacy metrics: both node- and container-level core allocation ratios + // are normalized to a percentage for AMD devices. + newWant := ` +# HELP hami_gpu_core_allocated_ratio Device core allocated for a certain GPU +# TYPE hami_gpu_core_allocated_ratio gauge +hami_gpu_core_allocated_ratio{device_index="0",device_type="AMDGPU",device_uuid="AMD-1",node="node-1"} 50 +# HELP hami_vgpu_core_allocated_ratio vGPU core allocated from a container +# TYPE hami_vgpu_core_allocated_ratio gauge +hami_vgpu_core_allocated_ratio{container_index="0",device_uuid="AMD-1",namespace="default",node="node-1",pod="amd-pod"} 50 +` + if err := promtestutil.CollectAndCompare( + newCollector, + strings.NewReader(newWant), + "hami_gpu_core_allocated_ratio", + "hami_vgpu_core_allocated_ratio", + ); err != nil { + t.Fatalf("unexpected non-legacy collecting result:\n%s", err) + } + + // Legacy metrics keep the raw CU count for backward compatibility. + legacyCollector := ClusterManagerCollector{ + ClusterManager: &ClusterManager{LegacyMetrics: true}, + metricsProvider: &fakeMetricsProvider{ + nodeUsage: nodeUsage, + quotaManager: device.NewQuotaManager(), + podManager: pm, + }, + } + legacyWant := ` +# HELP vGPUCoreAllocated vGPU core allocated from a container +# TYPE vGPUCoreAllocated gauge +vGPUCoreAllocated{containeridx="0",deviceuuid="AMD-1",nodename="node-1",podname="amd-pod",podnamespace="default"} 32 +` + if err := promtestutil.CollectAndCompare( + legacyCollector, + strings.NewReader(legacyWant), + "vGPUCoreAllocated", + ); err != nil { + t.Fatalf("unexpected legacy collecting result:\n%s", err) + } +} + func TestClusterManagerCollectorSkipsMemoryRatioWithNonPositiveTotalMemory(t *testing.T) { nodeUsage := map[string]*schedulerpkg.NodeUsage{ "node-1": {