diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index ea4b4bf0b4..8fb92c27cd 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -88,62 +88,62 @@ var ( hostGPUdesc = prometheus.NewDesc( "hami_host_gpu_memory_used_bytes", "GPU device memory usage in bytes", - []string{"device_index", "device_uuid", "device_type"}, nil, + []string{"node_name", "device_index", "device_uuid", "device_type"}, nil, ) hostGPUUtilizationdesc = prometheus.NewDesc( "hami_host_gpu_utilization_ratio", "GPU core utilization ratio (0-100)", - []string{"device_index", "device_uuid", "device_type"}, nil, + []string{"node_name", "device_index", "device_uuid", "device_type"}, nil, ) ctrvGPUdesc = prometheus.NewDesc( "hami_vgpu_memory_used_bytes", "vGPU device memory usage in bytes", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrvGPUlimitdesc = prometheus.NewDesc( "hami_vgpu_memory_limit_bytes", "vGPU device memory limit in bytes", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceMemorydesc = prometheus.NewDesc( "hami_container_device_memory_bytes", `Container device memory usage in bytes`, - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceUtilizationdesc = prometheus.NewDesc( "hami_container_device_utilization_ratio", "Container device SM utilization ratio", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceLastKernelDesc = prometheus.NewDesc( "hami_container_last_kernel_elapsed_seconds", "Seconds since last kernel execution in container", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceMigInfo = prometheus.NewDesc( "hami_mig_device_info", "MIG device information for container", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid", "instance_id"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid", "instance_id"}, nil, ) ctrDeviceMemoryContextDesc = prometheus.NewDesc( "hami_vgpu_memory_context_bytes", "Container device memory context size in bytes", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceMemoryModuleDesc = prometheus.NewDesc( "hami_vgpu_memory_module_bytes", "Container device memory module size in bytes", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ctrDeviceMemoryBufferDesc = prometheus.NewDesc( "hami_vgpu_memory_buffer_bytes", "Container device memory buffer size in bytes", - []string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, + []string{"node_name", "namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil, ) ) @@ -163,42 +163,42 @@ func initLegacyDescriptors() { legacyHostGPUdesc = prometheus.NewDesc( "HostGPUMemoryUsage", "GPU device memory usage", - []string{"deviceidx", "deviceuuid", "devicetype"}, nil, + []string{"nodename", "deviceidx", "deviceuuid", "devicetype"}, nil, ) legacyHostGPUUtilizationdesc = prometheus.NewDesc( "HostCoreUtilization", "GPU core utilization", - []string{"deviceidx", "deviceuuid", "devicetype"}, nil, + []string{"nodename", "deviceidx", "deviceuuid", "devicetype"}, nil, ) legacyCtrvGPUdesc = prometheus.NewDesc( "vGPU_device_memory_usage_in_bytes", "vGPU device usage", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, ) legacyCtrvGPUlimitdesc = prometheus.NewDesc( "vGPU_device_memory_limit_in_bytes", "vGPU device limit", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, ) legacyCtrDeviceMemorydesc = prometheus.NewDesc( "Device_memory_desc_of_container", "Container device memory description", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil, ) legacyCtrDeviceUtilizationdesc = prometheus.NewDesc( "Device_utilization_desc_of_container", "Container device utilization description", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, ) legacyCtrDeviceLastKernelDesc = prometheus.NewDesc( "Device_last_kernel_of_container", "Container device last kernel description", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid"}, nil, ) legacyCtrDeviceMigInfo = prometheus.NewDesc( "MigInfo", "Mig device information for container", - []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "instanceid"}, nil, + []string{"nodename", "podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "instanceid"}, nil, ) } @@ -221,6 +221,8 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) { ch <- hostGPUUtilizationdesc ch <- ctrDeviceMemorydesc ch <- ctrDeviceUtilizationdesc + ch <- ctrDeviceLastKernelDesc + ch <- ctrDeviceMigInfo ch <- ctrDeviceMemoryContextDesc ch <- ctrDeviceMemoryModuleDesc ch <- ctrDeviceMemoryBufferDesc @@ -308,6 +310,11 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { } func (cc ClusterManagerCollector) collectGPUInfo(ch chan<- prometheus.Metric) error { + nodeName := getNodeName() + if nodeName == "" { + return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName) + } + if err := cc.initNVML(); err != nil { return err } @@ -319,7 +326,7 @@ func (cc ClusterManagerCollector) collectGPUInfo(ch chan<- prometheus.Metric) er } for ii := range devnum { - if err := cc.collectGPUDeviceMetrics(ch, ii); err != nil { + if err := cc.collectGPUDeviceMetrics(ch, nodeName, ii); err != nil { klog.Error("Failed to collect metrics for GPU device ", ii, ": ", err) } } @@ -343,24 +350,28 @@ func (cc ClusterManagerCollector) getDeviceCount() (int, error) { return devnum, nil } -func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.Metric, index int) error { +func (cc ClusterManagerCollector) collectGPUDeviceMetrics(ch chan<- prometheus.Metric, nodeName string, index int) error { hdev, nvret := nvml.DeviceGetHandleByIndex(index) if nvret != nvml.SUCCESS { return fmt.Errorf("nvml DeviceGetHandleByIndex err: %s", nvml.ErrorString(nvret)) } - if err := cc.collectGPUMemoryMetrics(ch, hdev, index); err != nil { + if err := cc.collectGPUMemoryMetrics(ch, nodeName, hdev, index); err != nil { return err } - if err := cc.collectGPUUtilizationMetrics(ch, hdev, index); err != nil { + if err := cc.collectGPUUtilizationMetrics(ch, nodeName, hdev, index); err != nil { return err } return nil } -func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error { +func getNodeName() string { + return os.Getenv(util.NodeNameEnvName) +} + +func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.Metric, nodeName string, hdev nvml.Device, index int) error { memory, ret := hdev.GetMemoryInfo() if ret == nvml.ERROR_NOT_SUPPORTED { klog.V(3).Infof("Memory metrics not supported for device %d (unified memory architecture), skipping", index) @@ -386,17 +397,17 @@ func (cc ClusterManagerCollector) collectGPUMemoryMetrics(ch chan<- prometheus.M hostGPUdesc, prometheus.GaugeValue, float64(memory.Used), - fmt.Sprint(index), uuid, deviceName, + nodeName, fmt.Sprint(index), uuid, deviceName, ) sendLegacyMetric(ch, legacyHostGPUdesc, prometheus.GaugeValue, float64(memory.Used), - fmt.Sprint(index), uuid, deviceName, + nodeName, fmt.Sprint(index), uuid, deviceName, ) return nil } -func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometheus.Metric, hdev nvml.Device, index int) error { +func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometheus.Metric, nodeName string, hdev nvml.Device, index int) error { util, nvret := hdev.GetUtilizationRates() if nvret != nvml.SUCCESS { return fmt.Errorf("nvml GetUtilizationRates err: %s", nvml.ErrorString(nvret)) @@ -418,18 +429,18 @@ func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometh hostGPUUtilizationdesc, prometheus.GaugeValue, float64(util.Gpu), - fmt.Sprint(index), uuid, deviceName, + nodeName, fmt.Sprint(index), uuid, deviceName, ) sendLegacyMetric(ch, legacyHostGPUUtilizationdesc, prometheus.GaugeValue, float64(util.Gpu), - fmt.Sprint(index), uuid, deviceName, + nodeName, fmt.Sprint(index), uuid, deviceName, ) return nil } func (cc ClusterManagerCollector) collectPodAndContainerInfo(ch chan<- prometheus.Metric) error { - nodeName := os.Getenv(util.NodeNameEnvName) + nodeName := getNodeName() if nodeName == "" { return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName) } @@ -466,7 +477,7 @@ func (cc ClusterManagerCollector) collectPodAndContainerInfo(ch chan<- prometheu for _, c := range podContainers { if c.ContainerName == ctr.Name { klog.V(5).Infof("Processing Container %s in Pod %s/%s", ctr.Name, pod.Namespace, pod.Name) - if err := cc.collectContainerMetrics(ch, pod, ctr, c, nowSec); err != nil { + if err := cc.collectContainerMetrics(ch, nodeName, pod, ctr, c, nowSec); err != nil { klog.Errorf("Failed to collect metrics for container %s in Pod %s/%s: %v", ctr.Name, pod.Namespace, pod.Name, err) } break // Exit the inner loop after finding the matching container @@ -479,7 +490,7 @@ func (cc ClusterManagerCollector) collectPodAndContainerInfo(ch chan<- prometheu return nil } -func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, pod *corev1.Pod, ctr corev1.Container, c *nvidia.ContainerUsage, nowSec int64) error { +func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.Metric, nodeName string, pod *corev1.Pod, ctr corev1.Container, c *nvidia.ContainerUsage, nowSec int64) error { // Validate inputs if c == nil || c.Info == nil { klog.Errorf("Container or ContainerInfo is nil for Pod %s/%s, Container %s", pod.Namespace, pod.Name, ctr.Name) @@ -508,7 +519,7 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M smUtil := c.Info.DeviceSmUtil(i) lastKernelTime := c.Info.LastKernelTime() - labels := []string{pod.Namespace, pod.Name, ctr.Name, fmt.Sprint(i), uuid} + labels := []string{nodeName, pod.Namespace, pod.Name, ctr.Name, fmt.Sprint(i), uuid} if err := sendMetric(ch, ctrvGPUdesc, prometheus.GaugeValue, float64(memoryTotal), labels...); err != nil { klog.Errorf("Failed to send memoryTotal metric: %v", err) @@ -564,7 +575,7 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M } func (cc ClusterManagerCollector) collectPodAndContainerMigInfo(ch chan<- prometheus.Metric) error { - nodeName := os.Getenv(util.NodeNameEnvName) + nodeName := getNodeName() if nodeName == "" { return fmt.Errorf("node name environment variable %s is not set", util.NodeNameEnvName) } @@ -598,7 +609,7 @@ func (cc ClusterManagerCollector) collectPodAndContainerMigInfo(ch chan<- promet klog.Errorf("Failed to get mig InstanceId for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err) continue } - labels := []string{pod.Namespace, pod.Name, container.Name, fmt.Sprint(idx), uuid, fmt.Sprint(gpuInstanceId)} + labels := []string{nodeName, pod.Namespace, pod.Name, container.Name, fmt.Sprint(idx), uuid, fmt.Sprint(gpuInstanceId)} if err := sendMetric(ch, ctrDeviceMigInfo, prometheus.GaugeValue, 1, labels...); err != nil { klog.Errorf("Failed to send mig info metric for device %s in Pod %s/%s, container %s: %v", ctrDev.UUID, pod.Namespace, pod.Name, container.Name, err) return err diff --git a/cmd/vGPUmonitor/metrics_test.go b/cmd/vGPUmonitor/metrics_test.go new file mode 100644 index 0000000000..84f917c441 --- /dev/null +++ b/cmd/vGPUmonitor/metrics_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/Project-HAMi/HAMi/pkg/util" +) + +func TestAllMetricDescriptorsIncludeNodeName(t *testing.T) { + ch := make(chan *prometheus.Desc, 30) + cc := ClusterManagerCollector{ + ClusterManager: &ClusterManager{ + LegacyMetrics: true, + }, + } + initLegacyDescriptors() + + cc.Describe(ch) + close(ch) + + descCount := 0 + for desc := range ch { + descCount++ + descStr := desc.String() + if strings.Contains(descStr, "fqName: \"hami_") { + if !strings.Contains(descStr, "node_name") { + t.Errorf("standard descriptor %s does not contain node_name label", descStr) + } + } else { + if !strings.Contains(descStr, "nodename") { + t.Errorf("legacy descriptor %s does not contain nodename label", descStr) + } + } + } + + expectedCount := 19 // 11 standard + 8 legacy descriptors + if descCount != expectedCount { + t.Errorf("expected exactly %d descriptors, got %d", expectedCount, descCount) + } +} + +func TestGetNodeName(t *testing.T) { + // Environment variable set + testNode := "test-gpu-node-01" + t.Setenv(util.NodeNameEnvName, testNode) + if name := getNodeName(); name != testNode { + t.Errorf("expected getNodeName() to return %s, got %s", testNode, name) + } + + // Environment variable unset + os.Unsetenv(util.NodeNameEnvName) + if name := getNodeName(); name != "" { + t.Errorf("expected getNodeName() to return empty string when env var is unset, got %s", name) + } +} + +func TestSendMetric(t *testing.T) { + ch := make(chan prometheus.Metric, 5) + + // Valid metric sending + err := sendMetric(ch, hostGPUdesc, prometheus.GaugeValue, 1024, "node-1", "0", "gpu-uuid-1", "NVIDIA-A100") + if err != nil { + t.Errorf("expected sendMetric to succeed, got %v", err) + } + + // Legacy metric sending + initLegacyDescriptors() + sendLegacyMetric(ch, legacyHostGPUdesc, prometheus.GaugeValue, 1024, "node-1", "0", "gpu-uuid-1", "NVIDIA-A100") + + close(ch) + count := 0 + for range ch { + count++ + } + if count != 2 { + t.Errorf("expected 2 metrics in channel, got %d", count) + } +}