diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index f943a6349e..8c76d8deef 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -70,6 +70,12 @@ var ( []string{"device_index", "device_uuid", "device_type"}, nil, ) + hostGPUMemoryUtilizationdesc = prometheus.NewDesc( + "hami_host_gpu_memory_controller_utilization_ratio", + "GPU memory controller utilization ratio (0-100)", + []string{"device_index", "device_uuid", "device_type"}, nil, + ) + ctrvGPUdesc = prometheus.NewDesc( "hami_vgpu_memory_used_bytes", "vGPU device memory usage in bytes", @@ -191,6 +197,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) { ch <- ctrvGPUdesc ch <- ctrvGPUlimitdesc ch <- hostGPUUtilizationdesc + ch <- hostGPUMemoryUtilizationdesc ch <- ctrDeviceMemorydesc ch <- ctrDeviceUtilizationdesc ch <- ctrDeviceLastKernelDesc @@ -356,6 +363,13 @@ func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometh fmt.Sprint(index), uuid, deviceName, ) + if err := sendMetric(ch, hostGPUMemoryUtilizationdesc, prometheus.GaugeValue, + float64(util.Memory), + fmt.Sprint(index), uuid, deviceName, + ); err != nil { + return fmt.Errorf("nvml send memory controller utilization: %w", err) + } + return nil } diff --git a/cmd/vGPUmonitor/metrics_test.go b/cmd/vGPUmonitor/metrics_test.go index 2c31e4d221..6cb1feeb4c 100644 --- a/cmd/vGPUmonitor/metrics_test.go +++ b/cmd/vGPUmonitor/metrics_test.go @@ -17,8 +17,10 @@ limitations under the License. package main import ( + "strings" "testing" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -29,12 +31,10 @@ import ( func TestDescribeCollectSync(t *testing.T) { reg := prometheus.NewPedanticRegistry() - t.Setenv(util.NodeNameEnvName, "test-node") client := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) podLister := informerFactory.Core().V1().Pods().Lister() - c := &ClusterManager{ Zone: "test-zone", LegacyMetrics: false, @@ -42,15 +42,12 @@ func TestDescribeCollectSync(t *testing.T) { containerLister: &nvidia.ContainerLister{}, } cc := ClusterManagerCollector{ClusterManager: c} - if err := reg.Register(cc); err != nil { t.Fatalf("Failed to register ClusterManagerCollector (non-legacy): %v", err) } - if _, err := reg.Gather(); err != nil { t.Errorf("Gather failed (non-legacy): %v", err) } - regLegacy := prometheus.NewPedanticRegistry() cLegacy := &ClusterManager{ Zone: "test-zone-legacy", @@ -60,7 +57,6 @@ func TestDescribeCollectSync(t *testing.T) { } initLegacyDescriptors() ccLegacy := ClusterManagerCollector{ClusterManager: cLegacy} - if err := regLegacy.Register(ccLegacy); err != nil { t.Fatalf("Failed to register ClusterManagerCollector (legacy): %v", err) } @@ -68,3 +64,40 @@ func TestDescribeCollectSync(t *testing.T) { t.Errorf("Gather failed (legacy): %v", err) } } + +func TestDescribeRegistersMemoryControllerUtilization(t *testing.T) { + c := &ClusterManager{Zone: "test-zone", LegacyMetrics: false} + cc := ClusterManagerCollector{ClusterManager: c} + descCh := make(chan *prometheus.Desc, 32) + cc.Describe(descCh) + close(descCh) + for d := range descCh { + if strings.Contains(d.String(), "hami_host_gpu_memory_controller_utilization_ratio") { + return + } + } + t.Error("hami_host_gpu_memory_controller_utilization_ratio not found in Describe output") +} + +func TestCollectMemoryControllerUtilizationValue(t *testing.T) { + const wantVal = float64(73) + m, err := prometheus.NewConstMetric( + hostGPUMemoryUtilizationdesc, + prometheus.GaugeValue, + wantVal, + "0", "GPU-abc123", "NVIDIA-A100", + ) + if err != nil { + t.Fatalf("NewConstMetric: %v", err) + } + var dm dto.Metric + if err := m.Write(&dm); err != nil { + t.Fatalf("Write: %v", err) + } + if dm.Gauge == nil { + t.Fatal("expected gauge metric, got nil") + } + if *dm.Gauge.Value != wantVal { + t.Fatalf("want %v, got %v", wantVal, *dm.Gauge.Value) + } +}