feat: add hami_host_gpu_memory_controller_utilization_ratio metric

cmd/vGPUmonitor/metrics.go

@@ -67,6 +67,12 @@ var (
 		[]string{"node", "device_index", "device_uuid", "device_type"}, nil,
 	)
 
+	hostGPUMemoryUtilizationdesc = prometheus.NewDesc(
+		"hami_host_gpu_memory_controller_utilization_ratio",
+		"GPU memory controller utilization ratio (0-100)",
+		[]string{"device_index", "device_uuid", "device_type"}, nil,
+	)
+
 	ctrvGPUdesc = prometheus.NewDesc(
 		"hami_vgpu_memory_used_bytes",
 		"vGPU device memory usage in bytes",
@@ -188,6 +194,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- ctrvGPUdesc
 	ch <- ctrvGPUlimitdesc
 	ch <- hostGPUUtilizationdesc
+	ch <- hostGPUMemoryUtilizationdesc
 	ch <- ctrDeviceMemorydesc
 	ch <- ctrDeviceUtilizationdesc
 	ch <- ctrDeviceLastKernelDesc
@@ -363,6 +370,13 @@ func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometh
 		nodeName, fmt.Sprint(index), uuid, deviceName,
 	)
 
+	if err := sendMetric(ch, hostGPUMemoryUtilizationdesc, prometheus.GaugeValue,
+		float64(utilRates.Memory),
+		fmt.Sprint(index), uuid, deviceName,
+	); err != nil {
+		return fmt.Errorf("nvml send memory controller utilization: %w", err)
+	}
+
 	return nil
 }
 

cmd/vGPUmonitor/metrics_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	nvmlmock "github.com/NVIDIA/go-nvml/pkg/nvml/mock"
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
 
@@ -32,28 +33,23 @@ import (
 
 func TestDescribeCollectSync(t *testing.T) {
 	reg := prometheus.NewPedanticRegistry()
-
 	t.Setenv(util.NodeNameEnvName, "test-node")
 	client := fake.NewSimpleClientset()
 	informerFactory := informers.NewSharedInformerFactory(client, 0)
 	podLister := informerFactory.Core().V1().Pods().Lister()
-
 	c := &ClusterManager{
 		Zone:            "test-zone",
 		LegacyMetrics:   false,
 		PodLister:       podLister,
 		containerLister: &nvidia.ContainerLister{},
 	}
 	cc := ClusterManagerCollector{ClusterManager: c}
-
 	if err := reg.Register(cc); err != nil {
 		t.Fatalf("Failed to register ClusterManagerCollector (non-legacy): %v", err)
 	}
-
 	if _, err := reg.Gather(); err != nil {
 		t.Errorf("Gather failed (non-legacy): %v", err)
 	}
-
 	regLegacy := prometheus.NewPedanticRegistry()
 	cLegacy := &ClusterManager{
 		Zone:            "test-zone-legacy",
@@ -63,7 +59,6 @@ func TestDescribeCollectSync(t *testing.T) {
 	}
 	initLegacyDescriptors()
 	ccLegacy := ClusterManagerCollector{ClusterManager: cLegacy}
-
 	if err := regLegacy.Register(ccLegacy); err != nil {
 		t.Fatalf("Failed to register ClusterManagerCollector (legacy): %v", err)
 	}
@@ -75,7 +70,6 @@ func TestDescribeCollectSync(t *testing.T) {
 func TestHostGPUMetricsDescriptorsIncludeNodeLabel(t *testing.T) {
 	initLegacyDescriptors()
 
-	// Verify standard host GPU descriptors include "node" label
 	hostGPUString := hostGPUdesc.String()
 	if !strings.Contains(hostGPUString, `"node"`) && !strings.Contains(hostGPUString, `node`) {
 		t.Errorf("hostGPUdesc does not contain 'node' label: %s", hostGPUString)
@@ -86,7 +80,6 @@ func TestHostGPUMetricsDescriptorsIncludeNodeLabel(t *testing.T) {
 		t.Errorf("hostGPUUtilizationdesc does not contain 'node' label: %s", hostGPUUtilString)
 	}
 
-	// Verify legacy host GPU descriptors include "nodeid" label
 	legacyHostGPUString := legacyHostGPUdesc.String()
 	if !strings.Contains(legacyHostGPUString, `"nodeid"`) && !strings.Contains(legacyHostGPUString, `nodeid`) {
 		t.Errorf("legacyHostGPUdesc does not contain 'nodeid' label: %s", legacyHostGPUString)
@@ -103,13 +96,11 @@ func TestHostGPUMetricsMissingNodeName(t *testing.T) {
 
 	cc := ClusterManagerCollector{}
 
-	// Test collectGPUUtilizationMetrics with missing NODE_NAME
 	err := cc.collectGPUUtilizationMetrics(nil, nil, 0)
 	if err == nil || !strings.Contains(err.Error(), "node name environment variable") {
 		t.Errorf("expected missing node name error from collectGPUUtilizationMetrics, got: %v", err)
 	}
 
-	// Test collectGPUMemoryMetrics with missing NODE_NAME
 	mockDev := &nvmlmock.Device{
 		GetMemoryInfoFunc: func() (nvml.Memory, nvml.Return) {
 			return nvml.Memory{Used: 100}, nvml.SUCCESS
@@ -170,3 +161,56 @@ func TestHostGPUMetricsCollectionSuccess(t *testing.T) {
 		t.Errorf("expected at least 4 metrics, got %d", count)
 	}
 }
+
+func TestDescribeRegistersMemoryControllerUtilization(t *testing.T) {
+	c := &ClusterManager{Zone: "test-zone", LegacyMetrics: false}
+	cc := ClusterManagerCollector{ClusterManager: c}
+	descCh := make(chan *prometheus.Desc, 32)
+	cc.Describe(descCh)
+	close(descCh)
+	for d := range descCh {
+		if strings.Contains(d.String(), "hami_host_gpu_memory_controller_utilization_ratio") {
+			return
+		}
+	}
+	t.Error("hami_host_gpu_memory_controller_utilization_ratio not found in Describe output")
+}
+
+func TestCollectMemoryControllerUtilizationValue(t *testing.T) {
+	t.Setenv(util.NodeNameEnvName, "test-node")
+	const wantMemory = uint32(73)
+	mockDev := &nvmlmock.Device{
+		GetUtilizationRatesFunc: func() (nvml.Utilization, nvml.Return) {
+			return nvml.Utilization{Gpu: 10, Memory: wantMemory}, nvml.SUCCESS
+		},
+		GetUUIDFunc: func() (string, nvml.Return) {
+			return "GPU-abc123", nvml.SUCCESS
+		},
+		GetNameFunc: func() (string, nvml.Return) {
+			return "A100", nvml.SUCCESS
+		},
+	}
+	ch := make(chan prometheus.Metric, 10)
+	c := &ClusterManager{Zone: "test-zone", LegacyMetrics: false}
+	cc := ClusterManagerCollector{ClusterManager: c}
+	if err := cc.collectGPUUtilizationMetrics(ch, mockDev, 0); err != nil {
+		t.Fatalf("collectGPUUtilizationMetrics: %v", err)
+	}
+	close(ch)
+	var found bool
+	for m := range ch {
+		var dm dto.Metric
+		if err := m.Write(&dm); err != nil {
+			continue
+		}
+		if dm.Gauge == nil {
+			continue
+		}
+		if *dm.Gauge.Value == float64(wantMemory) {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("expected memory controller utilization metric with value %v", wantMemory)
+	}
+}