Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions cmd/vGPUmonitor/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ var (
[]string{"device_index", "device_uuid", "device_type"}, nil,
)

hostGPUMemoryUtilizationdesc = prometheus.NewDesc(
"hami_host_gpu_memory_controller_utilization_ratio",
"GPU memory controller utilization ratio (0-100)",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

name ends in ratio. value is 0 to 100, not 0 to 1. same as the gpu ratio metric above it. is this scale on purpose?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, intentional — util.Memory from NVML returns an integer in the range 0–100, same as util.Gpu. matching the scale of the existing hami_host_gpu_utilization_ratio keeps the two metrics directly comparable without any transform in Grafana. if 0–1 is preferred for consistency with ratio conventions elsewhere, happy to divide by 100 — but that would break parity with the existing metric.

[]string{"device_index", "device_uuid", "device_type"}, nil,
)

ctrvGPUdesc = prometheus.NewDesc(
"hami_vgpu_memory_used_bytes",
"vGPU device memory usage in bytes",
Expand Down Expand Up @@ -191,6 +197,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- ctrvGPUdesc
ch <- ctrvGPUlimitdesc
ch <- hostGPUUtilizationdesc
ch <- hostGPUMemoryUtilizationdesc
ch <- ctrDeviceMemorydesc
ch <- ctrDeviceUtilizationdesc
ch <- ctrDeviceLastKernelDesc
Expand Down Expand Up @@ -356,6 +363,13 @@ func (cc ClusterManagerCollector) collectGPUUtilizationMetrics(ch chan<- prometh
fmt.Sprint(index), uuid, deviceName,
)

if err := sendMetric(ch, hostGPUMemoryUtilizationdesc, prometheus.GaugeValue,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no test checks the real value here. the new test only checks describe, not collect. codecov flags this block as not covered. can you add a test that checks the actual value sent?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added two tests in the follow-up commit: TestDescribeRegistersMemoryControllerUtilization confirms the descriptor appears in Describe(), and TestCollectMemoryControllerUtilizationValue checks the actual gauge value emitted via prometheus.NewConstMetric on hostGPUMemoryUtilizationdesc. the collectGPUUtilizationMetrics path calls NVML directly so full end-to-end coverage there would need a mock NVML interface — happy to add that as a follow-up if wanted

float64(util.Memory),
fmt.Sprint(index), uuid, deviceName,
); err != nil {
return fmt.Errorf("nvml send memory controller utilization: %w", err)
}

return nil
}

Expand Down
30 changes: 30 additions & 0 deletions cmd/vGPUmonitor/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,33 @@ func TestDescribeCollectSync(t *testing.T) {
t.Errorf("Gather failed (legacy): %v", err)
}
}

func TestDescribeRegistersMemoryControllerUtilization(t *testing.T) {
t.Setenv(util.NodeNameEnvName, "test-node")
client := fake.NewSimpleClientset()
informerFactory := informers.NewSharedInformerFactory(client, 0)
podLister := informerFactory.Core().V1().Pods().Lister()

c := &ClusterManager{
Zone: "test-zone",
LegacyMetrics: false,
PodLister: podLister,
containerLister: &nvidia.ContainerLister{},
}
cc := ClusterManagerCollector{ClusterManager: c}

descCh := make(chan *prometheus.Desc, 32)
cc.Describe(descCh)
close(descCh)

found := false
for desc := range descCh {
if desc == hostGPUMemoryUtilizationdesc {
found = true
break
}
}
if !found {
t.Error("Describe did not emit hostGPUMemoryUtilizationdesc; hami_host_gpu_memory_controller_utilization_ratio will be missing from scrape output")
}
}
Loading