diff --git a/cmd/vGPUmonitor/feedback_test.go b/cmd/vGPUmonitor/feedback_test.go index 10c2177790..e2a557b219 100644 --- a/cmd/vGPUmonitor/feedback_test.go +++ b/cmd/vGPUmonitor/feedback_test.go @@ -38,6 +38,7 @@ type stubInfo struct { ctxSize []uint64 modSize []uint64 bufSize []uint64 + offset []uint64 smUtil []uint64 lastKernel int64 } @@ -60,7 +61,7 @@ func (s *stubInfo) DeviceUUID(i int) string { func (s *stubInfo) DeviceMemoryContextSize(i int) uint64 { return slot(s.ctxSize, i) } func (s *stubInfo) DeviceMemoryModuleSize(i int) uint64 { return slot(s.modSize, i) } func (s *stubInfo) DeviceMemoryBufferSize(i int) uint64 { return slot(s.bufSize, i) } -func (s *stubInfo) DeviceMemoryOffset(int) uint64 { return 0 } +func (s *stubInfo) DeviceMemoryOffset(i int) uint64 { return slot(s.offset, i) } func (s *stubInfo) DeviceMemoryTotal(i int) uint64 { return slot(s.total, i) } func (s *stubInfo) DeviceSmUtil(i int) uint64 { return slot(s.smUtil, i) } func (s *stubInfo) SetDeviceSmLimit(uint64) {} diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index f93e3c803b..4cc16a16ce 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -463,7 +463,8 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M klog.Errorf("Failed to send device memory desc: %v", err) return err } - memoryOffset := memoryTotal - memoryContextSize - memoryModuleSize - memoryBufferSize + // Send legacy metric with additional memory details + memoryOffset := c.Info.DeviceMemoryOffset(i) // Get the memory offset for the device memoryLabels := append(labels, fmt.Sprint(memoryContextSize), fmt.Sprint(memoryModuleSize), fmt.Sprint(memoryBufferSize), fmt.Sprint(memoryOffset)) sendLegacyMetric(ch, legacyCtrDeviceMemorydesc, prometheus.GaugeValue, float64(memoryTotal), memoryLabels...) diff --git a/cmd/vGPUmonitor/metrics_container_test.go b/cmd/vGPUmonitor/metrics_container_test.go index 46c53a5606..fe6bded71e 100644 --- a/cmd/vGPUmonitor/metrics_container_test.go +++ b/cmd/vGPUmonitor/metrics_container_test.go @@ -154,3 +154,47 @@ func TestCollectContainerMetricsSkipsInvalidUTF8UUID(t *testing.T) { t.Errorf("got %d metrics for an invalid UUID, want 0", len(metrics)) } } + +// TestCollectContainerMetricsLegacyOffsetNoUnderflow verifies that the legacy +// device-memory offset label uses the recorded value without uint64 underflow. +func TestCollectContainerMetricsLegacyOffsetNoUnderflow(t *testing.T) { + legacyCtrDeviceMemorydesc = prometheus.NewDesc( + "Device_memory_desc_of_container", + "Container device memory description", + []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil, + ) + t.Cleanup(func() { legacyCtrDeviceMemorydesc = prev }) + + cu := &nvidia.ContainerUsage{Info: &stubInfo{ + uuids: []string{testUUID}, + total: []uint64{100}, + ctxSize: []uint64{1000}, + modSize: []uint64{500}, + bufSize: []uint64{300}, + offset: []uint64{7}, + }} + + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "trainer-0"}} + ctr := corev1.Container{Name: "worker"} + cc := ClusterManagerCollector{ClusterManager: &ClusterManager{LegacyMetrics: true}} + ch := make(chan prometheus.Metric, 64) + if err := cc.collectContainerMetrics(ch, pod, ctr, cu, 0); err != nil { + t.Fatal(err) + } + close(ch) + + found := false + for m := range ch { + if m.Desc() != legacyCtrDeviceMemorydesc { + continue + } + found = true + _, labels := gaugeValue(t, m) + if labels["offset"] != "7" { + t.Errorf("offset label = %q, want %q (got a wrapped/underflowed value instead of the recorded offset)", labels["offset"], "7") + } + } + if !found { + t.Fatal("legacy device memory metric was not emitted") + } +} \ No newline at end of file