From 791b29ee46cde8efd9e44d2d5fab88d04ae46719 Mon Sep 17 00:00:00 2001 From: Venkat Date: Tue, 11 Aug 2026 00:09:02 +0530 Subject: [PATCH 1/3] fix(monitor): avoid uint64 underflow when deriving the legacy device-memory offset label Signed-off-by: Venkat --- cmd/vGPUmonitor/feedback_test.go | 3 +- cmd/vGPUmonitor/metrics.go | 3 +- cmd/vGPUmonitor/metrics_container_test.go | 46 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/cmd/vGPUmonitor/feedback_test.go b/cmd/vGPUmonitor/feedback_test.go index 10c2177790..e2a557b219 100644 --- a/cmd/vGPUmonitor/feedback_test.go +++ b/cmd/vGPUmonitor/feedback_test.go @@ -38,6 +38,7 @@ type stubInfo struct { ctxSize []uint64 modSize []uint64 bufSize []uint64 + offset []uint64 smUtil []uint64 lastKernel int64 } @@ -60,7 +61,7 @@ func (s *stubInfo) DeviceUUID(i int) string { func (s *stubInfo) DeviceMemoryContextSize(i int) uint64 { return slot(s.ctxSize, i) } func (s *stubInfo) DeviceMemoryModuleSize(i int) uint64 { return slot(s.modSize, i) } func (s *stubInfo) DeviceMemoryBufferSize(i int) uint64 { return slot(s.bufSize, i) } -func (s *stubInfo) DeviceMemoryOffset(int) uint64 { return 0 } +func (s *stubInfo) DeviceMemoryOffset(i int) uint64 { return slot(s.offset, i) } func (s *stubInfo) DeviceMemoryTotal(i int) uint64 { return slot(s.total, i) } func (s *stubInfo) DeviceSmUtil(i int) uint64 { return slot(s.smUtil, i) } func (s *stubInfo) SetDeviceSmLimit(uint64) {} diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index f93e3c803b..4cc16a16ce 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -463,7 +463,8 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M klog.Errorf("Failed to send device memory desc: %v", err) return err } - memoryOffset := memoryTotal - memoryContextSize - memoryModuleSize - memoryBufferSize + // Send legacy metric with additional memory details + memoryOffset := c.Info.DeviceMemoryOffset(i) // Get the memory offset for the device memoryLabels := append(labels, fmt.Sprint(memoryContextSize), fmt.Sprint(memoryModuleSize), fmt.Sprint(memoryBufferSize), fmt.Sprint(memoryOffset)) sendLegacyMetric(ch, legacyCtrDeviceMemorydesc, prometheus.GaugeValue, float64(memoryTotal), memoryLabels...) diff --git a/cmd/vGPUmonitor/metrics_container_test.go b/cmd/vGPUmonitor/metrics_container_test.go index 46c53a5606..a31ddaf204 100644 --- a/cmd/vGPUmonitor/metrics_container_test.go +++ b/cmd/vGPUmonitor/metrics_container_test.go @@ -154,3 +154,49 @@ func TestCollectContainerMetricsSkipsInvalidUTF8UUID(t *testing.T) { t.Errorf("got %d metrics for an invalid UUID, want 0", len(metrics)) } } + +// TestCheckBlocking_MultiDevice tests CheckBlocking() with multiple devices, some +// of which are contended, some of which are clear, and some of which are missing from the switch +func TestCollectContainerMetricsLegacyOffsetNoUnderflow(t *testing.T) { + // This test ensures that the legacy metric for device memory offset is emitted correctly, even when the total memory is smaller than the sum of context, module, and buffer sizes. It checks that the offset label reflects the actual recorded offset without wrapping or underflowing. + prev := legacyCtrDeviceMemorydesc + legacyCtrDeviceMemorydesc = prometheus.NewDesc( + "Device_memory_desc_of_container", + "Container device memory description", + []string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil, + ) + t.Cleanup(func() { legacyCtrDeviceMemorydesc = prev }) + + cu := &nvidia.ContainerUsage{Info: &stubInfo{ + uuids: []string{testUUID}, + total: []uint64{100}, + ctxSize: []uint64{1000}, + modSize: []uint64{500}, + bufSize: []uint64{300}, + offset: []uint64{7}, + }} + + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "trainer-0"}} + ctr := corev1.Container{Name: "worker"} + cc := ClusterManagerCollector{ClusterManager: &ClusterManager{LegacyMetrics: true}} + ch := make(chan prometheus.Metric, 64) + if err := cc.collectContainerMetrics(ch, pod, ctr, cu, 0); err != nil { + t.Fatal(err) + } + close(ch) + + found := false + for m := range ch { + if m.Desc() != legacyCtrDeviceMemorydesc { + continue + } + found = true + _, labels := gaugeValue(t, m) + if labels["offset"] != "7" { + t.Errorf("offset label = %q, want %q (got a wrapped/underflowed value instead of the recorded offset)", labels["offset"], "7") + } + } + if !found { + t.Fatal("legacy device memory metric was not emitted") + } +} \ No newline at end of file From 001f6afe2ea610c070d6586e4fcefde09c324a2f Mon Sep 17 00:00:00 2001 From: Venkat Date: Tue, 11 Aug 2026 00:24:55 +0530 Subject: [PATCH 2/3] fix Signed-off-by: Venkat --- cmd/vGPUmonitor/metrics_container_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/vGPUmonitor/metrics_container_test.go b/cmd/vGPUmonitor/metrics_container_test.go index a31ddaf204..3ee1e45baa 100644 --- a/cmd/vGPUmonitor/metrics_container_test.go +++ b/cmd/vGPUmonitor/metrics_container_test.go @@ -155,8 +155,8 @@ func TestCollectContainerMetricsSkipsInvalidUTF8UUID(t *testing.T) { } } -// TestCheckBlocking_MultiDevice tests CheckBlocking() with multiple devices, some -// of which are contended, some of which are clear, and some of which are missing from the switch +// TestCollectContainerMetricsLegacyOffsetNoUnderflow verifies that the legacy +// device-memory offset label uses the recorded value without uint64 underflow. func TestCollectContainerMetricsLegacyOffsetNoUnderflow(t *testing.T) { // This test ensures that the legacy metric for device memory offset is emitted correctly, even when the total memory is smaller than the sum of context, module, and buffer sizes. It checks that the offset label reflects the actual recorded offset without wrapping or underflowing. prev := legacyCtrDeviceMemorydesc From 823d949b52b5b94f7a299d838f67565c27b1f929 Mon Sep 17 00:00:00 2001 From: Venkat Date: Tue, 11 Aug 2026 00:28:31 +0530 Subject: [PATCH 3/3] fix-2 Signed-off-by: Venkat --- cmd/vGPUmonitor/metrics_container_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/vGPUmonitor/metrics_container_test.go b/cmd/vGPUmonitor/metrics_container_test.go index 3ee1e45baa..fe6bded71e 100644 --- a/cmd/vGPUmonitor/metrics_container_test.go +++ b/cmd/vGPUmonitor/metrics_container_test.go @@ -158,8 +158,6 @@ func TestCollectContainerMetricsSkipsInvalidUTF8UUID(t *testing.T) { // TestCollectContainerMetricsLegacyOffsetNoUnderflow verifies that the legacy // device-memory offset label uses the recorded value without uint64 underflow. func TestCollectContainerMetricsLegacyOffsetNoUnderflow(t *testing.T) { - // This test ensures that the legacy metric for device memory offset is emitted correctly, even when the total memory is smaller than the sum of context, module, and buffer sizes. It checks that the offset label reflects the actual recorded offset without wrapping or underflowing. - prev := legacyCtrDeviceMemorydesc legacyCtrDeviceMemorydesc = prometheus.NewDesc( "Device_memory_desc_of_container", "Container device memory description",