Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/vGPUmonitor/feedback_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type stubInfo struct {
ctxSize []uint64
modSize []uint64
bufSize []uint64
offset []uint64
smUtil []uint64
lastKernel int64
}
Expand All @@ -60,7 +61,7 @@ func (s *stubInfo) DeviceUUID(i int) string {
func (s *stubInfo) DeviceMemoryContextSize(i int) uint64 { return slot(s.ctxSize, i) }
func (s *stubInfo) DeviceMemoryModuleSize(i int) uint64 { return slot(s.modSize, i) }
func (s *stubInfo) DeviceMemoryBufferSize(i int) uint64 { return slot(s.bufSize, i) }
func (s *stubInfo) DeviceMemoryOffset(int) uint64 { return 0 }
func (s *stubInfo) DeviceMemoryOffset(i int) uint64 { return slot(s.offset, i) }
func (s *stubInfo) DeviceMemoryTotal(i int) uint64 { return slot(s.total, i) }
func (s *stubInfo) DeviceSmUtil(i int) uint64 { return slot(s.smUtil, i) }
func (s *stubInfo) SetDeviceSmLimit(uint64) {}
Expand Down
3 changes: 2 additions & 1 deletion cmd/vGPUmonitor/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,8 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M
klog.Errorf("Failed to send device memory desc: %v", err)
return err
}
memoryOffset := memoryTotal - memoryContextSize - memoryModuleSize - memoryBufferSize
// Send legacy metric with additional memory details
memoryOffset := c.Info.DeviceMemoryOffset(i) // Get the memory offset for the device
memoryLabels := append(labels, fmt.Sprint(memoryContextSize), fmt.Sprint(memoryModuleSize), fmt.Sprint(memoryBufferSize), fmt.Sprint(memoryOffset))
sendLegacyMetric(ch, legacyCtrDeviceMemorydesc, prometheus.GaugeValue, float64(memoryTotal), memoryLabels...)

Expand Down
44 changes: 44 additions & 0 deletions cmd/vGPUmonitor/metrics_container_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,3 +154,47 @@ func TestCollectContainerMetricsSkipsInvalidUTF8UUID(t *testing.T) {
t.Errorf("got %d metrics for an invalid UUID, want 0", len(metrics))
}
}

// TestCollectContainerMetricsLegacyOffsetNoUnderflow verifies that the legacy
// device-memory offset label uses the recorded value without uint64 underflow.
func TestCollectContainerMetricsLegacyOffsetNoUnderflow(t *testing.T) {
legacyCtrDeviceMemorydesc = prometheus.NewDesc(
"Device_memory_desc_of_container",
"Container device memory description",
[]string{"podnamespace", "podname", "ctrname", "vdeviceid", "deviceuuid", "context", "module", "data", "offset"}, nil,
)
t.Cleanup(func() { legacyCtrDeviceMemorydesc = prev })

cu := &nvidia.ContainerUsage{Info: &stubInfo{
uuids: []string{testUUID},
total: []uint64{100},
ctxSize: []uint64{1000},
modSize: []uint64{500},
bufSize: []uint64{300},
offset: []uint64{7},
}}

pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Name: "trainer-0"}}
ctr := corev1.Container{Name: "worker"}
cc := ClusterManagerCollector{ClusterManager: &ClusterManager{LegacyMetrics: true}}
ch := make(chan prometheus.Metric, 64)
if err := cc.collectContainerMetrics(ch, pod, ctr, cu, 0); err != nil {
t.Fatal(err)
}
close(ch)

found := false
for m := range ch {
if m.Desc() != legacyCtrDeviceMemorydesc {
continue
}
found = true
_, labels := gaugeValue(t, m)
if labels["offset"] != "7" {
t.Errorf("offset label = %q, want %q (got a wrapped/underflowed value instead of the recorded offset)", labels["offset"], "7")
}
}
if !found {
t.Fatal("legacy device memory metric was not emitted")
}
}
Loading