From 07210fff2af679e68bcf8e68a71cbe1cf1d9415f Mon Sep 17 00:00:00 2001 From: Nakshatra Sharma Date: Sat, 8 Aug 2026 12:17:19 +0530 Subject: [PATCH] fix(monitor): use IsValidUUID to skip uninitialised device UUIDs in scrape path DeviceUUID always returns a 96-byte string (from [96]byte), so the prior guard len(uuid) < 40 could never fire. Containers starting up whose UUID has not yet been written by libvgpu passed through with 40 null bytes emitted as a Prometheus label value. Replace the dead length check with IsValidUUID, which tests uuid[0] != 0 and correctly detects uninitialized slots. Update stubInfo.IsValidUUID in the test helper to match the same semantics, and replace the short-UUID test case with an uninitialized-UUID case that reflects the real scenario. Signed-off-by: Nakshatra Sharma --- cmd/vGPUmonitor/feedback_test.go | 20 +++++++++++--------- cmd/vGPUmonitor/metrics.go | 9 ++++----- cmd/vGPUmonitor/metrics_container_test.go | 8 ++++---- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cmd/vGPUmonitor/feedback_test.go b/cmd/vGPUmonitor/feedback_test.go index 10c2177790..b55590fe8a 100644 --- a/cmd/vGPUmonitor/feedback_test.go +++ b/cmd/vGPUmonitor/feedback_test.go @@ -64,15 +64,17 @@ func (s *stubInfo) DeviceMemoryOffset(int) uint64 { return 0 } func (s *stubInfo) DeviceMemoryTotal(i int) uint64 { return slot(s.total, i) } func (s *stubInfo) DeviceSmUtil(i int) uint64 { return slot(s.smUtil, i) } func (s *stubInfo) SetDeviceSmLimit(uint64) {} -func (s *stubInfo) IsValidUUID(i int) bool { return i < len(s.uuids) } -func (s *stubInfo) DeviceMemoryLimit(i int) uint64 { return slot(s.limit, i) } -func (s *stubInfo) SetDeviceMemoryLimit(uint64) {} -func (s *stubInfo) LastKernelTime() int64 { return s.lastKernel } -func (s *stubInfo) GetPriority() int { return s.priority } -func (s *stubInfo) GetRecentKernel() int32 { return 1 } -func (s *stubInfo) SetRecentKernel(int32) {} -func (s *stubInfo) GetUtilizationSwitch() int32 { return 0 } -func (s *stubInfo) SetUtilizationSwitch(int32) {} +func (s *stubInfo) IsValidUUID(i int) bool { + return i >= 0 && i < len(s.uuids) && len(s.uuids[i]) > 0 && s.uuids[i][0] != 0 +} +func (s *stubInfo) DeviceMemoryLimit(i int) uint64 { return slot(s.limit, i) } +func (s *stubInfo) SetDeviceMemoryLimit(uint64) {} +func (s *stubInfo) LastKernelTime() int64 { return s.lastKernel } +func (s *stubInfo) GetPriority() int { return s.priority } +func (s *stubInfo) GetRecentKernel() int32 { return 1 } +func (s *stubInfo) SetRecentKernel(int32) {} +func (s *stubInfo) GetUtilizationSwitch() int32 { return 0 } +func (s *stubInfo) SetUtilizationSwitch(int32) {} func TestCheckFunctionsHighPriority(t *testing.T) { sw := map[string]UtilizationPerDevice{"gpu-0": {0, 1}} diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index f943a6349e..d248c6f110 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -428,14 +428,13 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M // Iterate through each device for i := range c.Info.DeviceNum() { - uuid := c.Info.DeviceUUID(i) - if len(uuid) < 40 { - klog.Warningf("Device %d in Pod %s/%s, Container %s has invalid UUID length %d (shared memory not yet initialised); skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name, len(uuid)) + if !c.Info.IsValidUUID(i) { + klog.Warningf("Device %d in Pod %s/%s, Container %s UUID not yet initialised; skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name) continue } - uuid = uuid[0:40] // Ensure UUID is truncated to 40 characters + uuid := c.Info.DeviceUUID(i)[0:40] if !utf8.ValidString(uuid) { - klog.Warningf("Device %d in Pod %s/%s, Container %s has invalid UTF-8 UUID (shared memory not yet initialised); skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name) + klog.Warningf("Device %d in Pod %s/%s, Container %s has invalid UTF-8 UUID; skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name) continue } diff --git a/cmd/vGPUmonitor/metrics_container_test.go b/cmd/vGPUmonitor/metrics_container_test.go index 46c53a5606..87b647a165 100644 --- a/cmd/vGPUmonitor/metrics_container_test.go +++ b/cmd/vGPUmonitor/metrics_container_test.go @@ -133,13 +133,13 @@ func TestCollectContainerMetricsBadInput(t *testing.T) { if _, err := collectContainer(t, &nvidia.ContainerUsage{}, 0); err == nil { t.Error("nil ContainerUsage.Info should return an error") } - short := &nvidia.ContainerUsage{Info: &stubInfo{uuids: []string{"gpu-short"}}} - metrics, err := collectContainer(t, short, 0) + uninit := &nvidia.ContainerUsage{Info: &stubInfo{uuids: []string{"\x00" + strings.Repeat("\x00", 39)}}} + metrics, err := collectContainer(t, uninit, 0) if err != nil { - t.Fatalf("a UUID shorter than 40 chars should be skipped, not error: %v", err) + t.Fatalf("uninitialized UUID should be skipped, not error: %v", err) } if len(metrics) != 0 { - t.Errorf("got %d metrics for a short UUID, want 0", len(metrics)) + t.Errorf("got %d metrics for an uninitialized UUID, want 0", len(metrics)) } }