diff --git a/cmd/vGPUmonitor/metrics.go b/cmd/vGPUmonitor/metrics.go index d13e628850..1877116c75 100644 --- a/cmd/vGPUmonitor/metrics.go +++ b/cmd/vGPUmonitor/metrics.go @@ -499,8 +499,8 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M for i := range c.Info.DeviceNum() { uuid := c.Info.DeviceUUID(i) if len(uuid) < 40 { - klog.Errorf("Invalid UUID length for device %d in Pod %s/%s, Container %s", i, pod.Namespace, pod.Name, ctr.Name) - return fmt.Errorf("invalid UUID length for device %d", i) + klog.Warningf("Device %d in Pod %s/%s, Container %s has invalid UUID length %d (shared memory not yet initialised); skipping until next scrape", i, pod.Namespace, pod.Name, ctr.Name, len(uuid)) + continue } uuid = uuid[0:40] // Ensure UUID is truncated to 40 characters if !utf8.ValidString(uuid) {