From 87e9b6023695150862e5e3033b2ee9728e9e04fa Mon Sep 17 00:00:00 2001 From: shellyco-code Date: Fri, 31 Jul 2026 14:16:01 +0530 Subject: [PATCH 1/2] fix: skip faulty devices instead of panicking during registration Signed-off-by: shellyco-code --- .../nvidiadevice/nvinternal/plugin/register.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index 9f32accdaa..663d194927 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -106,12 +106,12 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { ndev, ret := nvml.DeviceGetHandleByUUID(UUID) if ret != nvml.SUCCESS { klog.Errorln("nvml new device by index error uuid=", UUID, "err=", ret) - panic(0) + continue } idx, ret := ndev.GetIndex() if ret != nvml.SUCCESS { klog.Errorln("nvml get index error ret=", ret) - panic(0) + continue } memoryTotal := 0 memory, ret := ndev.GetMemoryInfo() @@ -133,12 +133,12 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { } default: klog.Error("nvml get memory error ret=", ret) - panic(0) + continue } Model, ret := ndev.GetName() if ret != nvml.SUCCESS { klog.Error("nvml get name error ret=", ret) - panic(0) + continue } registeredmem := int32(memoryTotal / 1024 / 1024) From 4efc068c8ad1f91e48ebfcc7f6813d3d228135d7 Mon Sep 17 00:00:00 2001 From: shellyco-code Date: Fri, 31 Jul 2026 15:38:24 +0530 Subject: [PATCH 2/2] fix: address maintainer feedback on skip path Signed-off-by: shellyco-code --- .../nvidiadevice/nvinternal/plugin/register.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index 663d194927..f0497ad902 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -133,6 +133,7 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { } default: klog.Error("nvml get memory error ret=", ret) + // continue skips to the next device in the outer loop, not just the switch continue } Model, ret := ndev.GetName() @@ -185,6 +186,11 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { }) klog.V(3).Infof("Registered device id=%v, memory=%vMB, type=%v, numa=%v, health=%v", idx, registeredmem, Model, numa, health) } + + if len(res) == 0 && len(devs) > 0 { + klog.Warningf("All %d GPU devices failed NVML queries and were skipped. Returning an empty device list.", len(devs)) + } + return &res }