diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go index 9f32accdaa..f0497ad902 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/register.go @@ -106,12 +106,12 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { ndev, ret := nvml.DeviceGetHandleByUUID(UUID) if ret != nvml.SUCCESS { klog.Errorln("nvml new device by index error uuid=", UUID, "err=", ret) - panic(0) + continue } idx, ret := ndev.GetIndex() if ret != nvml.SUCCESS { klog.Errorln("nvml get index error ret=", ret) - panic(0) + continue } memoryTotal := 0 memory, ret := ndev.GetMemoryInfo() @@ -133,12 +133,13 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { } default: klog.Error("nvml get memory error ret=", ret) - panic(0) + // continue skips to the next device in the outer loop, not just the switch + continue } Model, ret := ndev.GetName() if ret != nvml.SUCCESS { klog.Error("nvml get name error ret=", ret) - panic(0) + continue } registeredmem := int32(memoryTotal / 1024 / 1024) @@ -185,6 +186,11 @@ func (plugin *NvidiaDevicePlugin) getAPIDevices() *[]*device.DeviceInfo { }) klog.V(3).Infof("Registered device id=%v, memory=%vMB, type=%v, numa=%v, health=%v", idx, registeredmem, Model, numa, health) } + + if len(res) == 0 && len(devs) > 0 { + klog.Warningf("All %d GPU devices failed NVML queries and were skipped. Returning an empty device list.", len(devs)) + } + return &res }