Fix device plugin recovery from Unhealthy state

pkg/device-plugin/nvidiadevice/nvinternal/rm/health.go

@@ -40,6 +40,7 @@ import (
 
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	"k8s.io/klog/v2"
+	kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
 const (
@@ -56,8 +57,8 @@ const (
 	envEnableHealthChecks = "DP_ENABLE_HEALTHCHECKS"
 )
 
-// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices
-func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devices, unhealthy chan<- *Device, disableNVML <-chan bool) error {
+// CheckHealth performs health checks on a set of devices, writing to the 'health' channel with any unhealthy devices
+func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devices, health chan<- *Device, disableNVML <-chan bool) error {
 	xids := getHealthCheckXids()
 	if xids.IsAllDisabled() {
 		return nil
@@ -123,7 +124,7 @@ func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devic
 		}
 		if ret != nvml.SUCCESS {
 			klog.Infof("Marking device %v as unhealthy: %v", d.ID, ret)
-			unhealthy <- d
+			health <- d
 		}
 	}
 
@@ -146,7 +147,7 @@ func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devic
 		if ret != nvml.SUCCESS {
 			klog.Infof("Error waiting for event: %v; Marking all devices as unhealthy", ret)
 			for _, d := range devices {
-				unhealthy <- d
+				health <- d
 			}
 			continue
 		}
@@ -167,7 +168,7 @@ func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devic
 			// If we cannot reliably determine the device UUID, we mark all devices as unhealthy.
 			klog.Infof("Failed to determine uuid for event %v: %v; Marking all devices as unhealthy.", e, ret)
 			for _, d := range devices {
-				unhealthy <- d
+				health <- d
 			}
 			continue
 		}
@@ -188,7 +189,7 @@ func (r *nvmlResourceManager) checkHealth(stop <-chan interface{}, devices Devic
 		}
 
 		klog.Infof("XidCriticalError: Xid=%d on Device=%s; marking device as unhealthy.", e.EventData, d.ID)
-		unhealthy <- d
+		health <- d
 	}
 }
 

pkg/device-plugin/nvidiadevice/nvinternal/rm/nvml_manager.go

@@ -113,12 +113,12 @@ func (r *nvmlResourceManager) GetDevicePaths(ids []string) []string {
 	return append(paths, r.Devices().Subset(ids).GetPaths()...)
 }
 
-// CheckHealth performs health checks on a set of devices, writing to the 'unhealthy' channel with any unhealthy devices
-func (r *nvmlResourceManager) CheckHealth(stop <-chan interface{}, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
+// CheckHealth performs health checks on a set of devices, writing to the 'health' channel with device health updates or transitions.
+func (r *nvmlResourceManager) CheckHealth(stop <-chan interface{}, health chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
 	for {
 		// first check if disableNVML channel signal is pass close into checkHealth function
 		// if signal is pass close, return error "close signal received"
-		err := r.checkHealth(stop, r.devices, unhealthy, disableNVML)
+		err := r.checkHealth(stop, r.devices, health, disableNVML)
 		// checkHealth returns nil when health checks are disabled or on shutdown.
 		if err != nil && err.Error() == "close signal received" {
 			ackDisableHealthChecks <- true

pkg/device-plugin/nvidiadevice/nvinternal/rm/rm.go

@@ -58,7 +58,7 @@ type ResourceManager interface {
 	Devices() Devices
 	GetDevicePaths([]string) []string
 	GetPreferredAllocation(available, required []string, size int) ([]string, error)
-	CheckHealth(stop <-chan interface{}, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error
+	CheckHealth(stop <-chan interface{}, health chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error
 	ValidateRequest(AnnotatedIDs) error
 }
 

pkg/device-plugin/nvidiadevice/nvinternal/rm/rm_mock.go

@@ -48,7 +48,7 @@ var _ ResourceManager = &ResourceManagerMock{}
 //
 //		// make and configure a mocked ResourceManager
 //		mockedResourceManager := &ResourceManagerMock{
-//			CheckHealthFunc: func(stop <-chan interface{}, unhealthy chan<- *Device) error {
+//			CheckHealthFunc: func(stop <-chan interface{}, health chan<- *Device) error {
 //				panic("mock out the CheckHealth method")
 //			},
 //			DevicesFunc: func() Devices {
@@ -74,7 +74,7 @@ var _ ResourceManager = &ResourceManagerMock{}
 //	}
 type ResourceManagerMock struct {
 	// CheckHealthFunc mocks the CheckHealth method.
-	CheckHealthFunc func(stop <-chan interface{}, unhealthy chan<- *Device) error
+	CheckHealthFunc func(stop <-chan interface{}, health chan<- *Device) error
 
 	// DevicesFunc mocks the Devices method.
 	DevicesFunc func() Devices
@@ -97,8 +97,8 @@ type ResourceManagerMock struct {
 		CheckHealth []struct {
 			// Stop is the stop argument value.
 			Stop <-chan interface{}
-			// Unhealthy is the unhealthy argument value.
-			Unhealthy chan<- *Device
+			// Health is the health argument value.
+			Health chan<- *Device
 		}
 		// Devices holds details about calls to the Devices method.
 		Devices []struct {
@@ -135,13 +135,13 @@ type ResourceManagerMock struct {
 }
 
 // CheckHealth calls CheckHealthFunc.
-func (mock *ResourceManagerMock) CheckHealth(stop <-chan interface{}, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
+func (mock *ResourceManagerMock) CheckHealth(stop <-chan interface{}, health chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
 	callInfo := struct {
-		Stop      <-chan interface{}
-		Unhealthy chan<- *Device
+		Stop   <-chan interface{}
+		Health chan<- *Device
 	}{
-		Stop:      stop,
-		Unhealthy: unhealthy,
+		Stop:   stop,
+		Health: health,
 	}
 	mock.lockCheckHealth.Lock()
 	mock.calls.CheckHealth = append(mock.calls.CheckHealth, callInfo)
@@ -152,16 +152,16 @@ func (mock *ResourceManagerMock) CheckHealth(stop <-chan interface{}, unhealthy
 		)
 		return errOut
 	}
-	return mock.CheckHealthFunc(stop, unhealthy)
+	return mock.CheckHealthFunc(stop, health)
 }
 
 // CheckHealthCalls gets all the calls that were made to CheckHealth.
 // Check the length with:
 //
 //	len(mockedResourceManager.CheckHealthCalls())
 func (mock *ResourceManagerMock) CheckHealthCalls() []struct {
-	Stop      <-chan interface{}
-	Unhealthy chan<- *Device
+	Stop   <-chan interface{}
+	Health chan<- *Device
 } {
 	var calls []struct {
 		Stop      <-chan interface{}

pkg/device-plugin/nvidiadevice/nvinternal/rm/tegra_manager.go

@@ -87,6 +87,6 @@ func (r *tegraResourceManager) GetDevicePaths(ids []string) []string {
 }
 
 // CheckHealth is disabled for the tegraResourceManager
-func (r *tegraResourceManager) CheckHealth(stop <-chan interface{}, unhealthy chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
+func (r *tegraResourceManager) CheckHealth(stop <-chan interface{}, health chan<- *Device, disableNVML <-chan bool, ackDisableHealthChecks chan<- bool) error {
 	return nil
 }