fix(device-plugin): make MigInstanceManager own its NVML init/shutdown

pkg/device-plugin/nvidiadevice/nvinternal/plugin/migmgr.go

@@ -59,6 +59,9 @@ type migInstance struct {
 
 // MigInstanceManager is the single authority over live MIG GI+CI state on a
 // node. Keys are the scheduler-reserved profile and physical placement.
+//
+// Callers must invoke Init once before using other methods, and Shutdown
+// once when done; NVML is not re-initialized per call.
 type MigInstanceManager struct {
 	mu                  sync.Mutex
 	gpuLocks            map[int]*sync.Mutex
@@ -74,6 +77,24 @@ func NewMigInstanceManager() *MigInstanceManager {
 	}
 }
 
+// Init initializes NVML for this manager. It must be called exactly once,
+// before any other MigInstanceManager method, and paired with a single
+// later call to Shutdown.
+func (m *MigInstanceManager) Init() error {
+	if nvret := nvml.Init(); nvret != nvml.SUCCESS {
+		return fmt.Errorf("nvml Init: %s", nvml.ErrorString(nvret))
+	}
+	return nil
+}
+
+// Shutdown releases the NVML session acquired by Init. Safe to call once,
+// when the manager is no longer needed.
+func (m *MigInstanceManager) Shutdown() {
+	if nvret := nvml.Shutdown(); nvret != nvml.SUCCESS {
+		klog.ErrorS(fmt.Errorf("%s", nvml.ErrorString(nvret)), "nvml Shutdown failed")
+	}
+}
+
 func (m *MigInstanceManager) gpuLock(gpuIndex int) *sync.Mutex {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -95,6 +116,8 @@ func profileSliceKey(profile string) string {
 // ResetIdleGPUs prepares idle MIG-capable GPUs for on-demand instance creation
 // through NVML. Busy GPUs are left untouched; idle GPUs
 // have MIG mode enabled and all existing GI/CI instances destroyed.
+//
+// Requires NVML already initialized via Init.
 func (m *MigInstanceManager) ResetIdleGPUs(deviceCount int, inUse map[int]struct{}) ([]int, error) {
 	reset := []int{}
 	for gpuIndex := 0; gpuIndex < deviceCount; gpuIndex++ {
@@ -273,6 +296,8 @@ func destroyAllMigInstances(dev nvml.Device) error {
 }
 
 // Release destroys the GI+CI bound to the given MIG UUID.
+//
+// Requires NVML already initialized via Init.
 func (m *MigInstanceManager) Release(migUUID string) error {
 	m.mu.Lock()
 	key, ok := m.byAllocationMigUUID[migUUID]
@@ -308,6 +333,8 @@ func allocationKey(gpuIndex int, profile string, placement nvml.GpuInstancePlace
 // placement. It returns whether this call created the instance, allowing the
 // caller to roll back only its own partial allocation. It never retries
 // another placement.
+//
+// Requires NVML already initialized via Init.
 func (m *MigInstanceManager) EnsureAllocation(gpuIndex int, profile string, placement nvml.GpuInstancePlacement) (string, bool, error) {
 	key := allocationKey(gpuIndex, profile, placement)
 	lk := m.gpuLock(gpuIndex)
@@ -413,6 +440,7 @@ func (m *MigInstanceManager) AllocationRuntimeInfo(gpuIndex int, profile string,
 	}, true
 }
 
+// Requires NVML already initialized via Init.
 func (m *MigInstanceManager) AdoptAllocation(gpuIndex int, profile, migUUID string, placement nvml.GpuInstancePlacement, gpuInstanceID, computeInstanceID uint32) error {
 	lk := m.gpuLock(gpuIndex)
 	lk.Lock()
@@ -469,13 +497,17 @@ func (m *MigInstanceManager) AdoptAllocation(gpuIndex int, profile, migUUID stri
 	return fmt.Errorf("annotated MIG allocation %s profile=%s placement=%+v is not live", migUUID, profile, placement)
 }
 
+// Requires NVML already initialized via Init.
 func (m *MigInstanceManager) ReconcileActiveAllocations(active map[migAllocationKey]struct{}) error {
 	m.mu.Lock()
 	keys := make([]migAllocationKey, 0, len(m.byAllocation))
 	for key := range m.byAllocation {
 		keys = append(keys, key)
 	}
 	m.mu.Unlock()
+	if len(keys) == 0 {
+		return nil
+	}
 	for _, key := range keys {
 		if _, ok := active[key]; ok {
 			continue

pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go

@@ -193,6 +193,9 @@ func (o *options) devicePluginForResource(ctx context.Context, nvconfig *nvidia.
 	var migMgr *MigInstanceManager
 	if mode == "mig" {
 		migMgr = NewMigInstanceManager()
+		if err := migMgr.Init(); err != nil {
+			return nil, fmt.Errorf("init MIG instance manager: %w", err)
+		}
 	}
 	return &NvidiaDevicePlugin{
 		ctx:                        ctx,
@@ -340,6 +343,13 @@ func (plugin *NvidiaDevicePlugin) Start(kubeletSocket string) error {
 		// reconcile the manager with live Pods so completed or deleted Pods
 		// release their exact profile+placement allocation.
 		go plugin.runMigAnnotationReconciler(5 * time.Second)
+		// The manager's NVML session is owned by the plugin's lifetime, not
+		// by individual Start/Stop cycles (Stop only restarts the gRPC
+		// server); release it once when the plugin is finally torn down.
+		go func() {
+			<-plugin.ctx.Done()
+			plugin.migMgr.Shutdown()
+		}()
 	}
 
 	return nil