Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/migmgr.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ type migInstance struct {

// MigInstanceManager is the single authority over live MIG GI+CI state on a
// node. Keys are the scheduler-reserved profile and physical placement.
//
// Callers must invoke Init once before using other methods, and Shutdown
// once when done; NVML is not re-initialized per call.
type MigInstanceManager struct {
mu sync.Mutex
gpuLocks map[int]*sync.Mutex
Expand All @@ -74,6 +77,24 @@ func NewMigInstanceManager() *MigInstanceManager {
}
}

// Init initializes NVML for this manager. It must be called exactly once,
// before any other MigInstanceManager method, and paired with a single
// later call to Shutdown.
func (m *MigInstanceManager) Init() error {
if nvret := nvml.Init(); nvret != nvml.SUCCESS {
return fmt.Errorf("nvml Init: %s", nvml.ErrorString(nvret))
}
return nil
}

// Shutdown releases the NVML session acquired by Init. Safe to call once,
// when the manager is no longer needed.
func (m *MigInstanceManager) Shutdown() {
if nvret := nvml.Shutdown(); nvret != nvml.SUCCESS {
klog.ErrorS(fmt.Errorf("%s", nvml.ErrorString(nvret)), "nvml Shutdown failed")
}
}

func (m *MigInstanceManager) gpuLock(gpuIndex int) *sync.Mutex {
m.mu.Lock()
defer m.mu.Unlock()
Expand Down
10 changes: 10 additions & 0 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ func (o *options) devicePluginForResource(ctx context.Context, nvconfig *nvidia.
var migMgr *MigInstanceManager
if mode == "mig" {
migMgr = NewMigInstanceManager()
if err := migMgr.Init(); err != nil {
return nil, fmt.Errorf("init MIG instance manager: %w", err)
}
Comment thread
archlitchi marked this conversation as resolved.
Resolved
}
return &NvidiaDevicePlugin{
ctx: ctx,
Expand Down Expand Up @@ -340,6 +343,13 @@ func (plugin *NvidiaDevicePlugin) Start(kubeletSocket string) error {
// reconcile the manager with live Pods so completed or deleted Pods
// release their exact profile+placement allocation.
go plugin.runMigAnnotationReconciler(5 * time.Second)
// The manager's NVML session is owned by the plugin's lifetime, not
// by individual Start/Stop cycles (Stop only restarts the gRPC
// server); release it once when the plugin is finally torn down.
go func() {
<-plugin.ctx.Done()
plugin.migMgr.Shutdown()
}()
}

return nil
Expand Down
Loading