fix(device,plugin): stop MIG usage corruption and fd leak

pkg/device-plugin/nvidiadevice/nvinternal/plugin/lock.go

@@ -61,12 +61,12 @@ func createMigApplyLock(file string) error {
 		klog.Infof("MIG apply lock file already exists: %s", MigApplyLockFile)
 		return nil
 	}
-	_, err := os.Create(file)
+	f, err := os.Create(file)
 	if err != nil {
 		klog.Errorf("Failed to create MIG apply lock file: %v", err)
 		return err
 	}
-	return nil
+	return f.Close()
 }
 
 // RemoveMigApplyLock removes the lock file for MIG apply operation

pkg/device/nvidia/device.go

@@ -678,9 +678,9 @@ func (dev *NvidiaGPUDevices) migNeedsReset(n *device.DeviceUsage) bool {
 }
 
 func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceUsage, ctr *device.ContainerDevice) error {
-	n.Used++
 	if n.Mode == MigMode {
 		if dev.migNeedsReset(n) {
+			found := false
 		OuterLoop:
 			for tidx, templates := range n.MigTemplate {
 				for idx, template := range templates {
@@ -700,10 +700,14 @@ func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceU
 						}
 						n.MigUsage.Index = int32(tidx)
 						n.MigUsage.UsageList[usageListIdx].InUse = true
+						found = true
 						break OuterLoop
 					}
 				}
 			}
+			if !found {
+				return errors.New("mig template allocate resource fail")
+			}
 		} else {
 			found := false
 			for idx, val := range n.MigUsage.UsageList {
@@ -723,6 +727,7 @@ func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceU
 			}
 		}
 	}
+	n.Used++
 	n.Usedcores += ctr.Usedcores
 	n.Usedmem += ctr.Usedmem
 	return nil
@@ -839,7 +844,10 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
 			klog.V(5).InfoS(common.CardComputeUnitsExhausted, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
 			continue
 		}
-		if !nv.CustomFilterRule(allocated, request, tmpDevs[k.Type], dev) {
+		// CustomFilterRule must see the resolved memory request, not the raw (possibly zero) Memreq field.
+		resolvedReq := request
+		resolvedReq.Memreq = memreq
+		if !nv.CustomFilterRule(allocated, resolvedReq, tmpDevs[k.Type], dev) {
 			reason[common.CardNotFoundCustomFilterRule]++
 			klog.V(5).InfoS(common.CardNotFoundCustomFilterRule, "pod", klog.KObj(pod), "device", dev.ID, "device index", i)
 			continue

pkg/device/nvidia/device_test.go

@@ -2691,6 +2691,27 @@ func TestAddResourceUsage_MigNonResetNoSlot(t *testing.T) {
 	err := dev.AddResourceUsage(&corev1.Pod{}, usage, ctr)
 	assert.Assert(t, err != nil)
 	assert.Assert(t, strings.Contains(err.Error(), "mig template allocate resource fail"))
+	assert.Equal(t, usage.Used, int32(0))
+}
+
+func TestAddResourceUsage_MigResetNoFit(t *testing.T) {
+	dev := InitNvidiaDevice(NvidiaConfig{})
+	usage := &device.DeviceUsage{
+		Mode: MigMode,
+		MigTemplate: []device.Geometry{
+			{
+				{Name: "1g.5gb", Memory: 1024, Core: 14, Count: 1},
+			},
+		},
+	}
+	ctr := &device.ContainerDevice{UUID: "GPU-0", Usedmem: 4096}
+	err := dev.AddResourceUsage(&corev1.Pod{}, usage, ctr)
+	assert.Assert(t, err != nil)
+	assert.Assert(t, strings.Contains(err.Error(), "mig template allocate resource fail"))
+	// No template fit: usage counters must not reflect a phantom allocation.
+	assert.Equal(t, usage.Usedmem, int32(0))
+	assert.Equal(t, usage.Used, int32(0))
+	assert.Assert(t, !strings.Contains(ctr.UUID, "["))
 }
 
 func TestCustomFilterRule_MigEmptyUsageWithTemplate(t *testing.T) {
@@ -2857,6 +2878,33 @@ func TestFit_MutexPolicy(t *testing.T) {
 	assert.Equal(t, fit, false)
 }
 
+func TestFit_MigPercentageRequestRejectsUndersizedTemplate(t *testing.T) {
+	config := NvidiaConfig{
+		ResourceCountName:            "nvidia.com/gpu",
+		ResourceMemoryName:           "nvidia.com/gpumem",
+		ResourceCoreName:             "nvidia.com/gpucores",
+		ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+	}
+	nv := InitNvidiaDevice(config)
+
+	// The only MIG template offers 1024MiB slots, but the pod requests 4096MiB (50% of 8192MiB) via MemPercentagereq.
+	devices := []*device.DeviceUsage{
+		{
+			ID: "dev-0", Index: 0, Used: 0, Count: 1,
+			Totalmem: 8192, Totalcore: 100, Type: NvidiaGPUDevice, Health: true,
+			Mode: MigMode,
+			MigTemplate: []device.Geometry{
+				{
+					{Name: "1g.5gb", Memory: 1024, Core: 14, Count: 1},
+				},
+			},
+		},
+	}
+	req := device.ContainerDeviceRequest{Nums: 1, MemPercentagereq: 50, Coresreq: 10, Type: NvidiaGPUDevice}
+	fit, _, _ := nv.Fit(devices, req, &corev1.Pod{}, &device.NodeInfo{}, &device.PodDevices{})
+	assert.Equal(t, fit, false)
+}
+
 func TestFit_TopologyExactMatch(t *testing.T) {
 	config := NvidiaConfig{
 		ResourceCountName:            "nvidia.com/gpu",

pkg/scheduler/webhook_test.go

@@ -486,13 +486,13 @@ func TestFitResourceQuotaNonNvidia(t *testing.T) {
 	// One MLU vmemory unit is 256 MiB, so a limit of 100 units leaves room for
 	// 25600 MiB. Comparing the request against the raw 100 would deny every pod.
 	qm.Quotas["mlu-mem"] = &device.DeviceQuota{
-		"cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 0, Limit: 100},
+		"cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 0, Limit: 100, LimitSet: true},
 	}
 	qm.Quotas["mlu-core"] = &device.DeviceQuota{
-		"cambricon.com/mlu.smlu.vcore": &device.Quota{Used: 20, Limit: 50},
+		"cambricon.com/mlu.smlu.vcore": &device.Quota{Used: 20, Limit: 50, LimitSet: true},
 	}
 	qm.Quotas["dcu-mem"] = &device.DeviceQuota{
-		"hygon.com/dcumem": &device.Quota{Used: 0, Limit: 1000},
+		"hygon.com/dcumem": &device.Quota{Used: 0, Limit: 1000, LimitSet: true},
 	}
 	t.Cleanup(func() {
 		for _, ns := range []string{"mlu-mem", "mlu-core", "dcu-mem"} {
@@ -590,7 +590,7 @@ func TestFitResourceQuotaCountsEveryDevice(t *testing.T) {
 	qm := device.NewQuotaManager()
 	// 60 units is 15360 MiB of headroom.
 	qm.Quotas["mlu-multi"] = &device.DeviceQuota{
-		"cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 0, Limit: 60},
+		"cambricon.com/mlu.smlu.vmemory": &device.Quota{Used: 0, Limit: 60, LimitSet: true},
 	}
 	t.Cleanup(func() { delete(qm.Quotas, "mlu-multi") })
 
@@ -649,7 +649,7 @@ func TestFitResourceQuotaAscendMemoryFactor(t *testing.T) {
 
 	qm := device.NewQuotaManager()
 	qm.Quotas["ascend"] = &device.DeviceQuota{
-		"huawei.com/Ascend910B-memory": &device.Quota{Used: 0, Limit: 8192},
+		"huawei.com/Ascend910B-memory": &device.Quota{Used: 0, Limit: 8192, LimitSet: true},
 	}
 	t.Cleanup(func() { delete(qm.Quotas, "ascend") })