Support live per-GPU cordon via node annotation

pkg/device/common/common.go

@@ -30,6 +30,7 @@ const (
 	CardInsufficientMemory            = "CardInsufficientMemory"
 	CardInsufficientCore              = "CardInsufficientCore"
 	CardNotHealth                     = "CardNotHealth"
+	CardCordoned                      = "CardCordoned"
 	NumaNotFit                        = "NumaNotFit"
 	ExclusiveDeviceAllocateConflict   = "ExclusiveDeviceAllocateConflict"
 	CardNotFoundCustomFilterRule      = "CardNotFoundCustomFilterRule"

pkg/device/nvidia/device.go

@@ -51,6 +51,12 @@ const (
 	// GPUNoUseUUID annotation specifies a comma-separated list of GPU UUIDs to exclude.
 	GPUNoUseUUID = "nvidia.com/nouse-gpuuuid"
 	AllocateMode = "nvidia.com/vgpu-mode"
+	// DeviceCordonAnnotation is a node annotation holding a comma-separated list of
+	// GPU UUIDs to exclude from new allocations, without affecting pods already
+	// running on those devices. It's a live, per-GPU equivalent of `kubectl cordon`;
+	// unlike FilterDeviceToRegister, it takes effect immediately and needs no
+	// device-plugin restart.
+	DeviceCordonAnnotation = "hami.io/device-cordon"
 
 	MigMode      = "mig"
 	HamiCoreMode = "hami-core"
@@ -684,6 +690,28 @@ func fitQuota(pod *corev1.Pod, tmpDevs map[string]device.ContainerDevices, alloc
 	klog.V(4).Infoln("Allocating...", mem, "cores", core)
 	return device.GetLocalCache().FitQuota(ns, mem, MemoryFactor, core, NvidiaGPUDevice)
 }
+
+// cordonedDevices parses the DeviceCordonAnnotation off the node into a UUID
+// lookup set. Missing node, missing annotation, or an empty value all mean
+// "nothing cordoned". Malformed entries (stray whitespace/commas) are
+// tolerated by trimming and skipping empties, same as CheckUUID's parsing.
+func cordonedDevices(nodeInfo *device.NodeInfo) map[string]struct{} {
+	cordoned := make(map[string]struct{})
+	if nodeInfo == nil || nodeInfo.Node == nil {
+		return cordoned
+	}
+	raw, ok := nodeInfo.Node.Annotations[DeviceCordonAnnotation]
+	if !ok || strings.TrimSpace(raw) == "" {
+		return cordoned
+	}
+	for uuid := range strings.SplitSeq(raw, ",") {
+		if uuid = strings.TrimSpace(uuid); uuid != "" {
+			cordoned[uuid] = struct{}{}
+		}
+	}
+	return cordoned
+}
+
 func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) {
 	k := request
 	originReq := k.Nums
@@ -695,6 +723,7 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
 	gpuPolicy := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod)
 	needTopology := util.PolicyContains(gpuPolicy, util.GPUSchedulerPolicyTopology)
 	isMutex := util.PolicyContains(gpuPolicy, util.GPUSchedulerPolicyMutex)
+	cordoned := cordonedDevices(nodeInfo)
 	for i := len(devices) - 1; i >= 0; i-- {
 		dev := devices[i]
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
@@ -703,6 +732,11 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
 			klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health)
 			continue
 		}
+		if _, isCordoned := cordoned[dev.ID]; isCordoned {
+			reason[common.CardCordoned]++
+			klog.V(5).InfoS(common.CardCordoned, "pod", klog.KObj(pod), "device", dev.ID)
+			continue
+		}
 		found, numa := nv.checkType(pod.GetAnnotations(), *dev, k)
 		if !found {
 			reason[common.CardTypeMismatch]++

pkg/device/nvidia/device_test.go

@@ -1223,6 +1223,80 @@ func TestDevices_Fit(t *testing.T) {
 	}
 }
 
+func TestFit_DeviceCordon(t *testing.T) {
+	config := NvidiaConfig{
+		ResourceCountName:            "nvidia.com/gpu",
+		ResourceMemoryName:           "nvidia.com/gpumem",
+		ResourceCoreName:             "nvidia.com/gpumem",
+		ResourceMemoryPercentageName: "nvidia.com/gpumem-percentage",
+	}
+	dev := InitNvidiaDevice(config)
+
+	newDevices := func() []*device.DeviceUsage {
+		return []*device.DeviceUsage{
+			{ID: "dev-0", Count: 100, Totalmem: 128, Totalcore: 100, Type: NvidiaGPUDevice, Health: true},
+			{ID: "dev-1", Count: 100, Totalmem: 128, Totalcore: 100, Type: NvidiaGPUDevice, Health: true},
+		}
+	}
+	request := device.ContainerDeviceRequest{Nums: 1, Memreq: 64, Coresreq: 50, Type: NvidiaGPUDevice}
+	pod := &corev1.Pod{}
+
+	nodeWithCordon := func(uuids string) *device.NodeInfo {
+		return &device.NodeInfo{Node: &corev1.Node{
+			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{DeviceCordonAnnotation: uuids}},
+		}}
+	}
+
+	t.Run("cordoned device is skipped, healthy sibling still fits", func(t *testing.T) {
+		fit, result, reason := dev.Fit(newDevices(), request, pod, nodeWithCordon("dev-1, "), &device.PodDevices{})
+		if !fit {
+			t.Fatalf("expected fit, got reason: %s", reason)
+		}
+		if got := result[NvidiaGPUDevice][0].UUID; got != "dev-0" {
+			t.Errorf("expected dev-0 (dev-1 is cordoned), got %s", got)
+		}
+	})
+
+	t.Run("all devices cordoned fails with CardCordoned reason", func(t *testing.T) {
+		fit, _, reason := dev.Fit(newDevices(), request, pod, nodeWithCordon("dev-0,dev-1"), &device.PodDevices{})
+		if fit {
+			t.Fatal("expected no fit, all devices are cordoned")
+		}
+		if reason != "2/2 CardCordoned" {
+			t.Errorf("expected reason %q, got %q", "2/2 CardCordoned", reason)
+		}
+	})
+
+	t.Run("running pods on a cordoned device are unaffected", func(t *testing.T) {
+		devices := newDevices()
+		devices[1].Used = 1
+		devices[1].Usedcores = 50
+		devices[1].Usedmem = 64
+		fit, result, _ := dev.Fit(devices, request, pod, nodeWithCordon("dev-1"), &device.PodDevices{})
+		if !fit {
+			t.Fatal("expected fit onto the non-cordoned device")
+		}
+		if got := result[NvidiaGPUDevice][0].UUID; got != "dev-0" {
+			t.Errorf("expected dev-0, got %s", got)
+		}
+		if devices[1].Used != 1 {
+			t.Errorf("cordon must not touch existing usage on dev-1, got Used=%d", devices[1].Used)
+		}
+	})
+
+	t.Run("no annotation or no node info means nothing cordoned", func(t *testing.T) {
+		for name, ni := range map[string]*device.NodeInfo{
+			"node present, annotation absent": {Node: &corev1.Node{}},
+			"NodeInfo.Node is nil":            {},
+		} {
+			fit, _, reason := dev.Fit(newDevices(), request, pod, ni, &device.PodDevices{})
+			if !fit {
+				t.Errorf("%s: expected fit, got reason: %s", name, reason)
+			}
+		}
+	})
+}
+
 func TestDevices_AddResourceUsage(t *testing.T) {
 	dev := &NvidiaGPUDevices{}
 	usage := &device.DeviceUsage{ID: "dev-0", Usedcores: 15, Usedmem: 2000}