From 4ca2a4eef30d70a879ec731fdf5bbaf010829015 Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Sun, 2 Aug 2026 18:52:42 +0530 Subject: [PATCH 1/7] fix: skip unhealthy devices in Fit() for all non-nvidia backends The nvidia backend already checks dev.Health before allocating a device. The same guard was missing from all other backends: amd, awsneuron, biren, cambricon, enflame (device.go + gcu.go), hygon, iluvatar, kunlun (device.go + vdevice.go), metax, mthreads, and vastai. For loop-based backends, add the dev.Health check at the top of the device loop before any other checks, mirroring the nvidia pattern. For graph-select backends (kunlun), add the check inside the fitness functions (FitXPU, FitVXPU) so unhealthy devices are excluded from graph selection. For awsneuron, add the check in both the single-device loop path and inside continuousDeviceAvailable() for multi-device topology-aware selection. Add a CardNotHealth test case for each touched backend that already has a Fit test table (amd, awsneuron, biren, cambricon, enflame/gcu, hygon, iluvatar, kunlun/vdevice, metax, mthreads, vastai). Ascend is handled separately in #2241. Follows up on #2241 which fixed the same gap for ascend. Signed-off-by: Eshiv Pandey --- pkg/device/amd/device.go | 6 ++++- pkg/device/amd/device_test.go | 16 ++++++++++++ pkg/device/awsneuron/device.go | 8 ++++-- pkg/device/awsneuron/device_test.go | 31 +++++++++++++++++++++++ pkg/device/biren/device.go | 6 ++++- pkg/device/biren/device_test.go | 38 +++++++++++++++++++++++++---- pkg/device/cambricon/device.go | 6 ++++- pkg/device/cambricon/device_test.go | 38 +++++++++++++++++++++++++---- pkg/device/enflame/device.go | 6 ++++- pkg/device/enflame/device_test.go | 3 +++ pkg/device/enflame/gcu.go | 6 ++++- pkg/device/enflame/gcu_test.go | 36 ++++++++++++++++++++++++--- pkg/device/hygon/device.go | 6 ++++- pkg/device/hygon/device_test.go | 38 +++++++++++++++++++++++++---- pkg/device/iluvatar/device.go | 6 ++++- pkg/device/iluvatar/device_test.go | 27 ++++++++++++++++++++ pkg/device/kunlun/device.go | 2 +- pkg/device/kunlun/vdevice.go | 3 +++ pkg/device/kunlun/vdevice_test.go | 6 +++++ pkg/device/metax/device.go | 6 ++++- pkg/device/metax/device_test.go | 28 +++++++++++++++++++++ pkg/device/mthreads/device.go | 6 ++++- pkg/device/mthreads/device_test.go | 28 +++++++++++++++++++++ pkg/device/vastai/device.go | 6 ++++- pkg/device/vastai/device_test.go | 28 +++++++++++++++++++++ 25 files changed, 358 insertions(+), 32 deletions(-) diff --git a/pkg/device/amd/device.go b/pkg/device/amd/device.go index 98b748dc87..d316bbc534 100644 --- a/pkg/device/amd/device.go +++ b/pkg/device/amd/device.go @@ -284,7 +284,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type, "dev=", dev) _, found, _ := amddevice.checkType(pod.GetAnnotations(), *dev, k) if !found { diff --git a/pkg/device/amd/device_test.go b/pkg/device/amd/device_test.go index 8d7e943d29..ad4bacdcc9 100644 --- a/pkg/device/amd/device_test.go +++ b/pkg/device/amd/device_test.go @@ -458,4 +458,20 @@ func TestDevices_Fit(t *testing.T) { assert.Equal(t, false, ok) assert.Assert(t, strings.Contains(reason, common.CardTimeSlicingExhausted)) }) + + t.Run("unhealthy device is rejected", func(t *testing.T) { + devices := []*device.DeviceUsage{ + { + ID: "dev-0", Index: 0, Used: 0, Count: 2, + Usedmem: 0, Totalmem: 1000, Totalcore: 100, Usedcores: 0, + Type: AMDDevice, Health: false, CustomInfo: map[string]any{}, + }, + } + req := device.ContainerDeviceRequest{Nums: 1, Type: AMDDevice, Memreq: 100, MemPercentagereq: 0, Coresreq: 10} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} + + ok, _, reason := dev.Fit(devices, req, pod, &device.NodeInfo{}, &device.PodDevices{}) + assert.Equal(t, false, ok) + assert.Assert(t, strings.Contains(reason, common.CardNotHealth)) + }) } diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 690a865012..ce31199704 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -313,7 +313,7 @@ func continuousDeviceAvailable(devices []*device.DeviceUsage, start int, count i res := []int{} iterator := start for iterator < start+count { - if devices[iterator].Used > 0 { + if devices[iterator].Used > 0 || !devices[iterator].Health { return []int{} } res = append(res, iterator) @@ -398,7 +398,11 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic dev.CustomInfo[AWSUsageInfo] = int(dev.Usedcores) } klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type, "dev=", dev) if !strings.Contains(dev.Type, k.Type) { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index 2f63018b79..b7d9237033 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -860,6 +860,37 @@ func TestDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "1/1 ExclusiveDeviceAllocateConflict", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 2, + Usedmem: 0, + Totalmem: 0, + Totalcore: 3, + Usedcores: 0, + Numa: 0, + Type: AWSNeuronDevice, + Health: false, + CustomInfo: map[string]any{ + AWSNodeType: "trn", + }, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 2, + Type: AWSNeuronDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { diff --git a/pkg/device/biren/device.go b/pkg/device/biren/device.go index 10b1655cd4..93b7dbb980 100644 --- a/pkg/device/biren/device.go +++ b/pkg/device/biren/device.go @@ -196,7 +196,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := br.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/biren/device_test.go b/pkg/device/biren/device_test.go index 31d8321836..4712cb0181 100644 --- a/pkg/device/biren/device_test.go +++ b/pkg/device/biren/device_test.go @@ -734,7 +734,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -807,10 +807,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 0, + Totalcore: 0, + Usedcores: 0, + Numa: 0, + Type: BirenDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 0, + Type: BirenDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -825,10 +853,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[BirenDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[BirenDevice])) + } if test.wantFit { - if len(result[BirenDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[BirenDevice])) - } for idx, id := range test.wantDevIDs { if id != result[BirenDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[BirenDevice][idx].UUID) diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 0f1eaa3433..232dc20a87 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -330,7 +330,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := cam.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/cambricon/device_test.go b/pkg/device/cambricon/device_test.go index eb3ffa007e..fca3ac9b8f 100644 --- a/pkg/device/cambricon/device_test.go +++ b/pkg/device/cambricon/device_test.go @@ -977,7 +977,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1078,10 +1078,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 128, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: CambriconMLUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 64, + MemPercentagereq: 0, + Coresreq: 50, + Type: CambriconMLUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -1096,10 +1124,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[CambriconMLUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[CambriconMLUDevice])) + } if test.wantFit { - if len(result[CambriconMLUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[CambriconMLUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[CambriconMLUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[CambriconMLUDevice][idx].UUID) diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index f5343a1a85..1aafe7972b 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -408,7 +408,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := enf.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/enflame/device_test.go b/pkg/device/enflame/device_test.go index cfa3e2e7cd..0f73563d76 100644 --- a/pkg/device/enflame/device_test.go +++ b/pkg/device/enflame/device_test.go @@ -167,6 +167,7 @@ func TestFit_SelectProfileByRequest(t *testing.T) { Used: 0, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", @@ -203,6 +204,7 @@ func TestFit_SelectProfileByMemoryCoreRequest(t *testing.T) { Used: 0, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", @@ -289,6 +291,7 @@ func TestFit_MutexRejectsUsedDevice(t *testing.T) { Used: 1, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index 5d1bc4e836..642adbe37f 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -150,7 +150,11 @@ func (gcuDev *GCUDevices) Fit(devices []*device.DeviceUsage, request device.Cont for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } if !gcuDev.checkType(k) { reason[common.CardTypeMismatch]++ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type) diff --git a/pkg/device/enflame/gcu_test.go b/pkg/device/enflame/gcu_test.go index 6e4bf2c3f5..cb0df277c7 100644 --- a/pkg/device/enflame/gcu_test.go +++ b/pkg/device/enflame/gcu_test.go @@ -612,10 +612,38 @@ func TestGCUDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 100, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: EnflameGCUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 100, + MemPercentagereq: 100, + Coresreq: 100, + Type: EnflameGCUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -630,10 +658,10 @@ func TestGCUDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[EnflameGCUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGCUDevice])) + } if test.wantFit { - if len(result[EnflameGCUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGCUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[EnflameGCUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[EnflameGCUDevice][idx].UUID) diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index f09bbb0e66..548586e25e 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -256,7 +256,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := dcu.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/hygon/device_test.go b/pkg/device/hygon/device_test.go index 0811002227..f54f2d4e3d 100644 --- a/pkg/device/hygon/device_test.go +++ b/pkg/device/hygon/device_test.go @@ -1070,7 +1070,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1171,10 +1171,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: HygonDCUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: HygonDCUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -1189,10 +1217,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[HygonDCUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[HygonDCUDevice])) + } if test.wantFit { - if len(result[HygonDCUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[HygonDCUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[HygonDCUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[HygonDCUDevice][idx].UUID) diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index b11df66daa..3057cc6c87 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -268,7 +268,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := ilu.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index b2cf0efb99..4fc33538e6 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -589,6 +589,33 @@ func Test_Fit(t *testing.T) { wantLen: 0, wantDevIDs: []string{}, }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 128, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: "MR-V100", + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Type: "MR-V100", + Memreq: 64, + MemPercentagereq: 0, + Coresreq: 50, + }, + annos: map[string]string{}, + wantOK: false, + wantLen: 0, + wantDevIDs: []string{}, + }, } for _, test := range tests { diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index b08e2b14fd..576503f617 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -217,5 +217,5 @@ func (dev *KunlunDevices) GetResourceNames() device.ResourceNames { } func FitXPU(device *device.DeviceUsage, request device.ContainerDeviceRequest) bool { - return device.Used == 0 + return device.Used == 0 && device.Health } diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index f148d9d1db..fc75d024f3 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -275,6 +275,9 @@ func (dev *KunlunVDevices) Fit(devices []*device.DeviceUsage, request device.Con } func FitVXPU(device *device.DeviceUsage, request device.ContainerDeviceRequest) bool { + if !device.Health { + return false + } if request.Memreq+device.Usedmem > device.Totalmem { return false } diff --git a/pkg/device/kunlun/vdevice_test.go b/pkg/device/kunlun/vdevice_test.go index daaecbc63b..44c5866738 100644 --- a/pkg/device/kunlun/vdevice_test.go +++ b/pkg/device/kunlun/vdevice_test.go @@ -526,6 +526,12 @@ func Test_FitVXPU_direct(t *testing.T) { request: device.ContainerDeviceRequest{Memreq: 24576}, want: false, }, + { + name: "unhealthy device is rejected", + usage: &device.DeviceUsage{Health: false, Used: 0, Usedmem: 0, Totalmem: 98304}, + request: device.ContainerDeviceRequest{Memreq: 24576}, + want: false, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1f7fe4648f..bcb909d210 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -232,7 +232,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := mat.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/metax/device_test.go b/pkg/device/metax/device_test.go index 9d8fdd6c20..2bc1aefbac 100644 --- a/pkg/device/metax/device_test.go +++ b/pkg/device/metax/device_test.go @@ -822,6 +822,34 @@ func TestMetaxDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "1/1 ExclusiveDeviceAllocateConflict", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: MetaxGPUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: MetaxGPUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 9e13ed075c..27a074e934 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -292,7 +292,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type) if !strings.Contains(dev.Type, k.Type) { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go index 901129eeaf..5ee0bbd4a1 100644 --- a/pkg/device/mthreads/device_test.go +++ b/pkg/device/mthreads/device_test.go @@ -1099,6 +1099,34 @@ func TestDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: MthreadsGPUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: MthreadsGPUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { diff --git a/pkg/device/vastai/device.go b/pkg/device/vastai/device.go index beccf16baa..48b0a9c10e 100644 --- a/pkg/device/vastai/device.go +++ b/pkg/device/vastai/device.go @@ -249,7 +249,11 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta } for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := va.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/vastai/device_test.go b/pkg/device/vastai/device_test.go index 99fef0d685..50aaf428b7 100644 --- a/pkg/device/vastai/device_test.go +++ b/pkg/device/vastai/device_test.go @@ -840,6 +840,34 @@ func TestDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 0, + Totalcore: 0, + Usedcores: 0, + Numa: 0, + Type: VastaiDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 0, + Type: VastaiDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { From 40dfe0b8d36f35cfa3e87a5d9592eaee3f49c73c Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Sun, 2 Aug 2026 20:37:21 +0530 Subject: [PATCH 2/7] fix: address CodeRabbit suggestions and CI lint failures - Fix duplicate Health field in awsneuron DeviceInfo struct literal - Remove invalid Health field from ContainerDevice/ContainerDeviceRequest literals (not valid fields on those types) - Fix undefined 'reason' var in iluvatar TestPatchAnnotations - Add wantReason assertion to iluvatar Test_Fit loop - Add Health: true to kunlun positive FitVXPU fixtures (Go zero-inits omitted bool to false, causing health check to reject valid fixtures) - Add CardNotHealth test cases for amd, biren, cambricon, enflame, hygon, iluvatar, kunlun, metax, mthreads, vastai, awsneuron backends Signed-off-by: Eshiv-Pandey Signed-off-by: Eshiv Pandey --- pkg/device/awsneuron/device_test.go | 6 +++--- pkg/device/iluvatar/device_test.go | 7 ++++++- pkg/device/kunlun/vdevice_test.go | 4 ++-- pkg/device/metax/device_test.go | 8 ++++---- pkg/device/mthreads/device_test.go | 8 ++++---- pkg/device/vastai/device_test.go | 8 ++++---- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index b7d9237033..2a525baa60 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -905,10 +905,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[AWSNeuronDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[AWSNeuronDevice])) + } if test.wantFit { - if len(result[AWSNeuronDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[AWSNeuronDevice])) - } for idx, id := range test.wantDevIDs { if id != result[AWSNeuronDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[AWSNeuronDevice][idx].UUID) diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index 4fc33538e6..13653441af 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -436,6 +436,7 @@ func Test_Fit(t *testing.T) { wantOK bool wantLen int wantDevIDs []string + wantReason string }{ { name: "fit success", @@ -615,6 +616,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", }, } @@ -626,7 +628,7 @@ func Test_Fit(t *testing.T) { Annotations: test.annos, }, } - ok, result, _ := dev.Fit(test.devices, test.request, pod, &device.NodeInfo{}, allocated) + ok, result, reason := dev.Fit(test.devices, test.request, pod, &device.NodeInfo{}, allocated) if test.wantOK { if len(result["MR-V100"]) != test.wantLen { t.Errorf("expected %d, got %d", test.wantLen, len(result["MR-V100"])) @@ -647,6 +649,9 @@ func Test_Fit(t *testing.T) { t.Errorf("expected %d, got %d", test.wantLen, len(result["MR-V100"])) } } + if reason != test.wantReason { + t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason) + } }) } } diff --git a/pkg/device/kunlun/vdevice_test.go b/pkg/device/kunlun/vdevice_test.go index 44c5866738..8bb03ee250 100644 --- a/pkg/device/kunlun/vdevice_test.go +++ b/pkg/device/kunlun/vdevice_test.go @@ -510,13 +510,13 @@ func Test_FitVXPU_direct(t *testing.T) { }, { name: "idle device always fits", - usage: &device.DeviceUsage{Used: 0, Usedmem: 0, Totalmem: 98304}, + usage: &device.DeviceUsage{Health: true, Used: 0, Usedmem: 0, Totalmem: 98304}, request: device.ContainerDeviceRequest{Memreq: 24576}, want: true, }, { name: "shared device with matching average memory", - usage: &device.DeviceUsage{Used: 2, Usedmem: 49152, Totalmem: 98304}, + usage: &device.DeviceUsage{Health: true, Used: 2, Usedmem: 49152, Totalmem: 98304}, request: device.ContainerDeviceRequest{Memreq: 24576}, want: true, }, diff --git a/pkg/device/metax/device_test.go b/pkg/device/metax/device_test.go index 2bc1aefbac..95ff9cf0cc 100644 --- a/pkg/device/metax/device_test.go +++ b/pkg/device/metax/device_test.go @@ -761,7 +761,7 @@ func TestMetaxDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -864,10 +864,10 @@ func TestMetaxDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[MetaxGPUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxGPUDevice])) + } if test.wantFit { - if len(result[MetaxGPUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxGPUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[MetaxGPUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[MetaxGPUDevice][idx].UUID) diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go index 5ee0bbd4a1..c608447029 100644 --- a/pkg/device/mthreads/device_test.go +++ b/pkg/device/mthreads/device_test.go @@ -994,7 +994,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1141,10 +1141,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[MthreadsGPUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MthreadsGPUDevice])) + } if test.wantFit { - if len(result[MthreadsGPUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MthreadsGPUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[MthreadsGPUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[MthreadsGPUDevice][idx].UUID) diff --git a/pkg/device/vastai/device_test.go b/pkg/device/vastai/device_test.go index 50aaf428b7..568fa31b42 100644 --- a/pkg/device/vastai/device_test.go +++ b/pkg/device/vastai/device_test.go @@ -734,7 +734,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -882,10 +882,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[VastaiDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[VastaiDevice])) + } if test.wantFit { - if len(result[VastaiDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[VastaiDevice])) - } for idx, id := range test.wantDevIDs { if id != result[VastaiDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[VastaiDevice][idx].UUID) From 0df0b9d55a4f53729ac994b160c7b27a83e43ca4 Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Sun, 2 Aug 2026 23:39:12 +0530 Subject: [PATCH 3/7] test: fix broken tests after health check added to Fit() After adding the !dev.Health early-exit to Fit() across all non-nvidia backends, two test files had devices without Health: true set: - enflame/device_test.go: TestFit_SelectProfileByRequest, TestFit_SelectProfileByMemoryCoreRequest, and TestFit_MutexRejectsUsedDevice used DeviceUsage structs with Health defaulting to false, causing the health gate to fire before the logic under test. Set Health: true on those devices. - iluvatar/device_test.go: Test_Fit existing failure-path cases had no wantReason set (empty string), but the test loop now checks reason unconditionally. Added the correct expected reason strings: CardInsufficientMemory, CardTypeMismatch, ExclusiveDeviceAllocateConflict. Signed-off-by: Eshiv Pandey --- pkg/device/iluvatar/device_test.go | 4 ++++ pkg/device/mthreads/device_test.go | 2 +- pkg/device/vastai/device_test.go | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index 13653441af..44d7b4c9fc 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -506,6 +506,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardInsufficientMemory", }, { name: "fit fail: core not enough", @@ -533,6 +534,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardInsufficientMemory", }, { name: "fit fail: type mismatch", @@ -560,6 +562,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardTypeMismatch", }, { name: "mutex policy rejects used device", @@ -589,6 +592,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 ExclusiveDeviceAllocateConflict", }, { name: "fit fail: CardNotHealth", diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go index c608447029..79df98e58f 100644 --- a/pkg/device/mthreads/device_test.go +++ b/pkg/device/mthreads/device_test.go @@ -1095,7 +1095,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, diff --git a/pkg/device/vastai/device_test.go b/pkg/device/vastai/device_test.go index 568fa31b42..072c2f5281 100644 --- a/pkg/device/vastai/device_test.go +++ b/pkg/device/vastai/device_test.go @@ -836,7 +836,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, From 80d4167a3ac6d4cc64822568f9c4a077ef5f0421 Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Mon, 3 Aug 2026 16:22:06 +0530 Subject: [PATCH 4/7] test: add Health: true to graphSelect test fixtures after health check added to Fit() The !dev.Health early-exit added to Fit() and continuousDeviceAvailable() caused graphSelect test fixtures to fail because DeviceUsage.Health defaults to false in Go, making every device appear unhealthy. Add Health: true to all DeviceUsage fixtures in: - awsneuron/device_test.go: Test_graphSelect - kunlun/device_test.go: Test_graphSelect, Test_graphSelectVXPU, TestKunlunVDevices_Fit_Mutex Signed-off-by: Eshiv Pandey --- pkg/device/awsneuron/device_test.go | 128 +++++++-------- pkg/device/kunlun/device_test.go | 242 ++++++++++++++-------------- 2 files changed, 185 insertions(+), 185 deletions(-) diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index 2a525baa60..db679ebcc2 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -453,24 +453,24 @@ func Test_graphSelect(t *testing.T) { c int }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ + {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ AWSNodeType: "inf2", }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, + {Index: 8, Used: 0, Health: true}, + {Index: 9, Used: 0, Health: true}, + {Index: 10, Used: 0, Health: true}, + {Index: 11, Used: 0, Health: true}, + {Index: 12, Used: 0, Health: true}, + {Index: 13, Used: 0, Health: true}, + {Index: 14, Used: 0, Health: true}, + {Index: 15, Used: 0, Health: true}, }, c: 16, }, @@ -483,24 +483,24 @@ func Test_graphSelect(t *testing.T) { c int }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ + {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ AWSNodeType: "trn", }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 1}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 1, Health: true}, + {Index: 8, Used: 0, Health: true}, + {Index: 9, Used: 0, Health: true}, + {Index: 10, Used: 0, Health: true}, + {Index: 11, Used: 0, Health: true}, + {Index: 12, Used: 0, Health: true}, + {Index: 13, Used: 1, Health: true}, + {Index: 14, Used: 0, Health: true}, + {Index: 15, Used: 0, Health: true}, }, c: 8, }, @@ -513,24 +513,24 @@ func Test_graphSelect(t *testing.T) { c int }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ + {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ AWSNodeType: "trn", }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 1, Health: true}, + {Index: 8, Used: 0, Health: true}, + {Index: 9, Used: 0, Health: true}, + {Index: 10, Used: 0, Health: true}, + {Index: 11, Used: 0, Health: true}, + {Index: 12, Used: 0, Health: true}, + {Index: 13, Used: 0, Health: true}, + {Index: 14, Used: 0, Health: true}, + {Index: 15, Used: 0, Health: true}, }, c: 8, }, @@ -543,24 +543,24 @@ func Test_graphSelect(t *testing.T) { c int }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ + {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ AWSNodeType: "inf", }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 1}, - {Index: 15, Used: 0}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, + {Index: 8, Used: 0, Health: true}, + {Index: 9, Used: 0, Health: true}, + {Index: 10, Used: 0, Health: true}, + {Index: 11, Used: 0, Health: true}, + {Index: 12, Used: 0, Health: true}, + {Index: 13, Used: 0, Health: true}, + {Index: 14, Used: 1, Health: true}, + {Index: 15, Used: 0, Health: true}, }, c: 8, }, diff --git a/pkg/device/kunlun/device_test.go b/pkg/device/kunlun/device_test.go index 4eb82d1cbf..e3e9b9c139 100644 --- a/pkg/device/kunlun/device_test.go +++ b/pkg/device/kunlun/device_test.go @@ -34,7 +34,7 @@ func TestKunlunVDevices_Fit_Mutex(t *testing.T) { // FitVXPU would accept them for sharing. devices := make([]*device.DeviceUsage, 8) for i := range devices { - devices[i] = &device.DeviceUsage{Index: uint(i), Used: 1, Usedmem: 24576, Totalmem: 98304} + devices[i] = &device.DeviceUsage{Index: uint(i), Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true} } req := device.ContainerDeviceRequest{Nums: 1, Memreq: 24576} nodeInfo := &device.NodeInfo{} @@ -69,14 +69,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8}, }, @@ -89,14 +89,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8}, }, @@ -109,14 +109,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2}, }, @@ -129,14 +129,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1}, }, @@ -149,14 +149,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 1}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 1, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1}, }, @@ -169,14 +169,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1}, - {Index: 1, Used: 1}, - {Index: 2, Used: 1}, - {Index: 3, Used: 0}, - {Index: 4, Used: 1}, - {Index: 5, Used: 1}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 1, Health: true}, + {Index: 1, Used: 1, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 1, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2}, }, @@ -189,14 +189,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 1}, - {Index: 3, Used: 1}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 1, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 1, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -209,14 +209,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 1}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 1, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -229,14 +229,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 1}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -267,14 +267,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8, Memreq: 24576}, }, @@ -287,14 +287,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4, Memreq: 24576}, }, @@ -307,14 +307,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4, Memreq: 24576}, }, @@ -327,14 +327,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 1, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 6, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 7, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request + {Index: 0, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 1, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 6, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 7, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request }, c: device.ContainerDeviceRequest{Nums: 2, Memreq: 24576}, }, @@ -347,14 +347,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1, Memreq: 24576}, }, @@ -367,14 +367,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 6, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 6, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2, Memreq: 24576}, }, From beb7f68d6719b2d8a46de0926a65782fffe92aa0 Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Mon, 3 Aug 2026 18:08:50 +0530 Subject: [PATCH 5/7] test: add CardNotHealth coverage for enflame and restore health check lost in rebase The rebase onto latest master dropped two changes from the original PR: 1. The !dev.Health check in enflame/device.go Fit() was lost 2. Health: true on test fixtures in enflame/device_test.go was lost This commit restores both and adds a TestFit_UnhealthyDeviceRejected test to bring enflame/device.go patch coverage to 100%. Signed-off-by: Eshiv Pandey --- pkg/device/enflame/device_test.go | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg/device/enflame/device_test.go b/pkg/device/enflame/device_test.go index 0f73563d76..8c63c412f4 100644 --- a/pkg/device/enflame/device_test.go +++ b/pkg/device/enflame/device_test.go @@ -314,6 +314,38 @@ func TestFit_MutexRejectsUsedDevice(t *testing.T) { assert.Equal(t, reason, "1/1 ExclusiveDeviceAllocateConflict") } +func TestFit_UnhealthyDeviceRejected(t *testing.T) { + dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"}) + devices := []*device.DeviceUsage{ + { + ID: "node-a-enflame-drs-0", + Index: 0, + Count: 6, + Used: 0, + Totalmem: 40960, + Type: EnflameVGCUDevice, + Health: false, + CustomInfo: map[string]any{ + "minor": "0", + "index": "0", + "profiles": map[string]string{ + "1g.6gb": "0", + "3g.20gb": "1", + "6g.40gb": "2", + }, + }, + }, + } + req := device.ContainerDeviceRequest{ + Nums: 1, + Type: EnflameVGCUDevice, + Memreq: 3, + } + fit, _, reason := dev.Fit(devices, req, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}}, &device.NodeInfo{}, &device.PodDevices{}) + assert.Equal(t, fit, false) + assert.Assert(t, strings.Contains(reason, "CardNotHealth")) +} + func TestPatchAnnotations_DRSFields(t *testing.T) { dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"}) pod := &corev1.Pod{ From 8f6700257e125c4f05142bf9ed734340582dd4e9 Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Tue, 4 Aug 2026 17:23:03 +0530 Subject: [PATCH 6/7] refactor(awsneuron): use helper functions for test device construction Signed-off-by: Eshiv Pandey --- pkg/device/awsneuron/device_test.go | 311 +++++++--------------------- 1 file changed, 71 insertions(+), 240 deletions(-) diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index db679ebcc2..74a7fc1bed 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -437,6 +437,49 @@ func Test_countMaskAvailable(t *testing.T) { } } +// makeDeviceUsages builds a 16-element []*device.DeviceUsage slice for use in +// Test_graphSelect. nodeType is stored in index 0's CustomInfo under AWSNodeType. +// usedOverrides maps device index to its Used value; all other indices default to 0. +// health sets the Health field on every entry. +func makeDeviceUsages(nodeType string, usedOverrides map[int]int32, health bool) []*device.DeviceUsage { + const total = 16 + devices := make([]*device.DeviceUsage, total) + for i := 0; i < total; i++ { + du := &device.DeviceUsage{Index: uint(i), Health: health} + if i == 0 { + du.CustomInfo = map[string]any{AWSNodeType: nodeType} + } + if used, ok := usedOverrides[i]; ok { + du.Used = used + } + devices[i] = du + } + return devices +} + +// makeAWSDeviceUsage constructs a single *device.DeviceUsage for use in TestDevices_Fit. +// id is the device UUID, index is the device index, used/count are the sharing-slot +// fields, totalcore/usedcores are the core-mask fields, nodeType goes into +// CustomInfo[AWSNodeType], and health marks whether the device is schedulable. +func makeAWSDeviceUsage(id string, index uint, used, count, totalcore, usedcores int32, nodeType string, health bool) *device.DeviceUsage { + return &device.DeviceUsage{ + ID: id, + Index: index, + Used: used, + Count: count, + Usedmem: 0, + Totalmem: 0, + Totalcore: totalcore, + Usedcores: usedcores, + Numa: 0, + Type: AWSNeuronDevice, + Health: health, + CustomInfo: map[string]any{ + AWSNodeType: nodeType, + }, + } +} + func Test_graphSelect(t *testing.T) { tests := []struct { name string @@ -452,26 +495,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ - AWSNodeType: "inf2", - }}, - {Index: 1, Used: 0, Health: true}, - {Index: 2, Used: 0, Health: true}, - {Index: 3, Used: 0, Health: true}, - {Index: 4, Used: 0, Health: true}, - {Index: 5, Used: 0, Health: true}, - {Index: 6, Used: 0, Health: true}, - {Index: 7, Used: 0, Health: true}, - {Index: 8, Used: 0, Health: true}, - {Index: 9, Used: 0, Health: true}, - {Index: 10, Used: 0, Health: true}, - {Index: 11, Used: 0, Health: true}, - {Index: 12, Used: 0, Health: true}, - {Index: 13, Used: 0, Health: true}, - {Index: 14, Used: 0, Health: true}, - {Index: 15, Used: 0, Health: true}, - }, + d: makeDeviceUsages("inf2", nil, true), c: 16, }, want1: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, @@ -482,26 +506,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ - AWSNodeType: "trn", - }}, - {Index: 1, Used: 0, Health: true}, - {Index: 2, Used: 0, Health: true}, - {Index: 3, Used: 0, Health: true}, - {Index: 4, Used: 0, Health: true}, - {Index: 5, Used: 0, Health: true}, - {Index: 6, Used: 0, Health: true}, - {Index: 7, Used: 1, Health: true}, - {Index: 8, Used: 0, Health: true}, - {Index: 9, Used: 0, Health: true}, - {Index: 10, Used: 0, Health: true}, - {Index: 11, Used: 0, Health: true}, - {Index: 12, Used: 0, Health: true}, - {Index: 13, Used: 1, Health: true}, - {Index: 14, Used: 0, Health: true}, - {Index: 15, Used: 0, Health: true}, - }, + d: makeDeviceUsages("trn", map[int]int32{7: 1, 13: 1}, true), c: 8, }, want1: []int{}, @@ -512,26 +517,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ - AWSNodeType: "trn", - }}, - {Index: 1, Used: 0, Health: true}, - {Index: 2, Used: 0, Health: true}, - {Index: 3, Used: 0, Health: true}, - {Index: 4, Used: 0, Health: true}, - {Index: 5, Used: 0, Health: true}, - {Index: 6, Used: 0, Health: true}, - {Index: 7, Used: 1, Health: true}, - {Index: 8, Used: 0, Health: true}, - {Index: 9, Used: 0, Health: true}, - {Index: 10, Used: 0, Health: true}, - {Index: 11, Used: 0, Health: true}, - {Index: 12, Used: 0, Health: true}, - {Index: 13, Used: 0, Health: true}, - {Index: 14, Used: 0, Health: true}, - {Index: 15, Used: 0, Health: true}, - }, + d: makeDeviceUsages("trn", map[int]int32{7: 1}, true), c: 8, }, want1: []int{8, 9, 10, 11, 12, 13, 14, 15}, @@ -542,26 +528,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Health: true, CustomInfo: map[string]any{ - AWSNodeType: "inf", - }}, - {Index: 1, Used: 0, Health: true}, - {Index: 2, Used: 0, Health: true}, - {Index: 3, Used: 0, Health: true}, - {Index: 4, Used: 0, Health: true}, - {Index: 5, Used: 1, Health: true}, - {Index: 6, Used: 0, Health: true}, - {Index: 7, Used: 0, Health: true}, - {Index: 8, Used: 0, Health: true}, - {Index: 9, Used: 0, Health: true}, - {Index: 10, Used: 0, Health: true}, - {Index: 11, Used: 0, Health: true}, - {Index: 12, Used: 0, Health: true}, - {Index: 13, Used: 0, Health: true}, - {Index: 14, Used: 1, Health: true}, - {Index: 15, Used: 0, Health: true}, - }, + d: makeDeviceUsages("inf", map[int]int32{5: 1, 14: 1}, true), c: 8, }, want1: []int{6, 7, 8, 9, 10, 11, 12, 13}, @@ -595,38 +562,8 @@ func TestDevices_Fit(t *testing.T) { { name: "fit success", devices: []*device.DeviceUsage{ - { - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, - { - ID: "dev-1", - Index: 0, - Used: 0, - Count: 12, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + makeAWSDeviceUsage("dev-1", 0, 0, 12, 3, 0, "trn", true), }, request: device.ContainerDeviceRequest{ Nums: 1, @@ -643,22 +580,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: memory not enough", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 2, Memreq: 0, @@ -674,22 +598,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: core not enough", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 1, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 1, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -705,22 +616,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: type mismatch", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Health: true, - Type: AWSNeuronDevice, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Type: "OtherType", @@ -736,22 +634,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: user assign use uuid mismatch", - devices: []*device.DeviceUsage{{ - ID: "dev-1", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-1", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -767,22 +652,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: user assign no use uuid match", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -798,22 +670,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: card overused", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 2, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 2, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -830,22 +689,7 @@ func TestDevices_Fit(t *testing.T) { { name: "mutex policy rejects used device", devices: []*device.DeviceUsage{ - { - ID: "dev-0", - Index: 0, - Used: 1, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, + makeAWSDeviceUsage("dev-0", 0, 1, 2, 3, 0, "trn", true), }, request: device.ContainerDeviceRequest{ Nums: 1, @@ -862,22 +706,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: CardNotHealth", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: false, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", false), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, From 3e3fd8a2f7fccbfa2397d66232e58308bc36085e Mon Sep 17 00:00:00 2001 From: Eshiv Pandey Date: Tue, 4 Aug 2026 18:09:26 +0530 Subject: [PATCH 7/7] fix(awsneuron): resolve golangci-lint gofmt and modernize issues - Run gofmt -w to fix formatting - Replace for i := 0; i < total; i++ with for i := range 16 to satisfy the modernize linter (range over int, Go 1.22+) Signed-off-by: Eshiv Pandey --- pkg/device/awsneuron/device_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index 89ee12b32e..85a2d7d05e 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -442,9 +442,8 @@ func Test_countMaskAvailable(t *testing.T) { // usedOverrides maps device index to its Used value; all other indices default to 0. // health sets the Health field on every entry. func makeDeviceUsages(nodeType string, usedOverrides map[int]int32, health bool) []*device.DeviceUsage { - const total = 16 - devices := make([]*device.DeviceUsage, total) - for i := 0; i < total; i++ { + devices := make([]*device.DeviceUsage, 16) + for i := range 16 { du := &device.DeviceUsage{Index: uint(i), Health: health} if i == 0 { du.CustomInfo = map[string]any{AWSNodeType: nodeType} @@ -770,4 +769,4 @@ func TestDevices_Fit(t *testing.T) { } }) } -} \ No newline at end of file +}