diff --git a/pkg/device/amd/device.go b/pkg/device/amd/device.go index 98b748dc87..d316bbc534 100644 --- a/pkg/device/amd/device.go +++ b/pkg/device/amd/device.go @@ -284,7 +284,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type, "dev=", dev) _, found, _ := amddevice.checkType(pod.GetAnnotations(), *dev, k) if !found { diff --git a/pkg/device/amd/device_test.go b/pkg/device/amd/device_test.go index 8d7e943d29..ad4bacdcc9 100644 --- a/pkg/device/amd/device_test.go +++ b/pkg/device/amd/device_test.go @@ -458,4 +458,20 @@ func TestDevices_Fit(t *testing.T) { assert.Equal(t, false, ok) assert.Assert(t, strings.Contains(reason, common.CardTimeSlicingExhausted)) }) + + t.Run("unhealthy device is rejected", func(t *testing.T) { + devices := []*device.DeviceUsage{ + { + ID: "dev-0", Index: 0, Used: 0, Count: 2, + Usedmem: 0, Totalmem: 1000, Totalcore: 100, Usedcores: 0, + Type: AMDDevice, Health: false, CustomInfo: map[string]any{}, + }, + } + req := device.ContainerDeviceRequest{Nums: 1, Type: AMDDevice, Memreq: 100, MemPercentagereq: 0, Coresreq: 10} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} + + ok, _, reason := dev.Fit(devices, req, pod, &device.NodeInfo{}, &device.PodDevices{}) + assert.Equal(t, false, ok) + assert.Assert(t, strings.Contains(reason, common.CardNotHealth)) + }) } diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 27c3023070..3e4e44ce21 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -313,7 +313,7 @@ func continuousDeviceAvailable(devices []*device.DeviceUsage, start int, count i res := []int{} iterator := start for iterator < start+count { - if devices[iterator].Used > 0 { + if devices[iterator].Used > 0 || !devices[iterator].Health { return []int{} } res = append(res, iterator) @@ -398,7 +398,11 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic dev.CustomInfo[AWSUsageInfo] = int(dev.Usedcores) } klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type, "dev=", dev) if !strings.Contains(dev.Type, k.Type) { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/awsneuron/device_test.go b/pkg/device/awsneuron/device_test.go index 149fb25e73..85a2d7d05e 100644 --- a/pkg/device/awsneuron/device_test.go +++ b/pkg/device/awsneuron/device_test.go @@ -437,6 +437,48 @@ func Test_countMaskAvailable(t *testing.T) { } } +// makeDeviceUsages builds a 16-element []*device.DeviceUsage slice for use in +// Test_graphSelect. nodeType is stored in index 0's CustomInfo under AWSNodeType. +// usedOverrides maps device index to its Used value; all other indices default to 0. +// health sets the Health field on every entry. +func makeDeviceUsages(nodeType string, usedOverrides map[int]int32, health bool) []*device.DeviceUsage { + devices := make([]*device.DeviceUsage, 16) + for i := range 16 { + du := &device.DeviceUsage{Index: uint(i), Health: health} + if i == 0 { + du.CustomInfo = map[string]any{AWSNodeType: nodeType} + } + if used, ok := usedOverrides[i]; ok { + du.Used = used + } + devices[i] = du + } + return devices +} + +// makeAWSDeviceUsage constructs a single *device.DeviceUsage for use in TestDevices_Fit. +// id is the device UUID, index is the device index, used/count are the sharing-slot +// fields, totalcore/usedcores are the core-mask fields, nodeType goes into +// CustomInfo[AWSNodeType], and health marks whether the device is schedulable. +func makeAWSDeviceUsage(id string, index uint, used, count, totalcore, usedcores int32, nodeType string, health bool) *device.DeviceUsage { + return &device.DeviceUsage{ + ID: id, + Index: index, + Used: used, + Count: count, + Usedmem: 0, + Totalmem: 0, + Totalcore: totalcore, + Usedcores: usedcores, + Numa: 0, + Type: AWSNeuronDevice, + Health: health, + CustomInfo: map[string]any{ + AWSNodeType: nodeType, + }, + } +} + func Test_graphSelect(t *testing.T) { tests := []struct { name string @@ -452,26 +494,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ - AWSNodeType: "inf2", - }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, - }, + d: makeDeviceUsages("inf2", nil, true), c: 16, }, want1: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, @@ -482,26 +505,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ - AWSNodeType: "trn", - }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 1}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, - }, + d: makeDeviceUsages("trn", map[int]int32{7: 1, 13: 1}, true), c: 8, }, want1: []int{}, @@ -512,26 +516,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ - AWSNodeType: "trn", - }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 0}, - {Index: 15, Used: 0}, - }, + d: makeDeviceUsages("trn", map[int]int32{7: 1}, true), c: 8, }, want1: []int{8, 9, 10, 11, 12, 13, 14, 15}, @@ -542,26 +527,7 @@ func Test_graphSelect(t *testing.T) { d []*device.DeviceUsage c int }{ - d: []*device.DeviceUsage{ - {Index: 0, Used: 0, CustomInfo: map[string]any{ - AWSNodeType: "inf", - }}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, - {Index: 8, Used: 0}, - {Index: 9, Used: 0}, - {Index: 10, Used: 0}, - {Index: 11, Used: 0}, - {Index: 12, Used: 0}, - {Index: 13, Used: 0}, - {Index: 14, Used: 1}, - {Index: 15, Used: 0}, - }, + d: makeDeviceUsages("inf", map[int]int32{5: 1, 14: 1}, true), c: 8, }, want1: []int{6, 7, 8, 9, 10, 11, 12, 13}, @@ -595,38 +561,8 @@ func TestDevices_Fit(t *testing.T) { { name: "fit success", devices: []*device.DeviceUsage{ - { - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, - { - ID: "dev-1", - Index: 0, - Used: 0, - Count: 12, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + makeAWSDeviceUsage("dev-1", 0, 0, 12, 3, 0, "trn", true), }, request: device.ContainerDeviceRequest{ Nums: 1, @@ -643,22 +579,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: memory not enough", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 2, Memreq: 0, @@ -674,22 +597,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: core not enough", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 1, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 1, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -705,22 +615,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: type mismatch", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Health: true, - Type: AWSNeuronDevice, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Type: "OtherType", @@ -736,22 +633,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: user assign use uuid mismatch", - devices: []*device.DeviceUsage{{ - ID: "dev-1", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-1", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -767,22 +651,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: user assign no use uuid match", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 0, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -798,22 +669,9 @@ func TestDevices_Fit(t *testing.T) { }, { name: "fit fail: card overused", - devices: []*device.DeviceUsage{{ - ID: "dev-0", - Index: 0, - Used: 2, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }}, + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 2, 2, 3, 0, "trn", true), + }, request: device.ContainerDeviceRequest{ Nums: 1, Memreq: 0, @@ -830,22 +688,7 @@ func TestDevices_Fit(t *testing.T) { { name: "mutex policy rejects used device", devices: []*device.DeviceUsage{ - { - ID: "dev-0", - Index: 0, - Used: 1, - Count: 2, - Usedmem: 0, - Totalmem: 0, - Totalcore: 3, - Usedcores: 0, - Numa: 0, - Type: AWSNeuronDevice, - Health: true, - CustomInfo: map[string]any{ - AWSNodeType: "trn", - }, - }, + makeAWSDeviceUsage("dev-0", 0, 1, 2, 3, 0, "trn", true), }, request: device.ContainerDeviceRequest{ Nums: 1, @@ -863,16 +706,8 @@ func TestDevices_Fit(t *testing.T) { { name: "fit fail: NumaNotFit with multiple devices", devices: []*device.DeviceUsage{ - { - ID: "dev-0", Index: 0, Used: 0, Count: 2, Totalcore: 3, - Type: AWSNeuronDevice, Health: true, - CustomInfo: map[string]any{AWSNodeType: "trn"}, - }, - { - ID: "dev-1", Index: 1, Used: 0, Count: 2, Totalcore: 3, - Type: AWSNeuronDevice, Health: true, - CustomInfo: map[string]any{AWSNodeType: "trn"}, - }, + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", true), + makeAWSDeviceUsage("dev-1", 1, 0, 2, 3, 0, "trn", true), }, request: device.ContainerDeviceRequest{ Nums: 2, @@ -887,6 +722,24 @@ func TestDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "1/2 NumaNotFit", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{ + makeAWSDeviceUsage("dev-0", 0, 0, 2, 3, 0, "trn", false), + }, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 2, + Type: AWSNeuronDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -901,10 +754,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[AWSNeuronDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[AWSNeuronDevice])) + } if test.wantFit { - if len(result[AWSNeuronDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[AWSNeuronDevice])) - } for idx, id := range test.wantDevIDs { if id != result[AWSNeuronDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[AWSNeuronDevice][idx].UUID) diff --git a/pkg/device/biren/device.go b/pkg/device/biren/device.go index 10b1655cd4..93b7dbb980 100644 --- a/pkg/device/biren/device.go +++ b/pkg/device/biren/device.go @@ -196,7 +196,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String() for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := br.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/biren/device_test.go b/pkg/device/biren/device_test.go index 31d8321836..4712cb0181 100644 --- a/pkg/device/biren/device_test.go +++ b/pkg/device/biren/device_test.go @@ -734,7 +734,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -807,10 +807,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 0, + Totalcore: 0, + Usedcores: 0, + Numa: 0, + Type: BirenDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 0, + Type: BirenDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -825,10 +853,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[BirenDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[BirenDevice])) + } if test.wantFit { - if len(result[BirenDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[BirenDevice])) - } for idx, id := range test.wantDevIDs { if id != result[BirenDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[BirenDevice][idx].UUID) diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 0f1eaa3433..232dc20a87 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -330,7 +330,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := cam.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/cambricon/device_test.go b/pkg/device/cambricon/device_test.go index eb3ffa007e..fca3ac9b8f 100644 --- a/pkg/device/cambricon/device_test.go +++ b/pkg/device/cambricon/device_test.go @@ -977,7 +977,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1078,10 +1078,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 128, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: CambriconMLUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 64, + MemPercentagereq: 0, + Coresreq: 50, + Type: CambriconMLUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -1096,10 +1124,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[CambriconMLUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[CambriconMLUDevice])) + } if test.wantFit { - if len(result[CambriconMLUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[CambriconMLUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[CambriconMLUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[CambriconMLUDevice][idx].UUID) diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index f5343a1a85..1aafe7972b 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -408,7 +408,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := enf.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/enflame/device_test.go b/pkg/device/enflame/device_test.go index cfa3e2e7cd..8c63c412f4 100644 --- a/pkg/device/enflame/device_test.go +++ b/pkg/device/enflame/device_test.go @@ -167,6 +167,7 @@ func TestFit_SelectProfileByRequest(t *testing.T) { Used: 0, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", @@ -203,6 +204,7 @@ func TestFit_SelectProfileByMemoryCoreRequest(t *testing.T) { Used: 0, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", @@ -289,6 +291,7 @@ func TestFit_MutexRejectsUsedDevice(t *testing.T) { Used: 1, Totalmem: 40960, Type: EnflameVGCUDevice, + Health: true, CustomInfo: map[string]any{ "minor": "0", "index": "0", @@ -311,6 +314,38 @@ func TestFit_MutexRejectsUsedDevice(t *testing.T) { assert.Equal(t, reason, "1/1 ExclusiveDeviceAllocateConflict") } +func TestFit_UnhealthyDeviceRejected(t *testing.T) { + dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"}) + devices := []*device.DeviceUsage{ + { + ID: "node-a-enflame-drs-0", + Index: 0, + Count: 6, + Used: 0, + Totalmem: 40960, + Type: EnflameVGCUDevice, + Health: false, + CustomInfo: map[string]any{ + "minor": "0", + "index": "0", + "profiles": map[string]string{ + "1g.6gb": "0", + "3g.20gb": "1", + "6g.40gb": "2", + }, + }, + }, + } + req := device.ContainerDeviceRequest{ + Nums: 1, + Type: EnflameVGCUDevice, + Memreq: 3, + } + fit, _, reason := dev.Fit(devices, req, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}}, &device.NodeInfo{}, &device.PodDevices{}) + assert.Equal(t, fit, false) + assert.Assert(t, strings.Contains(reason, "CardNotHealth")) +} + func TestPatchAnnotations_DRSFields(t *testing.T) { dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"}) pod := &corev1.Pod{ diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index 5d1bc4e836..642adbe37f 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -150,7 +150,11 @@ func (gcuDev *GCUDevices) Fit(devices []*device.DeviceUsage, request device.Cont for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } if !gcuDev.checkType(k) { reason[common.CardTypeMismatch]++ klog.V(5).InfoS(common.CardTypeMismatch, "pod", klog.KObj(pod), "device", dev.ID, dev.Type, k.Type) diff --git a/pkg/device/enflame/gcu_test.go b/pkg/device/enflame/gcu_test.go index 6e4bf2c3f5..cb0df277c7 100644 --- a/pkg/device/enflame/gcu_test.go +++ b/pkg/device/enflame/gcu_test.go @@ -612,10 +612,38 @@ func TestGCUDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 100, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: EnflameGCUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 100, + MemPercentagereq: 100, + Coresreq: 100, + Type: EnflameGCUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -630,10 +658,10 @@ func TestGCUDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[EnflameGCUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGCUDevice])) + } if test.wantFit { - if len(result[EnflameGCUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGCUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[EnflameGCUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[EnflameGCUDevice][idx].UUID) diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index f09bbb0e66..548586e25e 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -256,7 +256,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := dcu.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/hygon/device_test.go b/pkg/device/hygon/device_test.go index 0811002227..f54f2d4e3d 100644 --- a/pkg/device/hygon/device_test.go +++ b/pkg/device/hygon/device_test.go @@ -1070,7 +1070,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1171,10 +1171,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: HygonDCUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: HygonDCUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -1189,10 +1217,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[HygonDCUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[HygonDCUDevice])) + } if test.wantFit { - if len(result[HygonDCUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[HygonDCUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[HygonDCUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[HygonDCUDevice][idx].UUID) diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index b11df66daa..3057cc6c87 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -268,7 +268,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := ilu.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/iluvatar/device_test.go b/pkg/device/iluvatar/device_test.go index b2cf0efb99..44d7b4c9fc 100644 --- a/pkg/device/iluvatar/device_test.go +++ b/pkg/device/iluvatar/device_test.go @@ -436,6 +436,7 @@ func Test_Fit(t *testing.T) { wantOK bool wantLen int wantDevIDs []string + wantReason string }{ { name: "fit success", @@ -505,6 +506,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardInsufficientMemory", }, { name: "fit fail: core not enough", @@ -532,6 +534,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardInsufficientMemory", }, { name: "fit fail: type mismatch", @@ -559,6 +562,7 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 CardTypeMismatch", }, { name: "mutex policy rejects used device", @@ -588,6 +592,35 @@ func Test_Fit(t *testing.T) { wantOK: false, wantLen: 0, wantDevIDs: []string{}, + wantReason: "1/1 ExclusiveDeviceAllocateConflict", + }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 128, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: "MR-V100", + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Type: "MR-V100", + Memreq: 64, + MemPercentagereq: 0, + Coresreq: 50, + }, + annos: map[string]string{}, + wantOK: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", }, } @@ -599,7 +632,7 @@ func Test_Fit(t *testing.T) { Annotations: test.annos, }, } - ok, result, _ := dev.Fit(test.devices, test.request, pod, &device.NodeInfo{}, allocated) + ok, result, reason := dev.Fit(test.devices, test.request, pod, &device.NodeInfo{}, allocated) if test.wantOK { if len(result["MR-V100"]) != test.wantLen { t.Errorf("expected %d, got %d", test.wantLen, len(result["MR-V100"])) @@ -620,6 +653,9 @@ func Test_Fit(t *testing.T) { t.Errorf("expected %d, got %d", test.wantLen, len(result["MR-V100"])) } } + if reason != test.wantReason { + t.Errorf("expected reason: %s, got reason: %s", test.wantReason, reason) + } }) } } diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index 4ed39e771e..914cd6dff4 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -217,5 +217,5 @@ func (dev *KunlunDevices) GetResourceNames() device.ResourceNames { } func FitXPU(device *device.DeviceUsage, request device.ContainerDeviceRequest) bool { - return device.Used == 0 + return device.Used == 0 && device.Health } diff --git a/pkg/device/kunlun/device_test.go b/pkg/device/kunlun/device_test.go index 648aacb8a1..f6e4550911 100644 --- a/pkg/device/kunlun/device_test.go +++ b/pkg/device/kunlun/device_test.go @@ -33,7 +33,7 @@ func TestKunlunVDevices_Fit_Mutex(t *testing.T) { // FitVXPU would accept them for sharing. devices := make([]*device.DeviceUsage, 8) for i := range devices { - devices[i] = &device.DeviceUsage{Index: uint(i), Used: 1, Usedmem: 24576, Totalmem: 98304} + devices[i] = &device.DeviceUsage{Index: uint(i), Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true} } req := device.ContainerDeviceRequest{Nums: 1, Memreq: 24576} nodeInfo := &device.NodeInfo{} @@ -94,14 +94,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8}, }, @@ -114,14 +114,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8}, }, @@ -134,14 +134,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2}, }, @@ -154,14 +154,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 1}, - {Index: 6, Used: 0}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1}, }, @@ -174,14 +174,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 1}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 1, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1}, }, @@ -194,14 +194,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1}, - {Index: 1, Used: 1}, - {Index: 2, Used: 1}, - {Index: 3, Used: 0}, - {Index: 4, Used: 1}, - {Index: 5, Used: 1}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 1, Health: true}, + {Index: 1, Used: 1, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 1, Health: true}, + {Index: 5, Used: 1, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2}, }, @@ -214,14 +214,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 1}, - {Index: 3, Used: 1}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 0}, - {Index: 7, Used: 1}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 1, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 0, Health: true}, + {Index: 7, Used: 1, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -234,14 +234,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 0}, - {Index: 3, Used: 1}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 0, Health: true}, + {Index: 3, Used: 1, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -254,14 +254,14 @@ func Test_graphSelect(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0}, - {Index: 1, Used: 0}, - {Index: 2, Used: 1}, - {Index: 3, Used: 0}, - {Index: 4, Used: 0}, - {Index: 5, Used: 0}, - {Index: 6, Used: 1}, - {Index: 7, Used: 0}, + {Index: 0, Used: 0, Health: true}, + {Index: 1, Used: 0, Health: true}, + {Index: 2, Used: 1, Health: true}, + {Index: 3, Used: 0, Health: true}, + {Index: 4, Used: 0, Health: true}, + {Index: 5, Used: 0, Health: true}, + {Index: 6, Used: 1, Health: true}, + {Index: 7, Used: 0, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4}, }, @@ -292,14 +292,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 8, Memreq: 24576}, }, @@ -312,14 +312,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4, Memreq: 24576}, }, @@ -332,14 +332,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 4, Memreq: 24576}, }, @@ -352,14 +352,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 1, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 6, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 7, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request + {Index: 0, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 1, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 5, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 6, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 7, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request }, c: device.ContainerDeviceRequest{Nums: 2, Memreq: 24576}, }, @@ -372,14 +372,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 1, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 3, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 6, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 1, Memreq: 24576}, }, @@ -392,14 +392,14 @@ func Test_graphSelectVXPU(t *testing.T) { c device.ContainerDeviceRequest }{ d: []*device.DeviceUsage{ - {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 2, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304}, // avgMem = 49152, doesn't match request - {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304}, - {Index: 6, Used: 1, Usedmem: 24576, Totalmem: 98304}, // avgMem = 24576, matches request - {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304}, + {Index: 0, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 1, Used: 2, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 2, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 3, Used: 1, Usedmem: 49152, Totalmem: 98304, Health: true}, // avgMem = 49152, doesn't match request + {Index: 4, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 5, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, + {Index: 6, Used: 1, Usedmem: 24576, Totalmem: 98304, Health: true}, // avgMem = 24576, matches request + {Index: 7, Used: 0, Usedmem: 0, Totalmem: 98304, Health: true}, }, c: device.ContainerDeviceRequest{Nums: 2, Memreq: 24576}, }, diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index 242844b2b3..486b8cae89 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -275,6 +275,9 @@ func (dev *KunlunVDevices) Fit(devices []*device.DeviceUsage, request device.Con } func FitVXPU(device *device.DeviceUsage, request device.ContainerDeviceRequest) bool { + if !device.Health { + return false + } if request.Memreq+device.Usedmem > device.Totalmem { return false } diff --git a/pkg/device/kunlun/vdevice_test.go b/pkg/device/kunlun/vdevice_test.go index daaecbc63b..8bb03ee250 100644 --- a/pkg/device/kunlun/vdevice_test.go +++ b/pkg/device/kunlun/vdevice_test.go @@ -510,13 +510,13 @@ func Test_FitVXPU_direct(t *testing.T) { }, { name: "idle device always fits", - usage: &device.DeviceUsage{Used: 0, Usedmem: 0, Totalmem: 98304}, + usage: &device.DeviceUsage{Health: true, Used: 0, Usedmem: 0, Totalmem: 98304}, request: device.ContainerDeviceRequest{Memreq: 24576}, want: true, }, { name: "shared device with matching average memory", - usage: &device.DeviceUsage{Used: 2, Usedmem: 49152, Totalmem: 98304}, + usage: &device.DeviceUsage{Health: true, Used: 2, Usedmem: 49152, Totalmem: 98304}, request: device.ContainerDeviceRequest{Memreq: 24576}, want: true, }, @@ -526,6 +526,12 @@ func Test_FitVXPU_direct(t *testing.T) { request: device.ContainerDeviceRequest{Memreq: 24576}, want: false, }, + { + name: "unhealthy device is rejected", + usage: &device.DeviceUsage{Health: false, Used: 0, Usedmem: 0, Totalmem: 98304}, + request: device.ContainerDeviceRequest{Memreq: 24576}, + want: false, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1f7fe4648f..bcb909d210 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -232,7 +232,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, numa := mat.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/metax/device_test.go b/pkg/device/metax/device_test.go index 9d8fdd6c20..95ff9cf0cc 100644 --- a/pkg/device/metax/device_test.go +++ b/pkg/device/metax/device_test.go @@ -761,7 +761,7 @@ func TestMetaxDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -822,6 +822,34 @@ func TestMetaxDevices_Fit(t *testing.T) { wantDevIDs: []string{}, wantReason: "1/1 ExclusiveDeviceAllocateConflict", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: MetaxGPUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: MetaxGPUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -836,10 +864,10 @@ func TestMetaxDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[MetaxGPUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxGPUDevice])) + } if test.wantFit { - if len(result[MetaxGPUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MetaxGPUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[MetaxGPUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[MetaxGPUDevice][idx].UUID) diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index e61beb2a77..2fb6be520b 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -292,7 +292,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co for i, v := range slices.Backward(devices) { dev := v klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } klog.V(3).InfoS("Type check", "device", dev.Type, "req", k.Type) if !strings.Contains(dev.Type, k.Type) { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/mthreads/device_test.go b/pkg/device/mthreads/device_test.go index 901129eeaf..79df98e58f 100644 --- a/pkg/device/mthreads/device_test.go +++ b/pkg/device/mthreads/device_test.go @@ -994,7 +994,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -1095,10 +1095,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 100, + Usedmem: 0, + Totalmem: 1280, + Totalcore: 100, + Usedcores: 0, + Numa: 0, + Type: MthreadsGPUDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 512, + MemPercentagereq: 0, + Coresreq: 50, + Type: MthreadsGPUDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -1113,10 +1141,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[MthreadsGPUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MthreadsGPUDevice])) + } if test.wantFit { - if len(result[MthreadsGPUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[MthreadsGPUDevice])) - } for idx, id := range test.wantDevIDs { if id != result[MthreadsGPUDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[MthreadsGPUDevice][idx].UUID) diff --git a/pkg/device/vastai/device.go b/pkg/device/vastai/device.go index beccf16baa..48b0a9c10e 100644 --- a/pkg/device/vastai/device.go +++ b/pkg/device/vastai/device.go @@ -249,7 +249,11 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta } for i, dev := range slices.Backward(devices) { klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i) - + if !dev.Health { + reason[common.CardNotHealth]++ + klog.V(5).InfoS(common.CardNotHealth, "pod", klog.KObj(pod), "device", dev.ID, "health", dev.Health) + continue + } _, found, _ := va.checkType(pod.GetAnnotations(), *dev, k) if !found { reason[common.CardTypeMismatch]++ diff --git a/pkg/device/vastai/device_test.go b/pkg/device/vastai/device_test.go index 99fef0d685..072c2f5281 100644 --- a/pkg/device/vastai/device_test.go +++ b/pkg/device/vastai/device_test.go @@ -734,7 +734,7 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 1, wantDevIDs: []string{}, wantReason: "1/1 AllocatedCardsInsufficientRequest", }, @@ -836,10 +836,38 @@ func TestDevices_Fit(t *testing.T) { }, annos: map[string]string{}, wantFit: false, - wantLen: 0, + wantLen: 2, wantDevIDs: []string{}, wantReason: "2/2 AllocatedCardsInsufficientRequest", }, + { + name: "fit fail: CardNotHealth", + devices: []*device.DeviceUsage{{ + ID: "dev-0", + Index: 0, + Used: 0, + Count: 1, + Usedmem: 0, + Totalmem: 0, + Totalcore: 0, + Usedcores: 0, + Numa: 0, + Type: VastaiDevice, + Health: false, + }}, + request: device.ContainerDeviceRequest{ + Nums: 1, + Memreq: 0, + MemPercentagereq: 0, + Coresreq: 0, + Type: VastaiDevice, + }, + annos: map[string]string{}, + wantFit: false, + wantLen: 0, + wantDevIDs: []string{}, + wantReason: "1/1 CardNotHealth", + }, } for _, test := range tests { @@ -854,10 +882,10 @@ func TestDevices_Fit(t *testing.T) { if fit != test.wantFit { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } + if len(result[VastaiDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[VastaiDevice])) + } if test.wantFit { - if len(result[VastaiDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[VastaiDevice])) - } for idx, id := range test.wantDevIDs { if id != result[VastaiDevice][idx].UUID { t.Errorf("expected device id: %s, got device id %s", id, result[VastaiDevice][idx].UUID)