diff --git a/pkg/device/devices.go b/pkg/device/devices.go index a167594ce7..b8b02544d6 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -508,6 +508,10 @@ func DecodePodDevices(checklist map[string]string, annos map[string]string) (Pod } func PlatternMIG(n *MigInUse, templates []Geometry, templateIdx int) { + if templateIdx < 0 || templateIdx >= len(templates) { + klog.ErrorS(nil, "MIG template index out of range", "templateIdx", templateIdx, "templateCount", len(templates)) + return + } var err error for _, val := range templates[templateIdx] { count := 0 diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0008ccf1c1..9e8a4a5d34 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -644,10 +644,18 @@ func (s *Scheduler) getNodesUsage(nodes *[]string, task *corev1.Pod) (*map[strin d.Device.Health = false continue } - tmpIdx, Instance, _ := device.ExtractMigTemplatesFromUUID(udevice.UUID) + tmpIdx, Instance, err := device.ExtractMigTemplatesFromUUID(udevice.UUID) + if err != nil { + klog.ErrorS(err, "skipping device with malformed MIG UUID", "uuid", udevice.UUID) + continue + } if len(d.Device.MigUsage.UsageList) == 0 { device.PlatternMIG(&d.Device.MigUsage, d.Device.MigTemplate, tmpIdx) } + if Instance < 0 || Instance >= len(d.Device.MigUsage.UsageList) { + klog.ErrorS(nil, "skipping device with out-of-range MIG instance", "uuid", udevice.UUID, "instance", Instance, "usageListLen", len(d.Device.MigUsage.UsageList)) + continue + } d.Device.MigUsage.UsageList[Instance].InUse = true klog.V(5).Infoln("add mig usage", d.Device.MigUsage, "template=", d.Device.MigTemplate, "uuid=", d.Device.ID) } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 318db0d603..64f9c7a8bf 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -128,6 +128,76 @@ func Test_getNodesUsage(t *testing.T) { assert.Equal(t, v.Devices.DeviceLists[0].Device.Usedcores, int32(20)) } +// Test_getNodesUsage_MalformedMIGUUID reproduces the crash where a pod carrying +// a malformed MIG UUID makes getNodesUsage panic. Because getNodesUsage runs in +// the background register loop (no recover), such a panic kills the whole +// scheduler process. getNodesUsage must skip the bad device instead of panicking. +func Test_getNodesUsage_MalformedMIGUUID(t *testing.T) { + tests := []struct { + name string + uuid string + mig []device.Geometry + }{ + { + // ExtractMigTemplatesFromUUID returns an error (no '-' inside the + // brackets); the error used to be discarded and templateIdx=-1 was + // passed to PlatternMIG -> templates[-1]. + name: "malformed uuid, error discarded", + uuid: "GPU-MIG33[999]", + mig: nil, + }, + { + // ExtractMigTemplatesFromUUID succeeds but the instance position is + // out of range for the built UsageList -> UsageList[999]. + name: "instance index out of range", + uuid: "GPU-MIG33[0-999]", + mig: []device.Geometry{{{Name: "1g.10gb", Core: 10, Memory: 10, Count: 1}}}, + }, + { + // ExtractMigTemplatesFromUUID succeeds but the template index is + // out of range for MigTemplate -> PlatternMIG indexes templates[5]. + name: "template index out of range", + uuid: "GPU-MIG33[5-0]", + mig: []device.Geometry{{{Name: "1g.10gb", Core: 10, Memory: 10, Count: 1}}}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + nodeMage := newNodeManager() + nodeMage.addNode("node-mig", &device.NodeInfo{ + ID: "node-mig", + Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-mig"}}, + Devices: map[string][]device.DeviceInfo{ + nvidia.NvidiaGPUDevice: {{ + ID: "GPU-MIG33", + Index: 0, + Count: 10, + Devmem: 1024, + Devcore: 100, + Mode: "mig", + MIGTemplate: tc.mig, + Health: true, + }}, + }, + }) + podMap := device.NewPodManager() + podMap.AddPod(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{UID: "poison", Name: "poison-pod", Namespace: "default"}, + }, "node-mig", device.PodDevices{ + "NVIDIA": device.PodSingleDevice{ + []device.ContainerDevice{{Idx: 0, UUID: tc.uuid, Usedmem: 4096}}, + }, + }) + s := Scheduler{nodeManager: nodeMage, podManager: podMap} + nodes := []string{"node-mig"} + require.NotPanics(t, func() { + _, _, _, err := s.getNodesUsage(&nodes, nil) + require.NoError(t, err) + }) + }) + } +} + // test case matrix /** | pod name | node name| pod status | annotations | result |