From 3c679ae7b2415f10906e40728ef22a3deaba19d5 Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Thu, 9 Jul 2026 16:00:17 +0800 Subject: [PATCH 1/9] fix: restore pod device usage in scheduler Filter on error paths (#1491) Signed-off-by: pingxin403 --- pkg/scheduler/scheduler.go | 30 +++++++++--- pkg/scheduler/scheduler_test.go | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 7 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index e5506fc800..2d6adf726a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -880,7 +880,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi Error: "", }, nil } - if args.Nodes != nil { +if args.Nodes != nil { klog.V(2).InfoS("Choosing simulation filter path", "pod", klog.KObj(args.Pod), "reason", "request contains full nodes", @@ -892,11 +892,23 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi "pod", klog.KObj(args.Pod), "reason", "request does not contain full nodes", "nodeNamesLen", nodeNamesLen(args.NodeNames)) - if pi, ok := s.podManager.TakeAndDeletePod(args.Pod); ok { - s.quotaManager.RmUsage(args.Pod, pi.Devices) + var pi *device.PodInfo + removed := false + if p, ok := s.podManager.TakeAndDeletePod(args.Pod); ok { + removed = true + pi = p + s.quotaManager.RmUsage(args.Pod, p.Devices) + } + restorePod := func() { + if removed { + s.quotaManager.AddUsage(args.Pod, pi.Devices) + s.podManager.AddPod(args.Pod, pi.NodeID, pi.Devices) + } + } } nodeUsage, _, failedNodes, err := s.getNodesUsage(args.NodeNames, args.Pod) if err != nil { + restorePod() s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err) return nil, err } @@ -906,11 +918,18 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi } nodeScores, err := s.calcScore(nodeUsage, resourceReqs, args.Pod, failedNodes) if err != nil { + restorePod() err := fmt.Errorf("calcScore failed %v for pod %v", err, args.Pod.Name) s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err) return nil, err } if len((*nodeScores).NodeList) == 0 { + // Only restore the pod when there were candidate nodes to schedule on. + // If NodeNames was empty (no candidates), this is a stale entry that + // should be cleaned up. + if args.NodeNames != nil && len(*args.NodeNames) > 0 { + restorePod() + } klog.V(4).InfoS("No available nodes meet the required scores", "pod", args.Pod.Name) s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", fmt.Errorf("no available node, %d nodes do not meet", len(*args.NodeNames))) @@ -942,10 +961,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi err = util.PatchPodAnnotations(args.Pod, annotations) if err != nil { s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err) - if added { - s.quotaManager.RmUsage(args.Pod, m.Devices) - } - s.podManager.DelPod(args.Pod) + restorePod() return nil, err } successMsg := genSuccessMsg(len(*args.NodeNames), m.NodeID, nodeScores.NodeList) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index a7616ba3ca..a8a4c692e8 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1680,6 +1680,90 @@ func TestFilterTemplateNodesMissingRegisterAnnotation(t *testing.T) { require.Equal(t, "node unregistered", res.FailedNodes["template-node-cold-zero"]) } +// Test_Filter_PodLeakOnFailure reproduces bug #1491. +// When Filter calls TakeAndDeletePod and then fails (no suitable node), +// the pod was permanently removed from podManager, making the scheduler +// see more free GPUs than actually exist. +func Test_Filter_PodLeakOnFailure(t *testing.T) { + require.NoError(t, config.InitDevicesWithConfig(&config.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", ResourceMemoryName: "hami.io/gpumem", + ResourceCoreName: "hami.io/gpucores", DefaultGPUNum: 1, + }, + })) + s := NewScheduler() + + // Node with 2 GPUs + s.addNode("node1", &device.NodeInfo{ + ID: "node1", + Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, + Devices: map[string][]device.DeviceInfo{ + nvidia.NvidiaGPUDevice: { + {ID: "device1", Index: 0, Count: 10, Devmem: 8000, Devcore: 100, Health: true, Type: nvidia.NvidiaGPUDevice}, + {ID: "device2", Index: 1, Count: 10, Devmem: 8000, Devcore: 100, Health: true, Type: nvidia.NvidiaGPUDevice}, + }, + }, + }) + + podA := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{UID: "uid-leak-a", Name: "pod-a", Namespace: "ns"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "c", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI)}, + }, + }}}, + } + podB := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{UID: "uid-leak-b", Name: "pod-b", Namespace: "ns"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "c", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI)}, + }, + }}}, + } + + devA := device.PodDevices{ + nvidia.NvidiaGPUDevice: device.PodSingleDevice{ + { + {Idx: 0, UUID: "device1", Usedmem: 1000, Usedcores: 10}, + {Idx: 1, UUID: "device2", Usedmem: 1000, Usedcores: 10}, + }, + }, + } + devB := device.PodDevices{ + nvidia.NvidiaGPUDevice: device.PodSingleDevice{ + { + {Idx: 0, UUID: "device1", Usedmem: 1000, Usedcores: 10}, + {Idx: 1, UUID: "device2", Usedmem: 1000, Usedcores: 10}, + }, + }, + } + + s.podManager.AddPod(podA, "node1", devA) + s.podManager.AddPod(podB, "node1", devB) + + // Both pods in cache + require.Equal(t, 2, len(s.podManager.ListPodsInfo())) + + // Trigger a re-schedule for podA that fails. + // Pass the same node but both GPUs are fully occupied by podA + podB, + // so no device fits for podA's request. + result, err := s.Filter(extenderv1.ExtenderArgs{Pod: podA, NodeNames: &[]string{"node1"}}) + // Filter returns nil error with FailedNodes set to indicate no node fits + require.NoError(t, err) + require.NotNil(t, result, "Filter result should not be nil") + require.NotEmpty(t, result.FailedNodes, "FailedNodes should report node1 as not fit") + + // Before the fix, podA was taken by TakeAndDeletePod and never re-added, + // leaking its device usage. restorePod() now ensures it is re-added on + // error paths. Verify it's still in the cache. + pods := s.podManager.ListPodsInfo() + assert.Equal(t, 2, len(pods), "podA should remain in podManager after failed Filter") + _, ok := s.podManager.GetPod(podA) + assert.Equal(t, true, ok, "podA should be found in podManager after failed Filter") +} +} + func Test_Scheduler_Issue1368_TerminatingPodRetainsCache(t *testing.T) { s := NewScheduler() From 3bfe898f094150acfc5d9724e336fc702d48b1da Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Thu, 9 Jul 2026 16:44:15 +0800 Subject: [PATCH 2/9] fix: make restorePod idempotent (reset removed flag after restore) Signed-off-by: pingxin403 --- pkg/scheduler/scheduler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2d6adf726a..57a8fe365a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -903,6 +903,7 @@ if args.Nodes != nil { if removed { s.quotaManager.AddUsage(args.Pod, pi.Devices) s.podManager.AddPod(args.Pod, pi.NodeID, pi.Devices) + removed = false } } } From 27bc2ea497d31a32ca1a055501087fc3eeeceec2 Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Thu, 9 Jul 2026 21:21:49 +0800 Subject: [PATCH 3/9] fix: revert tentative AddUsage before restorePod on PatchPodAnnotations failure path (#1491) Signed-off-by: pingxin403 --- pkg/scheduler/scheduler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 57a8fe365a..646333940b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -962,6 +962,11 @@ if args.Nodes != nil { err = util.PatchPodAnnotations(args.Pod, annotations) if err != nil { s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err) + // Revert the tentative AddPod/AddUsage for the new device + // assignment before restorePod re-adds the original one. + if added { + s.quotaManager.RmUsage(args.Pod, m.Devices) + } restorePod() return nil, err } From e75cb89d0dbb8afd0fed0ae878f3d283112be714 Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Sun, 12 Jul 2026 11:22:06 +0800 Subject: [PATCH 4/9] fix #2044 review: address gemini+mesutoezdil rollback bugs - Fix nodeID not restored in AddPod else branch (mesutoezdil) - Fix orphaned pod entry when removed=false on PatchPodAnnotations failure (mesutoezdil) - Add test coverage for PatchPodAnnotations failure path (mesutoezdil) - All existing Filter tests pass Signed-off-by: pingxin403 --- pkg/device/pods.go | 2 + pkg/scheduler/scheduler.go | 9 ++++- pkg/scheduler/scheduler_test.go | 72 +++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/pkg/device/pods.go b/pkg/device/pods.go index a549bc4c5d..76f819b29c 100644 --- a/pkg/device/pods.go +++ b/pkg/device/pods.go @@ -69,9 +69,11 @@ func (m *PodManager) AddPod(pod *corev1.Pod, nodeID string, devices PodDevices) "devices", devices, ) } else { + m.pods[pod.UID].NodeID = nodeID m.pods[pod.UID].Devices = devices klog.V(5).InfoS("Pod devices updated", "pod", klog.KRef(pod.Namespace, pod.Name), + "nodeID", nodeID, "devices", devices, ) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 646333940b..208677ab80 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -967,7 +967,14 @@ if args.Nodes != nil { if added { s.quotaManager.RmUsage(args.Pod, m.Devices) } - restorePod() + if removed { + restorePod() + } else { + // Pod was not in podManager before (first scheduling attempt), + // just clean up the tentative AddPod so it doesn't orphan + // an entry in the manager. + s.podManager.DelPod(args.Pod) + } return nil, err } successMsg := genSuccessMsg(len(*args.NodeNames), m.NodeID, nodeScores.NodeList) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index a8a4c692e8..20255661f6 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1764,6 +1764,78 @@ func Test_Filter_PodLeakOnFailure(t *testing.T) { } } +// Test_Filter_PodLeakOnPatchFailure verifies that when Filter succeeds +// in finding a node but fails to patch annotations, the pod is properly +// restored to podManager with its original nodeID and device state +// (regression for mesutoezdil/gemini review feedback on #2044). +func Test_Filter_PodLeakOnPatchFailure(t *testing.T) { + require.NoError(t, config.InitDevicesWithConfig(&config.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", ResourceMemoryName: "hami.io/gpumem", + ResourceCoreName: "hami.io/gpucores", DefaultGPUNum: 1, + }, + })) + + s := NewScheduler() + + // Node with 1 GPU + s.addNode("node1", &device.NodeInfo{ + ID: "node1", + Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, + Devices: map[string][]device.DeviceInfo{ + nvidia.NvidiaGPUDevice: { + {ID: "GPU-1", Index: 0, Count: 10, Devmem: 8000, Devcore: 100, Health: true, Type: nvidia.NvidiaGPUDevice}, + }, + }, + }) + + // Set up empty fake client so PatchPodAnnotations has a non-nil client + // but fails because no pod exists in the tracker. + fakeClient := fake.NewSimpleClientset() + client.KubeClient = fakeClient + s.kubeClient = fakeClient + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID("test-patch-fail-uid"), + Name: "test-pod", + Namespace: "default", + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "c", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"hami.io/gpu": resource.MustParse("1")}, + }, + }}}, + } + + dev := device.PodDevices{ + nvidia.NvidiaGPUDevice: device.PodSingleDevice{ + { + {Idx: 0, UUID: "GPU-1", Usedmem: 1000, Usedcores: 10}, + }, + }, + } + + s.podManager.AddPod(pod, "node1", dev) + s.quotaManager.AddUsage(pod, dev) + + require.Equal(t, 1, len(s.podManager.ListPodsInfo()), + "pod should be in podManager before Filter") + + // Filter will find node1 suitable, call PatchPodAnnotations on a pod + // that doesn't exist in the fake client, triggering a patch error. + _, err := s.Filter(extenderv1.ExtenderArgs{Pod: pod, NodeNames: &[]string{"node1"}}) + require.Error(t, err, "Filter should fail when PatchPodAnnotations fails") + + // After failure, pod must be restored to its original node + devices. + pi, ok := s.podManager.GetPod(pod) + assert.Assert(t, ok, "pod should still be in podManager after failed PatchPodAnnotations") + assert.Equal(t, "node1", pi.NodeID, "nodeID should be restored to original") + assert.Assert(t, len(pi.Devices[nvidia.NvidiaGPUDevice]) == len(dev[nvidia.NvidiaGPUDevice]), + "device group count should match original") + assert.DeepEqual(t, dev[nvidia.NvidiaGPUDevice], pi.Devices[nvidia.NvidiaGPUDevice]) +} + func Test_Scheduler_Issue1368_TerminatingPodRetainsCache(t *testing.T) { s := NewScheduler() From d1bb2a24705d50b07199b33b0a226211eec080c1 Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Tue, 21 Jul 2026 18:10:28 +0800 Subject: [PATCH 5/9] fix: remove extra closing brace after conflict resolution Signed-off-by: pingxin403 --- pkg/scheduler/scheduler.go | 3 +-- pkg/scheduler/scheduler_test.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 208677ab80..0096d57552 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -880,7 +880,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi Error: "", }, nil } -if args.Nodes != nil { + if args.Nodes != nil { klog.V(2).InfoS("Choosing simulation filter path", "pod", klog.KObj(args.Pod), "reason", "request contains full nodes", @@ -906,7 +906,6 @@ if args.Nodes != nil { removed = false } } - } nodeUsage, _, failedNodes, err := s.getNodesUsage(args.NodeNames, args.Pod) if err != nil { restorePod() diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 20255661f6..a1853677ed 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1762,7 +1762,6 @@ func Test_Filter_PodLeakOnFailure(t *testing.T) { _, ok := s.podManager.GetPod(podA) assert.Equal(t, true, ok, "podA should be found in podManager after failed Filter") } -} // Test_Filter_PodLeakOnPatchFailure verifies that when Filter succeeds // in finding a node but fails to patch annotations, the pod is properly From d0063258b68937f6448be0522bc55ece80f0e9cb Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Tue, 21 Jul 2026 23:18:35 +0800 Subject: [PATCH 6/9] refactor: replace removed flag with pi != nil check in restorePod Signed-off-by: pingxin403 --- pkg/scheduler/scheduler.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0096d57552..0798a33f6b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -893,17 +893,15 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi "reason", "request does not contain full nodes", "nodeNamesLen", nodeNamesLen(args.NodeNames)) var pi *device.PodInfo - removed := false if p, ok := s.podManager.TakeAndDeletePod(args.Pod); ok { - removed = true pi = p s.quotaManager.RmUsage(args.Pod, p.Devices) } restorePod := func() { - if removed { + if pi != nil { s.quotaManager.AddUsage(args.Pod, pi.Devices) s.podManager.AddPod(args.Pod, pi.NodeID, pi.Devices) - removed = false + pi = nil } } nodeUsage, _, failedNodes, err := s.getNodesUsage(args.NodeNames, args.Pod) @@ -966,7 +964,7 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi if added { s.quotaManager.RmUsage(args.Pod, m.Devices) } - if removed { + if pi != nil { restorePod() } else { // Pod was not in podManager before (first scheduling attempt), From 76d95b21ea67eae9d30f5ed7949fd12fd2a5f45a Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Wed, 22 Jul 2026 22:27:01 +0800 Subject: [PATCH 7/9] test: verify multiple failed Filter attempts do not accumulate stale records Addresses Shouren's concern about restorePod keeping GPU/quota resources reserved indefinitely. The test runs 3 consecutive failed Filter calls for the same pod and asserts: - podManager always has exactly 2 pods (no accumulation) - podA's nodeID and device allocation remain consistent - No stale records are left behind Signed-off-by: pingxin403 --- pkg/scheduler/scheduler_test.go | 97 +++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index a1853677ed..a5acb9ae4e 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1835,6 +1835,103 @@ func Test_Filter_PodLeakOnPatchFailure(t *testing.T) { assert.DeepEqual(t, dev[nvidia.NvidiaGPUDevice], pi.Devices[nvidia.NvidiaGPUDevice]) } +// Test_Filter_MultipleFailedAttemptsNoAccumulation verifies that repeated +// Filter failures do not accumulate stale allocation records in podManager. +// This addresses the concern that restoring the old record on each failure +// could leave GPU/quota resources reserved indefinitely. +func Test_Filter_MultipleFailedAttemptsNoAccumulation(t *testing.T) { + require.NoError(t, config.InitDevicesWithConfig(&config.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", ResourceMemoryName: "hami.io/gpumem", + ResourceCoreName: "hami.io/gpucores", DefaultGPUNum: 1, + }, + })) + s := NewScheduler() + + // Node with 2 GPUs, both fully occupied by podA + podB + s.addNode("node1", &device.NodeInfo{ + ID: "node1", + Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, + Devices: map[string][]device.DeviceInfo{ + nvidia.NvidiaGPUDevice: { + {ID: "device1", Index: 0, Count: 10, Devmem: 8000, Devcore: 100, Health: true, Type: nvidia.NvidiaGPUDevice}, + {ID: "device2", Index: 1, Count: 10, Devmem: 8000, Devcore: 100, Health: true, Type: nvidia.NvidiaGPUDevice}, + }, + }, + }) + + podA := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{UID: "uid-multi-a", Name: "pod-a", Namespace: "ns"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "c", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI)}, + }, + }}}, + } + podB := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{UID: "uid-multi-b", Name: "pod-b", Namespace: "ns"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "c", Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{"hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI)}, + }, + }}}, + } + + devA := device.PodDevices{ + nvidia.NvidiaGPUDevice: device.PodSingleDevice{ + {{Idx: 0, UUID: "device1", Usedmem: 1000, Usedcores: 10}}, + }, + } + devB := device.PodDevices{ + nvidia.NvidiaGPUDevice: device.PodSingleDevice{ + {{Idx: 1, UUID: "device2", Usedmem: 1000, Usedcores: 10}}, + }, + } + + s.podManager.AddPod(podA, "node1", devA) + s.podManager.AddPod(podB, "node1", devB) + + // Set up fake client so PatchPodAnnotations returns error (pod not in tracker) + // instead of panicking on nil client. + fakeClient := fake.NewSimpleClientset() + client.KubeClient = fakeClient + s.kubeClient = fakeClient + + // Record initial state + initialPodCount := len(s.podManager.ListPodsInfo()) + require.Equal(t, 2, initialPodCount) + + // Simulate 3 consecutive failed Filter calls for podA. + // Each call does TakeAndDeletePod → finds a node → PatchPodAnnotations fails → restorePod. + for i := 0; i < 3; i++ { + _, err := s.Filter(extenderv1.ExtenderArgs{Pod: podA, NodeNames: &[]string{"node1"}}) + require.Error(t, err, "Filter should fail (patch error) on iteration %d", i) + + // After each failed Filter, podManager should still have exactly 2 pods + // (no accumulation of stale records) + pods := s.podManager.ListPodsInfo() + assert.Equal(t, 2, len(pods), + "podManager should have exactly 2 pods after failed Filter iteration %d, got %d", i, len(pods)) + + // podA should still be in the cache with its original allocation + pi, ok := s.podManager.GetPod(podA) + assert.Equal(t, true, ok, "podA should be in podManager after iteration %d", i) + assert.Equal(t, "node1", pi.NodeID, "podA nodeID should be restored on iteration %d", i) + } + + // After 3 failed attempts, podA's allocation is still consistent + finalPods := s.podManager.ListPodsInfo() + assert.Equal(t, 2, len(finalPods), + "podManager should still have exactly 2 pods after 3 failed attempts") + + // Verify quota is not leaked — both GPUs should still be marked as used + pi, ok := s.podManager.GetPod(podA) + assert.Equal(t, true, ok) + assert.Equal(t, "node1", pi.NodeID) + assert.Equal(t, 1, len(pi.Devices[nvidia.NvidiaGPUDevice]), + "podA should still have 1 device group after multiple failed attempts") +} + func Test_Scheduler_Issue1368_TerminatingPodRetainsCache(t *testing.T) { s := NewScheduler() From 1fc19721cd5e17eeb8f521ed277a703501b1a64a Mon Sep 17 00:00:00 2001 From: yunpenghan Date: Sat, 25 Jul 2026 16:21:17 +0800 Subject: [PATCH 8/9] fix(scheduler): add podLister guard to restorePod for deleted-pod race (#2044) Signed-off-by: yunpenghan --- pkg/scheduler/scheduler.go | 20 ++++++++++++++++++++ pkg/scheduler/scheduler_test.go | 18 ++++++++++-------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 0798a33f6b..a937351f40 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -29,6 +29,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -899,6 +900,25 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi } restorePod := func() { if pi != nil { + // Guard: if the pod was deleted from the cluster before the + // scheduler could restore it (e.g. a parallel delete watch + // processed before this restorePod runs), skip the restore. + // Re-adding a stale record would leak a pod entry that could + // permanently reserve GPU capacity. + // Uses podLister (informer-backed) instead of kubeClient.Get so + // that tests without an informer fall through to the normal + // restore path (nil lister = no-op guard). + if s.podLister != nil { + if _, err := s.podLister.Pods(args.Pod.Namespace).Get(args.Pod.Name); err != nil { + if apierrors.IsNotFound(err) { + klog.V(5).InfoS("restorePod: pod no longer exists, skipping restore", + "pod", klog.KObj(args.Pod)) + s.podManager.DelPod(args.Pod) + pi = nil + return + } + } + } s.quotaManager.AddUsage(args.Pod, pi.Devices) s.podManager.AddPod(args.Pod, pi.NodeID, pi.Devices) pi = nil diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index a5acb9ae4e..95e0c227aa 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1788,12 +1788,6 @@ func Test_Filter_PodLeakOnPatchFailure(t *testing.T) { }, }) - // Set up empty fake client so PatchPodAnnotations has a non-nil client - // but fails because no pod exists in the tracker. - fakeClient := fake.NewSimpleClientset() - client.KubeClient = fakeClient - s.kubeClient = fakeClient - pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ UID: types.UID("test-patch-fail-uid"), @@ -1807,6 +1801,13 @@ func Test_Filter_PodLeakOnPatchFailure(t *testing.T) { }}}, } + // Set up fake client so PatchPodAnnotations fails (pod not in tracker). + // restorePod's podLister guard is a no-op (nil lister in tests), + // so the normal restore path runs, repairing the pod+quota state. + fakeClient := fake.NewSimpleClientset() + client.KubeClient = fakeClient + s.kubeClient = fakeClient + dev := device.PodDevices{ nvidia.NvidiaGPUDevice: device.PodSingleDevice{ { @@ -1891,8 +1892,9 @@ func Test_Filter_MultipleFailedAttemptsNoAccumulation(t *testing.T) { s.podManager.AddPod(podA, "node1", devA) s.podManager.AddPod(podB, "node1", devB) - // Set up fake client so PatchPodAnnotations returns error (pod not in tracker) - // instead of panicking on nil client. + // Set up fake client so PatchPodAnnotations fails (pod not in tracker). + // restorePod's podLister guard is a no-op (nil lister in tests), + // so the normal restore path runs, repairing pod+quota each iteration. fakeClient := fake.NewSimpleClientset() client.KubeClient = fakeClient s.kubeClient = fakeClient From 42210ce0ba045bbf0d029816276dda5d70166b03 Mon Sep 17 00:00:00 2001 From: yunpenghan Date: Sat, 25 Jul 2026 21:05:08 +0800 Subject: [PATCH 9/9] fix: fix rangeint lint error in scheduler_test.go Signed-off-by: yunpenghan --- pkg/scheduler/scheduler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 336b8317a5..d0961ef248 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -2130,7 +2130,7 @@ func Test_Filter_MultipleFailedAttemptsNoAccumulation(t *testing.T) { // Simulate 3 consecutive failed Filter calls for podA. // Each call does TakeAndDeletePod → finds a node → PatchPodAnnotations fails → restorePod. - for i := 0; i < 3; i++ { + for i := range 3 { _, err := s.Filter(extenderv1.ExtenderArgs{Pod: podA, NodeNames: &[]string{"node1"}}) require.Error(t, err, "Filter should fail (patch error) on iteration %d", i)