Skip to content

feat: new mutex GPU scheduling policy #2011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hami-robot merged 7 commits into from
Jul 20, 2026
Merged

feat: new mutex GPU scheduling policy #2011

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b83a621
fix: use Score as primary sort key in GPU scheduler policy
mesutoezdil Jul 3, 2026
4ea20e0
feat: new mutex GPU scheduling policy
mesutoezdil Jul 3, 2026
275f87b
merge: resolve conflict with upstream/master in scheduler.go
mesutoezdil Jul 13, 2026
cf041fd
Merge remote-tracking branch 'upstream/master' into feat/mutex-policy
mesutoezdil Jul 17, 2026
c3e2540
fix(kunlun): report exclusive device conflict instead of NumaNotFit u…
mesutoezdil Jul 17, 2026
87064a2
fix(kunlun): remove dangling log key in NumaNotFit InfoS call
mesutoezdil Jul 17, 2026
4d3b33c
fix(vastai): enforce whole-card mutex exclusivity in die mode
mesutoezdil Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pkg/device/amd/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -189,6 +190,7 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -216,6 +218,11 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}

klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)

Expand Down
25 changes: 25 additions & 0 deletions pkg/device/amd/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,31 @@ func TestDevices_Fit(t *testing.T) {
wantDevIDs: []string{},
wantReason: "1/1 CardTimeSlicingExhausted",
},
{
name: "mutex policy rejects used device",
devices: []*device.DeviceUsage{
{
ID: "dev-0",
Index: 0,
Used: 1,
Count: 2,
Totalcore: 3,
Numa: 0,
Type: AMDDevice,
Health: true,
CustomInfo: map[string]any{},
},
},
request: device.ContainerDeviceRequest{
Nums: 1,
Type: AMDDevice,
},
annos: map[string]string{"hami.io/gpu-scheduler-policy": "mutex"},
wantFit: false,
wantLen: 0,
wantDevIDs: []string{},
wantReason: "1/1 ExclusiveDeviceAllocateConflict",
},
}

for _, test := range tests {
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/ascend/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()

vnpuMode := ""
if pod != nil && pod.Annotations != nil {
Expand Down Expand Up @@ -499,6 +500,11 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {

@DSFans2014 DSFans2014 Jul 20, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hi @mesutoezdil I'm a bit confused about the mutex policy.
Judging from the code, the mutex scheduling policy is implemented as restricted, not best-effort. If a card is already allocated via the mutex policy, can it still be shared with other Pods?
If so, why is restricted used here? And if it cannot be shared with other Pods, why use mutex instead of just requesting a full card?

@mesutoezdil mesutoezdil Jul 20, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hi @mesutoezdil I'm a bit confused about the mutex policy. Judging from the code, the mutex scheduling policy is implemented as restricted, not best-effort. If a card is already allocated via the mutex policy, can it still be shared with other Pods? If so, why is restricted used here? And if it cannot be shared with other Pods, why use mutex instead of just requesting a full card?

hi @DSFans2014 as implemented, it's a placement-time preference for the requesting pod, not a device-level lock, it matches #2009's literal spec ("only allocates GPUs with no existing users, skipped if already in use"), but that spec never covered protecting the device from later non-mutex pods.
So today: a mutex pod won't join a busy card, but a later non-mutex pod can still join a card a mutex pod already claimed.
If true mutual exclusion is the goal, that needs a persistent reservation marker on the device (checked by every pod's Fit(), not just mutex ones). If this is the ‘new/ideal’ architecture, let’s open a new issue. WDYT?

@DSFans2014 DSFans2014 Jul 21, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I got, thank you
Moreover, it would be better to also add some explanation and usage about mutex in the documentation.

@mesutoezdil mesutoezdil Jul 21, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@DSFans2014 sure, opened Project-HAMi/website#625 covering both pages

reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
28 changes: 28 additions & 0 deletions pkg/device/ascend/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2001,6 +2001,34 @@ func TestDevices_Fit(t *testing.T) {
wantReason: "",
nodeAnnotation: map[string]string{VNPUNodeSelectorAnnotation: "true"},
},
{
name: "mutex policy rejects used device",
devices: []*device.DeviceUsage{
{
ID: "dev-0",
Index: 0,
Used: 1,
Count: 100,
Usedmem: 0,
Totalmem: 128,
Totalcore: 100,
Usedcores: 0,
Numa: 0,
Health: true,
},
},
request: device.ContainerDeviceRequest{
Nums: 1,
Memreq: 64,
MemPercentagereq: 0,
Coresreq: 50,
},
annos: map[string]string{"hami.io/gpu-scheduler-policy": "mutex"},
wantFit: false,
wantLen: 0,
wantDevIDs: []string{},
wantReason: "1/1 ExclusiveDeviceAllocateConflict",
},
}

for _, dev := range devs {
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/awsneuron/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if k.Nums > 1 {
alloc := graphSelect(devices, int(request.Nums))
if len(alloc) == 0 {
Expand Down Expand Up @@ -425,6 +426,11 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}

if countMaskAvailable(dev.Totalcore)-countMaskAvailable(dev.Usedcores) < k.Coresreq {
reason[common.CardInsufficientCore]++
Expand Down
33 changes: 33 additions & 0 deletions pkg/device/awsneuron/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,39 @@ func TestDevices_Fit(t *testing.T) {
wantDevIDs: []string{},
wantReason: "1/1 CardTimeSlicingExhausted",
},
{
name: "mutex policy rejects used device",
devices: []*device.DeviceUsage{
{
ID: "dev-0",
Index: 0,
Used: 1,
Count: 2,
Usedmem: 0,
Totalmem: 0,
Totalcore: 3,
Usedcores: 0,
Numa: 0,
Type: AWSNeuronDevice,
Health: true,
CustomInfo: map[string]any{
AWSNodeType: "trn",
},
},
},
request: device.ContainerDeviceRequest{
Nums: 1,
Memreq: 0,
MemPercentagereq: 0,
Coresreq: 2,
Type: AWSNeuronDevice,
},
annos: map[string]string{"hami.io/gpu-scheduler-policy": "mutex"},
wantFit: false,
wantLen: 0,
wantDevIDs: []string{},
wantReason: "1/1 ExclusiveDeviceAllocateConflict",
},
}

for _, test := range tests {
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/biren/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, dev := range slices.Backward(devices) {
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

Expand All @@ -213,6 +214,11 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Nums > 0 {
klog.V(5).InfoS("find fit device", "pod", klog.KObj(pod), "device", dev.ID)
k.Nums--
Expand Down
30 changes: 30 additions & 0 deletions pkg/device/biren/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,36 @@ func TestDevices_Fit(t *testing.T) {
wantDevIDs: []string{},
wantReason: "1/1 AllocatedCardsInsufficientRequest",
},
{
name: "mutex policy rejects used device",
devices: []*device.DeviceUsage{
{
ID: "dev-0",
Index: 0,
Used: 1,
Count: 2,
Usedmem: 0,
Totalmem: 0,
Totalcore: 0,
Usedcores: 0,
Numa: 0,
Type: BirenDevice,
Health: true,
},
},
request: device.ContainerDeviceRequest{
Nums: 1,
Memreq: 0,
MemPercentagereq: 0,
Coresreq: 0,
Type: BirenDevice,
},
annos: map[string]string{"hami.io/gpu-scheduler-policy": "mutex"},
wantFit: false,
wantLen: 0,
wantDevIDs: []string{},
wantReason: "1/1 ExclusiveDeviceAllocateConflict",
},
}

for _, test := range tests {
Expand Down
7 changes: 7 additions & 0 deletions pkg/device/cambricon/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"
"github.com/Project-HAMi/HAMi/pkg/util/client"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -325,6 +326,7 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -356,6 +358,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
30 changes: 30 additions & 0 deletions pkg/device/cambricon/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1009,6 +1009,36 @@ func TestDevices_Fit(t *testing.T) {
wantDevIDs: []string{"dev-0"},
wantReason: "",
},
{
name: "mutex policy rejects used device",
devices: []*device.DeviceUsage{
{
ID: "dev-0",
Index: 0,
Used: 1,
Count: 100,
Usedmem: 0,
Totalmem: 128,
Totalcore: 100,
Usedcores: 0,
Numa: 0,
Type: CambriconMLUDevice,
Health: true,
},
},
request: device.ContainerDeviceRequest{
Nums: 1,
Memreq: 64,
MemPercentagereq: 0,
Coresreq: 50,
Type: CambriconMLUDevice,
},
annos: map[string]string{"hami.io/gpu-scheduler-policy": "mutex"},
wantFit: false,
wantLen: 0,
wantDevIDs: []string{},
wantReason: "1/1 ExclusiveDeviceAllocateConflict",
},
}

for _, test := range tests {
Expand Down
7 changes: 7 additions & 0 deletions pkg/device/enflame/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (

"github.com/Project-HAMi/HAMi/pkg/device"
"github.com/Project-HAMi/HAMi/pkg/device/common"
"github.com/Project-HAMi/HAMi/pkg/util"

corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -384,6 +385,7 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con
klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
tmpDevs := make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
profile, profileMatch := enf.selectProfileByRequest(devices, k)
if !profileMatch {
reason[common.ModeNotFit]++
Expand Down Expand Up @@ -424,6 +426,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if dev.Totalmem-dev.Usedmem < profileMemoryMiB {
reason[common.CardInsufficientMemory]++
klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", profileMemoryMiB)
Expand Down
32 changes: 32 additions & 0 deletions pkg/device/enflame/device_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,38 @@ func TestFit_ProfileNotFound(t *testing.T) {
assert.Assert(t, strings.Contains(reason, "ModeNotFit"))
}

func TestFit_MutexRejectsUsedDevice(t *testing.T) {
dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"})
devices := []*device.DeviceUsage{
{
ID: "node-a-enflame-drs-0",
Index: 0,
Count: 2,
Used: 1,
Totalmem: 40960,
Type: EnflameVGCUDevice,
CustomInfo: map[string]any{
"minor": "0",
"index": "0",
"profiles": map[string]string{
"1g.6gb": "0",
"3g.20gb": "1",
"6g.40gb": "2",
},
},
},
}
req := device.ContainerDeviceRequest{
Nums: 1,
Type: EnflameVGCUDevice,
Memreq: 3,
}
annos := map[string]string{"hami.io/gpu-scheduler-policy": "mutex"}
fit, _, reason := dev.Fit(devices, req, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: annos}}, &device.NodeInfo{}, &device.PodDevices{})
assert.Equal(t, fit, false)
assert.Equal(t, reason, "1/1 ExclusiveDeviceAllocateConflict")
}

func TestPatchAnnotations_DRSFields(t *testing.T) {
dev := InitEnflameDevice(EnflameConfig{ResourceNameDRSGCU: "enflame.com/drs-gcu"})
pod := &corev1.Pod{
Expand Down
6 changes: 6 additions & 0 deletions pkg/device/hygon/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain
var tmpDevs map[string]device.ContainerDevices
tmpDevs = make(map[string]device.ContainerDevices)
reason := make(map[string]int)
isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
for i, v := range slices.Backward(devices) {
dev := v
klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
Expand Down Expand Up @@ -283,6 +284,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain
klog.V(5).InfoS(common.CardTimeSlicingExhausted, "pod", klog.KObj(pod), "device", dev.ID, "count", dev.Count, "used", dev.Used)
continue
}
if isMutex && dev.Used > 0 {
reason[common.ExclusiveDeviceAllocateConflict]++
klog.V(5).InfoS(common.ExclusiveDeviceAllocateConflict, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "used", dev.Used)
continue
}
if k.Coresreq > 100 {
klog.ErrorS(nil, "core limit can't exceed 100", "pod", klog.KObj(pod), "device", dev.ID)
k.Coresreq = 100
Expand Down
Loading
Loading