feat: support comma-separated gpu-scheduler-policy combinations

pkg/device/amd/device.go

@@ -282,7 +282,7 @@ func (amddevice *AMDDevices) Fit(devices []*device.DeviceUsage, request device.C
 	klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/ascend/device.go

@@ -433,7 +433,7 @@ func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerD
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 
 	vnpuMode := ""
 	if pod != nil && pod.Annotations != nil {

pkg/device/awsneuron/device.go

@@ -365,7 +365,7 @@ func (neuron *AWSNeuronDevices) Fit(devices []*device.DeviceUsage, request devic
 	klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	if k.Nums > 1 {
 		alloc := graphSelect(devices, int(request.Nums))
 		if len(alloc) == 0 {

pkg/device/biren/device.go

@@ -193,7 +193,7 @@ func (br *BirenDevices) Fit(devices []*device.DeviceUsage, request device.Contai
 	klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, dev := range slices.Backward(devices) {
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)
 		if !dev.Health {

pkg/device/cambricon/device.go

@@ -362,7 +362,7 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/enflame/device.go

@@ -385,7 +385,7 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con
 	klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	profile, profileMatch := enf.selectProfileByRequest(devices, k)
 	if !profileMatch {
 		reason[common.ModeNotFit]++

pkg/device/hygon/device.go

@@ -252,7 +252,7 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/iluvatar/device.go

@@ -268,7 +268,7 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/kunlun/vdevice.go

@@ -233,7 +233,7 @@ func (dev *KunlunVDevices) Fit(devices []*device.DeviceUsage, request device.Con
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
 
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	fitFn := FitFn(FitVXPU)
 	if isMutex {
 		// mutex: only idle devices are eligible, no sharing onto a used device.

pkg/device/metax/device.go

@@ -228,7 +228,7 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/metax/sdevice.go

@@ -317,7 +317,7 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con
 
 	// filter device
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	candidateDevices := []*device.DeviceUsage{}
 	for i, v := range slices.Backward(devices) {
 		dev := v

pkg/device/mthreads/device.go

@@ -291,7 +291,7 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co
 	var tmpDevs map[string]device.ContainerDevices
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	for i, v := range slices.Backward(devices) {
 		dev := v
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/nvidia/device.go

@@ -688,8 +688,8 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co
 	tmpDevs = make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
 	gpuPolicy := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod)
-	needTopology := gpuPolicy == util.GPUSchedulerPolicyTopology.String()
-	isMutex := gpuPolicy == util.GPUSchedulerPolicyMutex.String()
+	needTopology := util.PolicyContains(gpuPolicy, util.GPUSchedulerPolicyTopology)
+	isMutex := util.PolicyContains(gpuPolicy, util.GPUSchedulerPolicyMutex)
 	for i := len(devices) - 1; i >= 0; i-- {
 		dev := devices[i]
 		klog.V(4).InfoS("scoring pod", "pod", klog.KObj(pod), "device", dev.ID, "Memreq", k.Memreq, "MemPercentagereq", k.MemPercentagereq, "Coresreq", k.Coresreq, "Nums", k.Nums, "device index", i)

pkg/device/vastai/device.go

@@ -231,7 +231,7 @@ func (va *VastaiDevices) Fit(devices []*device.DeviceUsage, request device.Conta
 	klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", k)
 	tmpDevs := make(map[string]device.ContainerDevices)
 	reason := make(map[string]int)
-	isMutex := util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod) == util.GPUSchedulerPolicyMutex.String()
+	isMutex := util.PolicyContains(util.GetGPUSchedulerPolicyByPod(device.GPUSchedulerPolicy, pod), util.GPUSchedulerPolicyMutex)
 	dieMode := isDieMode(devices)
 	// Under mutex a physical card (AIC) is exclusive: in die mode a card is made
 	// of several dies, so if any die on a card is in use none of its sibling dies

pkg/scheduler/policy/gpu_policy.go

@@ -17,6 +17,8 @@ limitations under the License.
 package policy
 
 import (
+	"strings"
+
 	"github.com/Project-HAMi/HAMi/pkg/device"
 	"github.com/Project-HAMi/HAMi/pkg/util"
 
@@ -45,7 +47,43 @@ func (l DeviceUsageList) Swap(i, j int) {
 	l.DeviceLists[i], l.DeviceLists[j] = l.DeviceLists[j], l.DeviceLists[i]
 }
 
+// gpuSortKeyOrder lists the recognized sort-key policies, in the priority
+// used when a caller-supplied chain doesn't disambiguate them further.
+var gpuSortKeyOrder = []util.SchedulerPolicyName{
+	util.GPUSchedulerPolicyBinpack,
+	util.GPUSchedulerPolicySpread,
+	util.GPUSchedulerPolicyNuma,
+}
+
+// gpuSortKeyChain parses policy as a comma-separated ordered list and returns
+// the sort-key policies (binpack/spread/numa) it names, in the order written,
+// deduplicated. mutex and topology-aware are filters consumed via
+// util.PolicyContains in Fit(), not sort keys, so they're dropped here.
+func gpuSortKeyChain(policy string) []util.SchedulerPolicyName {
+	seen := make(map[util.SchedulerPolicyName]bool, len(gpuSortKeyOrder))
+	var chain []util.SchedulerPolicyName
+	for p := range strings.SplitSeq(policy, ",") {
+		name := util.SchedulerPolicyName(strings.TrimSpace(p))
+		for _, key := range gpuSortKeyOrder {
+			if name == key && !seen[key] {
+				chain = append(chain, key)
+				seen[key] = true
+			}
+		}
+	}
+	return chain
+}
+
 func (l DeviceUsageList) Less(i, j int) bool {
+	// Comma-separated policy: chain binpack/spread/numa as sort keys in the
+	// order the caller wrote them. mutex/topology-aware are filters applied
+	// in each device backend's Fit(), not sort keys, so they don't appear here.
+	// Bare "numa" also routes here: it's a chain token, not a legacy value.
+	if strings.Contains(l.Policy, ",") || l.Policy == util.GPUSchedulerPolicyNuma.String() {
+		return l.lessByChain(i, j)
+	}
+
+	// Single policy value: unchanged behavior.
 	si, sj := l.DeviceLists[i].Score, l.DeviceLists[j].Score
 	ni, nj := l.DeviceLists[i].Device.Numa, l.DeviceLists[j].Device.Numa
 	binpack := l.Policy == util.GPUSchedulerPolicyBinpack.String()
@@ -88,6 +126,46 @@ func (l DeviceUsageList) Less(i, j int) bool {
 	return ni < nj
 }
 
+// lessByChain compares devices i and j by the ordered sort-key chain parsed
+// from l.Policy, falling back to spread (today's default policy) when the
+// comma list names no recognized sort key (e.g. "mutex,topology-aware").
+func (l DeviceUsageList) lessByChain(i, j int) bool {
+	chain := gpuSortKeyChain(l.Policy)
+	if len(chain) == 0 {
+		chain = []util.SchedulerPolicyName{util.GPUSchedulerPolicySpread}
+	}
+	// numa-bind requires NUMA groups to stay contiguous for Fit's same-NUMA
+	// accumulation, so force numa as the primary key if the chain omits it.
+	if l.NumaBind && chain[0] != util.GPUSchedulerPolicyNuma {
+		withNuma := []util.SchedulerPolicyName{util.GPUSchedulerPolicyNuma}
+		for _, key := range chain {
+			if key != util.GPUSchedulerPolicyNuma {
+				withNuma = append(withNuma, key)
+			}
+		}
+		chain = withNuma
+	}
+	a, b := l.DeviceLists[i], l.DeviceLists[j]
+	for _, key := range chain {
+		switch key {
+		case util.GPUSchedulerPolicyBinpack:
+			if a.Score != b.Score {
+				return a.Score < b.Score
+			}
+		case util.GPUSchedulerPolicySpread:
+			if a.Score != b.Score {
+				return a.Score > b.Score
+			}
+		case util.GPUSchedulerPolicyNuma:
+			if a.Device.Numa != b.Device.Numa {
+				return a.Device.Numa < b.Device.Numa
+			}
+		}
+	}
+	// Deterministic tiebreak when every chained key is equal.
+	return a.Device.Index < b.Device.Index
+}
+
 func (l DeviceUsageList) DeepCopy() DeviceUsageList {
 	var deviceLists []*DeviceListsScore
 	if l.DeviceLists != nil {

pkg/scheduler/policy/gpu_policy_test.go

@@ -277,6 +277,107 @@ func TestDeviceUsageList_Less(t *testing.T) {
 			},
 			expectedLess: true,
 		},
+		{
+			name:   "Bare numa: NUMA is the sort key regardless of score",
+			policy: "numa",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 99},
+				{Device: &device.DeviceUsage{Numa: 1}, Score: 5},
+			},
+			expectedLess: true,
+		},
+		{
+			name:   "Bare numa: same NUMA falls to deterministic Index tiebreak",
+			policy: "numa",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Index: 3, Numa: 0}, Score: 10},
+				{Device: &device.DeviceUsage{Index: 1, Numa: 0}, Score: 20},
+			},
+			expectedLess: false,
+		},
+		{
+			name:   "Chain binpack,numa: score primary, numa tiebreak when scores equal",
+			policy: "binpack,numa",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 15},
+				{Device: &device.DeviceUsage{Numa: 1}, Score: 15},
+			},
+			expectedLess: true,
+		},
+		{
+			name:   "Chain spread,numa: score primary, numa tiebreak when scores equal",
+			policy: "spread,numa",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 15},
+				{Device: &device.DeviceUsage{Numa: 1}, Score: 15},
+			},
+			expectedLess: true,
+		},
+		{
+			name:   "Chain numa,spread: numa primary regardless of score",
+			policy: "numa,spread",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 1}, Score: 5},
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 99},
+			},
+			expectedLess: false,
+		},
+		{
+			name:   "Chain numa,spread: same NUMA falls back to spread (higher score first)",
+			policy: "numa,spread",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 20},
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 10},
+			},
+			expectedLess: true,
+		},
+		{
+			name:   "Chain mutex,spread: mutex is a Fit() filter, not a sort key; spread decides order",
+			policy: "mutex,spread",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Used: 5}, Score: 10},
+				{Device: &device.DeviceUsage{Used: 0}, Score: 20},
+			},
+			expectedLess: false,
+		},
+		{
+			name:   "Chain with only filter tokens (mutex,topology-aware) falls back to spread",
+			policy: "mutex,topology-aware",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 10},
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 20},
+			},
+			expectedLess: false,
+		},
+		{
+			name:   "Chain binpack,numa: deterministic Index tiebreak when score and numa both tie",
+			policy: "binpack,numa",
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Index: 5, Numa: 0}, Score: 10},
+				{Device: &device.DeviceUsage{Index: 2, Numa: 0}, Score: 10},
+			},
+			expectedLess: false,
+		},
+		{
+			name:     "Chain binpack,spread + NumaBind: numa forced primary to keep groups contiguous",
+			policy:   "binpack,spread",
+			numaBind: true,
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 99},
+				{Device: &device.DeviceUsage{Numa: 1}, Score: 5},
+			},
+			expectedLess: true,
+		},
+		{
+			name:     "Chain spread,numa + NumaBind: numa moved to front, spread breaks same-NUMA ties",
+			policy:   "spread,numa",
+			numaBind: true,
+			deviceLists: []*DeviceListsScore{
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 20},
+				{Device: &device.DeviceUsage{Numa: 0}, Score: 10},
+			},
+			expectedLess: true,
+		},
 	}
 
 	for _, tt := range tests {

pkg/util/types.go

@@ -78,6 +78,8 @@ const (
 	GPUSchedulerPolicyTopology SchedulerPolicyName = "topology-aware"
 	// GPUSchedulerPolicyMutex is GPU use mutex scheduler, only idle GPUs are allocated.
 	GPUSchedulerPolicyMutex SchedulerPolicyName = "mutex"
+	// GPUSchedulerPolicyNuma is GPU use numa scheduler, chained as a sort key alongside binpack/spread.
+	GPUSchedulerPolicyNuma SchedulerPolicyName = "numa"
 )
 
 const (

pkg/util/util.go

@@ -284,6 +284,19 @@ func GetGPUSchedulerPolicyByPod(defaultPolicy string, task *corev1.Pod) string {
 	return userGPUPolicy
 }
 
+// PolicyContains reports whether policy names name, treating policy as a
+// comma-separated ordered list (e.g. "binpack,numa"). A single value with no
+// comma is compared directly, so existing single-policy callers are unaffected.
+func PolicyContains(policy string, name SchedulerPolicyName) bool {
+	target := name.String()
+	for p := range strings.SplitSeq(policy, ",") {
+		if strings.TrimSpace(p) == target {
+			return true
+		}
+	}
+	return false
+}
+
 func IsPodInTerminatedState(pod *corev1.Pod) bool {
 	if pod == nil {
 		return false

pkg/util/util_test.go

@@ -787,6 +787,29 @@ func TestGetGPUSchedulerPolicyByPod(t *testing.T) {
 	}
 }
 
+func TestPolicyContains(t *testing.T) {
+	tests := []struct {
+		name   string
+		policy string
+		target SchedulerPolicyName
+		want   bool
+	}{
+		{"single value match", "mutex", GPUSchedulerPolicyMutex, true},
+		{"single value no match", "spread", GPUSchedulerPolicyMutex, false},
+		{"empty policy", "", GPUSchedulerPolicyMutex, false},
+		{"comma list match first", "mutex,spread", GPUSchedulerPolicyMutex, true},
+		{"comma list match last", "spread,numa,mutex", GPUSchedulerPolicyMutex, true},
+		{"comma list no match", "binpack,spread", GPUSchedulerPolicyMutex, false},
+		{"comma list with spaces", "mutex, spread, numa", GPUSchedulerPolicyNuma, true},
+		{"chain of sort keys, no filter", "binpack,spread,numa", GPUSchedulerPolicyTopology, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, PolicyContains(tt.policy, tt.target))
+		})
+	}
+}
+
 func TestSchedulerPolicyName_String(t *testing.T) {
 	tests := []struct {
 		policy SchedulerPolicyName