diff --git a/CHANGELOG.md b/CHANGELOG.md index 6866f672f3..f090b2b8fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -408,3 +408,8 @@ Add "NVIDIA_VISIBLE_DEVICES=none" to none-gpu tasks - Fix initialization error when using tensor parallelism on vLLM above 0.18 - Fix multiple device typos +## v2.9.1 - 2026-08-05 + +**Bug fixes** +- Fix panic on asymmetric NVML GPU Link topologies by logging a warning and returning a fallback score of 0 + diff --git a/pkg/device/nvidia/calculate_score.go b/pkg/device/nvidia/calculate_score.go index 54a874e196..3ee3fbe00b 100644 --- a/pkg/device/nvidia/calculate_score.go +++ b/pkg/device/nvidia/calculate_score.go @@ -22,6 +22,7 @@ import ( "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/go-nvml/pkg/nvml" + "k8s.io/klog/v2" ) // Device represents a GPU device as reported by NVML, including all of its @@ -211,8 +212,9 @@ func calculateGPUPairScore(gpu0 *Device, gpu1 *Device) int { } if len(gpu0.Links[gpu1.Index]) != len(gpu1.Links[gpu0.Index]) { - err := fmt.Errorf("internal error in bestEffort GPU allocator: all P2PLinks between 2 GPUs should be bidirectional") - panic(err) + klog.Warningf("internal error in bestEffort GPU allocator: all P2PLinks between 2 GPUs should be bidirectional, but got %d vs %d between GPU %s and GPU %s", + len(gpu0.Links[gpu1.Index]), len(gpu1.Links[gpu0.Index]), gpu0.UUID, gpu1.UUID) + return 0 } score := 0 diff --git a/pkg/device/nvidia/calculate_score_test.go b/pkg/device/nvidia/calculate_score_test.go index 115b6f4d1e..320ac6f7a0 100644 --- a/pkg/device/nvidia/calculate_score_test.go +++ b/pkg/device/nvidia/calculate_score_test.go @@ -108,6 +108,37 @@ func Test_calculateGPUScore(t *testing.T) { }, }, }, + { + name: "asymmetric links test", + args: []*Device{ + { + Index: 0, + nvlibDevice: nvlibDevice{ + UUID: "gpu0", + }, + Links: map[int][]P2PLink{ + 1: {{Type: SingleNVLINKLink}}, + }, + }, + { + Index: 1, + nvlibDevice: nvlibDevice{ + UUID: "gpu1", + }, + Links: map[int][]P2PLink{}, + }, + }, + want: ListDeviceScore{ + { + UUID: "gpu0", + Score: map[string]int{"gpu1": 0}, + }, + { + UUID: "gpu1", + Score: map[string]int{"gpu0": 0}, + }, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) {