From d1731a9e8485993196e2d6d121b41cc3983f3b11 Mon Sep 17 00:00:00 2001 From: AdityaShome Date: Fri, 31 Jul 2026 20:51:35 +0530 Subject: [PATCH 1/5] bounds-check MIG UUID parsing in nvidia-smi fallback Signed-off-by: AdityaShome --- .../nvidiadevice/nvinternal/plugin/util.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go index 6f4e1dca51..5cfbc202d3 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go @@ -161,16 +161,22 @@ func GetMigUUIDFromSmiOutput(output string, uuid string, idx int) string { continue } klog.Infoln("inspecting", val) - num := strings.Split(val, "Device")[1] - num = strings.Split(num, ":")[0] + deviceParts := strings.Split(val, "Device") + if len(deviceParts) < 2 { + continue + } + num := strings.Split(deviceParts[1], ":")[0] num = strings.TrimSpace(num) index, err := strconv.Atoi(num) if err != nil { klog.Fatal("atoi failed num=", num) } if index == idx { - outputStr := strings.Split(val, ":")[2] - outputStr = strings.TrimSpace(outputStr) + colonParts := strings.Split(val, ":") + if len(colonParts) < 3 { + continue + } + outputStr := strings.TrimSpace(colonParts[2]) outputStr = strings.TrimRight(outputStr, ")") return outputStr } From 83cb3b3a10b6522e842ac2d5dbc357f708f8dad6 Mon Sep 17 00:00:00 2001 From: AdityaShome Date: Fri, 31 Jul 2026 20:58:46 +0530 Subject: [PATCH 2/5] add regression tests for MIG UUID parsing bounds-check Signed-off-by: AdityaShome --- .../nvinternal/plugin/util_test.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go index 252c555468..10247170d9 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go @@ -846,3 +846,39 @@ func TestWriteMigConfig_RemovesStaleFileOnFailure(t *testing.T) { t.Errorf("expected stale config to be removed, stat err: %v", err) } } + +func TestGetMigUUIDFromSmiOutput_HappyPath(t *testing.T) { + uuid := "GPU-b8f3c1a2-0000-0000-0000-000000000000" + output := "GPU 0: NVIDIA A100-SXM4-40GB (UUID: " + uuid + ")\n" + + " MIG 3g.20gb Device 0: (UUID: MIG-11111111-1111-1111-1111-111111111111)\n" + + " MIG 3g.20gb Device 1: (UUID: MIG-22222222-2222-2222-2222-222222222222)\n" + + "GPU 1: NVIDIA A100-SXM4-40GB (UUID: GPU-c9f4d2b3-0000-0000-0000-000000000000)\n" + + got := GetMigUUIDFromSmiOutput(output, uuid, 1) + want := "MIG-22222222-2222-2222-2222-222222222222" + if got != want { + t.Errorf("expected %q, got %q", want, got) + } +} + +func TestGetMigUUIDFromSmiOutput_MalformedLineWithoutDeviceKeyword(t *testing.T) { + uuid := "GPU-b8f3c1a2-0000-0000-0000-000000000000" + output := "GPU 0: NVIDIA A100-SXM4-40GB (UUID: " + uuid + ")\n" + + " MIG-instance-not-yet-configured\n" + + got := GetMigUUIDFromSmiOutput(output, uuid, 0) + if got != "" { + t.Errorf("expected empty string for malformed line, got %q", got) + } +} + +func TestGetMigUUIDFromSmiOutput_MalformedLineTooFewColons(t *testing.T) { + uuid := "GPU-b8f3c1a2-0000-0000-0000-000000000000" + output := "GPU 0: NVIDIA A100-SXM4-40GB (UUID: " + uuid + ")\n" + + " MIG 3g.20gb Device 0: no-uuid-here\n" + + got := GetMigUUIDFromSmiOutput(output, uuid, 0) + if got != "" { + t.Errorf("expected empty string for malformed line, got %q", got) + } +} From 6292517905843f0bb096ee54e0f0f1f94229f52c Mon Sep 17 00:00:00 2001 From: AdityaShome Date: Sat, 1 Aug 2026 02:14:35 +0530 Subject: [PATCH 3/5] don't fatal-exit on bad index, validate UUID before returning Signed-off-by: AdityaShome --- .../nvidiadevice/nvinternal/plugin/util.go | 8 +++++-- .../nvinternal/plugin/util_test.go | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go index 5cfbc202d3..668c564236 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go @@ -169,15 +169,19 @@ func GetMigUUIDFromSmiOutput(output string, uuid string, idx int) string { num = strings.TrimSpace(num) index, err := strconv.Atoi(num) if err != nil { - klog.Fatal("atoi failed num=", num) + klog.Errorf("failed to parse device index from smi output line %q: %v", val, err) + continue } if index == idx { colonParts := strings.Split(val, ":") - if len(colonParts) < 3 { + if len(colonParts) < 3 || !strings.Contains(colonParts[1], "UUID") { continue } outputStr := strings.TrimSpace(colonParts[2]) outputStr = strings.TrimRight(outputStr, ")") + if !strings.HasPrefix(outputStr, "MIG-") { + continue + } return outputStr } } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go index 10247170d9..8900c0a5dd 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util_test.go @@ -882,3 +882,27 @@ func TestGetMigUUIDFromSmiOutput_MalformedLineTooFewColons(t *testing.T) { t.Errorf("expected empty string for malformed line, got %q", got) } } + +func TestGetMigUUIDFromSmiOutput_NonNumericDeviceIndex(t *testing.T) { + uuid := "GPU-b8f3c1a2-0000-0000-0000-000000000000" + output := "GPU 0: NVIDIA A100-SXM4-40GB (UUID: " + uuid + ")\n" + + " MIG 3g.20gb Device x: (UUID: MIG-11111111-1111-1111-1111-111111111111)\n" + + " MIG 3g.20gb Device 0: (UUID: MIG-22222222-2222-2222-2222-222222222222)\n" + + got := GetMigUUIDFromSmiOutput(output, uuid, 0) + want := "MIG-22222222-2222-2222-2222-222222222222" + if got != want { + t.Errorf("expected non-numeric device index line to be skipped and %q returned, got %q", want, got) + } +} + +func TestGetMigUUIDFromSmiOutput_MatchingIndexWithoutUUIDField(t *testing.T) { + uuid := "GPU-b8f3c1a2-0000-0000-0000-000000000000" + output := "GPU 0: NVIDIA A100-SXM4-40GB (UUID: " + uuid + ")\n" + + " MIG 3g.20gb Device 0: status: not-a-uuid\n" + + got := GetMigUUIDFromSmiOutput(output, uuid, 0) + if got != "" { + t.Errorf("expected empty string when the UUID field is missing, got %q", got) + } +} From af6bbd1fd3884ff7cd1b3402fef0a8722011e82d Mon Sep 17 00:00:00 2001 From: AdityaShome Date: Sat, 1 Aug 2026 09:04:45 +0530 Subject: [PATCH 4/5] skip appending empty MIG UUID in GetContainerDeviceStrArray Signed-off-by: AdityaShome --- pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go index 668c564236..9430fa775e 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go @@ -510,7 +510,12 @@ func (nv *NvidiaDevicePlugin) GetContainerDeviceStrArray(c device.ContainerDevic } } } - tmp = append(tmp, GetMigUUIDFromIndex(val.UUID, position)) + migUUID := GetMigUUIDFromIndex(val.UUID, position) + if migUUID == "" { + klog.Errorf("failed to resolve MIG UUID for %s at position %d, skipping", val.UUID, position) + continue + } + tmp = append(tmp, migUUID) } } klog.V(3).Infoln("mig current=", nv.migCurrent, ":", needsreset, "position=", position, "uuid lists", tmp) From 89319cd2a0ce1e3f94bd9209b63f9d32d7a2afc0 Mon Sep 17 00:00:00 2001 From: AdityaShome Date: Mon, 3 Aug 2026 13:59:00 +0530 Subject: [PATCH 5/5] return an error instead of skipping unresolved MIG UUIDs Signed-off-by: AdityaShome --- .../nvidiadevice/nvinternal/plugin/server.go | 7 +++++- .../nvidiadevice/nvinternal/plugin/util.go | 22 ++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 8a3890ead5..696f03323d 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -645,7 +645,12 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev } devreq = alignedDevreq } - response, err := plugin.getAllocateResponse(plugin.GetContainerDeviceStrArray(devreq)) + containerDevices, err := plugin.GetContainerDeviceStrArray(devreq) + if err != nil { + PodAllocationFailed(nodename, current, NodeLockNvidia) + return nil, fmt.Errorf("failed to resolve container devices: %w", err) + } + response, err := plugin.getAllocateResponse(containerDevices) if err != nil { return nil, fmt.Errorf("failed to get allocate response: %v", err) } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go index 9430fa775e..f158f2a733 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/util.go @@ -188,7 +188,7 @@ func GetMigUUIDFromSmiOutput(output string, uuid string, idx int) string { return "" } -func GetMigUUIDFromIndex(uuid string, idx int) string { +func GetMigUUIDFromIndex(uuid string, idx int) (string, error) { defer nvml.Shutdown() if nvret := nvml.Init(); nvret != nvml.SUCCESS { klog.Errorln("nvml Init err: ", nvret) @@ -212,15 +212,18 @@ func GetMigUUIDFromIndex(uuid string, idx int) string { klog.Fatalf("nvidia-smi -L failed with %s\n", err) } outStr := stdout.String() - uuid := GetMigUUIDFromSmiOutput(outStr, originuuid, idx) - return uuid + migUUID := GetMigUUIDFromSmiOutput(outStr, originuuid, idx) + if migUUID == "" { + return "", fmt.Errorf("failed to resolve MIG UUID for device %s at index %d via nvidia-smi fallback", originuuid, idx) + } + return migUUID, nil } res, ret := migdev.GetUUID() if ret != nvml.SUCCESS { klog.Error(`nvml get mig uuid error ret=`, ret) panic(0) } - return res + return res, nil } func GetMigGpuInstanceIdFromIndex(uuid string, idx int) (int, error) { @@ -467,7 +470,7 @@ func deepCopyMigConfig(src nvidia.MigConfigSpec) nvidia.MigConfigSpec { return dst } -func (nv *NvidiaDevicePlugin) GetContainerDeviceStrArray(c device.ContainerDevices) []string { +func (nv *NvidiaDevicePlugin) GetContainerDeviceStrArray(c device.ContainerDevices) ([]string, error) { tmp := []string{} needsreset := false position := 0 @@ -510,16 +513,15 @@ func (nv *NvidiaDevicePlugin) GetContainerDeviceStrArray(c device.ContainerDevic } } } - migUUID := GetMigUUIDFromIndex(val.UUID, position) - if migUUID == "" { - klog.Errorf("failed to resolve MIG UUID for %s at position %d, skipping", val.UUID, position) - continue + migUUID, err := GetMigUUIDFromIndex(val.UUID, position) + if err != nil { + return nil, fmt.Errorf("failed to resolve MIG UUID for %s at position %d: %w", val.UUID, position, err) } tmp = append(tmp, migUUID) } } klog.V(3).Infoln("mig current=", nv.migCurrent, ":", needsreset, "position=", position, "uuid lists", tmp) - return tmp + return tmp, nil } var podAllocationTrySuccess = func(nodeName string, devName string, lockName string, pod *corev1.Pod) {