Skip to content

fix(device-plugin): auto-set CUDA_DISABLE_CONTROL for whole-GPU allocations #2664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Rickydama3 wants to merge 6 commits into from
+131 −9
Closed

fix(device-plugin): auto-set CUDA_DISABLE_CONTROL for whole-GPU allocations #2664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
52f57a0
fix(device-plugin): auto-set CUDA_DISABLE_CONTROL for whole-GPU alloc…
Rickydama3 Aug 13, 2026
b4b24f5
fix(device-plugin): preserve explicit CUDA_DISABLE_CONTROL value
Rickydama3 Aug 13, 2026
8b46f63
fix(device-plugin): check SM limits for every allocated device
Rickydama3 Aug 14, 2026
9fd7e97
chore: trigger CI re-run for flaky helm-deploy timeout
Rickydama3 Aug 14, 2026
5c29744
chore: trigger CI re-run
Rickydama3 Aug 17, 2026
89d1051
fix(device-plugin): check SM limits for every allocated device
Rickydama3 Aug 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 117 additions & 0 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/alloc_refactor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,123 @@ func TestAllocate_MultiContainer_CUDA_DISABLE_CONTROL_FirstContainer(t *testing.
"container 1 should have ld.so.preload mounted")
}


func TestAllocate_WholeGPU_AutoSets_CUDA_DISABLE_CONTROL(t *testing.T) {
setupInRequestDevices(t)
plugin := newTestPlugin(t)

pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
UID: "pod-uid",
Annotations: map[string]string{
"hami.io/vgpu-devices-to-allocate": "GPU-aaa,NVIDIA,0,100:;",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{Name: "c0"}, // No explicit CUDA_DISABLE_CONTROL
},
},
}
setupFakeClient(t, pod)
mockAllocateGlobals(t, pod)

request := &kubeletdevicepluginv1beta1.AllocateRequest{
ContainerRequests: []*kubeletdevicepluginv1beta1.ContainerAllocateRequest{
{DevicesIds: []string{"GPU-aaa-0"}},
},
}

response, err := plugin.Allocate(context.Background(), request)
require.NoError(t, err)
require.Len(t, response.ContainerResponses, 1)

require.Equal(t, "true", response.ContainerResponses[0].Envs["CUDA_DISABLE_CONTROL"],
"CUDA_DISABLE_CONTROL should be auto-set to true for whole-GPU allocation")
require.False(t, hasLdSoPreloadMount(response.ContainerResponses[0].Mounts),
"ld.so.preload should NOT be mounted for whole-GPU allocation")
}

func TestAllocate_WholeGPU_Explicit_False_Preserves_Mount(t *testing.T) {
setupInRequestDevices(t)
plugin := newTestPlugin(t)

pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
UID: "pod-uid",
Annotations: map[string]string{
"hami.io/vgpu-devices-to-allocate": "GPU-aaa,NVIDIA,0,100:;",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{Name: "c0", Env: []corev1.EnvVar{
{Name: "CUDA_DISABLE_CONTROL", Value: "false"},
}},
},
},
}
setupFakeClient(t, pod)
mockAllocateGlobals(t, pod)

request := &kubeletdevicepluginv1beta1.AllocateRequest{
ContainerRequests: []*kubeletdevicepluginv1beta1.ContainerAllocateRequest{
{DevicesIds: []string{"GPU-aaa-0"}},
},
}

response, err := plugin.Allocate(context.Background(), request)
require.NoError(t, err)
require.Len(t, response.ContainerResponses, 1)

_, hasControl := response.ContainerResponses[0].Envs["CUDA_DISABLE_CONTROL"]
require.False(t, hasControl, "CUDA_DISABLE_CONTROL should not be auto-set if explicitly false")
require.True(t, hasLdSoPreloadMount(response.ContainerResponses[0].Mounts),
"ld.so.preload should be mounted since CUDA_DISABLE_CONTROL is explicitly false")
}

func TestAllocate_WholeGPU_MultiDevice_NonZeroSMOnSecondDevice(t *testing.T) {
setupInRequestDevices(t)
plugin := newTestPlugin(t)

pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
UID: "pod-uid",
Annotations: map[string]string{
"hami.io/vgpu-devices-to-allocate": "GPU-aaa,NVIDIA,0,100:GPU-bbb,NVIDIA,0,50:;",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{Name: "c0"},
},
},
}
setupFakeClient(t, pod)
mockAllocateGlobals(t, pod)

request := &kubeletdevicepluginv1beta1.AllocateRequest{
ContainerRequests: []*kubeletdevicepluginv1beta1.ContainerAllocateRequest{
{DevicesIds: []string{"GPU-aaa-0", "GPU-bbb-0"}},
},
}

response, err := plugin.Allocate(context.Background(), request)
require.NoError(t, err)
require.Len(t, response.ContainerResponses, 1)

_, hasControl := response.ContainerResponses[0].Envs["CUDA_DISABLE_CONTROL"]
require.False(t, hasControl, "CUDA_DISABLE_CONTROL should not be auto-set if any device is not whole-GPU")
require.True(t, hasLdSoPreloadMount(response.ContainerResponses[0].Mounts),
"ld.so.preload should be mounted if any device is not whole-GPU")
}

func TestAllocate_DeviceNumberMismatch(t *testing.T) {
setupInRequestDevices(t)
plugin := newTestPlugin(t)
Expand Down
23 changes: 14 additions & 9 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -820,9 +820,13 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev
}

if plugin.operatingMode != "mig" {
isWholeGPU := true
for i, dev := range devreq {
limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i)
response.Envs[limitKey] = fmt.Sprintf("%vm", dev.Usedmem)
if dev.Usedcores != 100 {
isWholeGPU = false
}
}
response.Envs["CUDA_DEVICE_SM_LIMIT"] = fmt.Sprint(devreq[0].Usedcores)
response.Envs["CUDA_DEVICE_MEMORY_SHARED_CACHE"] = fmt.Sprintf("%s/vgpu/%v.cache", hostHookPath, uuid.New().String())
Expand Down Expand Up @@ -853,20 +857,21 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev
HostPath: "/tmp/vgpulock",
ReadOnly: false},
)
found := false
hasControlSetting := false
controlDisabled := false
for _, val := range currentCtr.Env {
if strings.Compare(val.Name, "CUDA_DISABLE_CONTROL") == 0 {
// if env existed but is set to false or can not be parsed, ignore
t, _ := strconv.ParseBool(val.Value)
if !t {
continue
}
// only env existed and set to true, we mark it "found"
found = true
hasControlSetting = true
controlDisabled, _ = strconv.ParseBool(val.Value)
break
}
}
if !found {
if isWholeGPU && !hasControlSetting {

@mesutoezdil mesutoezdil Aug 18, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

with control off the pod stops writing the vgpu cache, so vgpumonitor loses metrics for it. this silently changes observability for every count-only pod, not just the vllm case.

@Rickydama3 Rickydama3 Aug 18, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since auto-setting this breaks vgpumonitor metrics for all exclusive pods, should we just drop this logic entirely and advise vLLM users to manually set CUDA_DISABLE_CONTROL="true" instead?

klog.Infof("Whole GPU allocation detected without explicit CUDA_DISABLE_CONTROL, auto-setting to true")
controlDisabled = true
response.Envs["CUDA_DISABLE_CONTROL"] = "true"
}
if !controlDisabled {
response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{ContainerPath: "/etc/ld.so.preload",
HostPath: hostHookPath + "/vgpu/ld.so.preload",
ReadOnly: true},
Expand Down
Loading