Repository navigation
fix(device-plugin): auto-set CUDA_DISABLE_CONTROL for whole-GPU allocations #2664
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
52f57a0
b4b24f5
8b46f63
9fd7e97
5c29744
89d1051
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -820,11 +820,18 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev | |
| } | ||
|
|
||
| if plugin.operatingMode != "mig" { | ||
| isWholeGPU := true | ||
| for i, dev := range devreq { | ||
| limitKey := fmt.Sprintf("CUDA_DEVICE_MEMORY_LIMIT_%v", i) | ||
| response.Envs[limitKey] = fmt.Sprintf("%vm", dev.Usedmem) | ||
| if dev.Usedmem != 0 { | ||
| isWholeGPU = false | ||
| } | ||
| } | ||
| response.Envs["CUDA_DEVICE_SM_LIMIT"] = fmt.Sprint(devreq[0].Usedcores) | ||
| if devreq[0].Usedcores != 0 { | ||
| isWholeGPU = false | ||
| } | ||
| response.Envs["CUDA_DEVICE_MEMORY_SHARED_CACHE"] = fmt.Sprintf("%s/vgpu/%v.cache", hostHookPath, uuid.New().String()) | ||
| if *plugin.schedulerConfig.DeviceMemoryScaling > 1 { | ||
| response.Envs["CUDA_OVERSUBSCRIBE"] = "true" | ||
|
|
@@ -853,20 +860,21 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev | |
| HostPath: "/tmp/vgpulock", | ||
| ReadOnly: false}, | ||
| ) | ||
| found := false | ||
| hasControlSetting := false | ||
| controlDisabled := false | ||
| for _, val := range currentCtr.Env { | ||
| if strings.Compare(val.Name, "CUDA_DISABLE_CONTROL") == 0 { | ||
| // if env existed but is set to false or can not be parsed, ignore | ||
| t, _ := strconv.ParseBool(val.Value) | ||
| if !t { | ||
| continue | ||
| } | ||
| // only env existed and set to true, we mark it "found" | ||
| found = true | ||
| hasControlSetting = true | ||
| controlDisabled, _ = strconv.ParseBool(val.Value) | ||
| break | ||
| } | ||
| } | ||
| if !found { | ||
| if isWholeGPU && !hasControlSetting { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. with control off the pod stops writing the vgpu cache, so vgpumonitor loses metrics for it. this silently changes observability for every count-only pod, not just the vllm case.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since auto-setting this breaks vgpumonitor metrics for all exclusive pods, should we just drop this logic entirely and advise vLLM users to manually set CUDA_DISABLE_CONTROL="true" instead? |
||
| klog.Infof("Whole GPU allocation detected without explicit CUDA_DISABLE_CONTROL, auto-setting to true") | ||
| controlDisabled = true | ||
| response.Envs["CUDA_DISABLE_CONTROL"] = "true" | ||
| } | ||
| if !controlDisabled { | ||
| response.Mounts = append(response.Mounts, &kubeletdevicepluginv1beta1.Mount{ContainerPath: "/etc/ld.so.preload", | ||
| HostPath: hostHookPath + "/vgpu/ld.so.preload", | ||
| ReadOnly: true}, | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.