Skip to content

chore: improve error handling in scheduler routes and device plugin #2279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
shinigami-777 wants to merge 3 commits into from
+52 −17
Closed

chore: improve error handling in scheduler routes and device plugin #2279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4ac24e1
chore: improve error handling in scheduler routes and device plugin
shinigami-777 Aug 2, 2026
ed3ffb5
chore: improve error handling
shinigami-777 Aug 2, 2026
2df4ad3
test: fix permission denied errors in Allocate tests
shinigami-777 Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,15 +312,19 @@ func (plugin *NvidiaDevicePlugin) Start(kubeletSocket string) error {
klog.Warning("Falling back to non‑MIG configuration")
} else {
outStr := stdout.Bytes()
yaml.Unmarshal(outStr, &plugin.migCurrent)
writeMigConfig(outStr)

HamiInitMigConfig, err := plugin.processMigConfigs(plugin.migCurrent.MigConfigs, deviceNumbers)
if err != nil {
klog.Infof("no device in node: %v", err)
if err := yaml.Unmarshal(outStr, &plugin.migCurrent); err != nil {
klog.Errorf("failed to unmarshal mig config: %v", err)

@mesutoezdil mesutoezdil Aug 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

logging alone is not enough, migCurrent stays zero and the next lines still run w/ it, pls fall back to non mig like the sibling branch above

@shinigami-777 shinigami-777 Aug 2, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, made it similar to the sibling branch above it.

klog.Warning("Falling back to non-MIG configuration")
} else {
plugin.migCurrent.MigConfigs["current"] = HamiInitMigConfig
migApplied = true
writeMigConfig(outStr)

HamiInitMigConfig, err := plugin.processMigConfigs(plugin.migCurrent.MigConfigs, deviceNumbers)
if err != nil {
klog.Infof("no device in node: %v", err)
} else {
plugin.migCurrent.MigConfigs["current"] = HamiInitMigConfig
migApplied = true
}
}
}
}
Expand Down Expand Up @@ -673,12 +677,27 @@ func (plugin *NvidiaDevicePlugin) Allocate(ctx context.Context, reqs *kubeletdev
response.Envs[util.CoreLimitSwitch] = "disable"
}
cacheFileHostDirectory := fmt.Sprintf("%s/vgpu/containers/%s_%s", hostHookPath, current.UID, currentCtr.Name)
os.RemoveAll(cacheFileHostDirectory)
if err := os.RemoveAll(cacheFileHostDirectory); err != nil {
PodAllocationFailed(nodename, current, NodeLockNvidia)
return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("failed to remove cache directory: %v", err)
}

os.MkdirAll(cacheFileHostDirectory, 0777)
os.Chmod(cacheFileHostDirectory, 0777)
os.MkdirAll("/tmp/vgpulock", 0777)
os.Chmod("/tmp/vgpulock", 0777)
if err := os.MkdirAll(cacheFileHostDirectory, 0777); err != nil {
PodAllocationFailed(nodename, current, NodeLockNvidia)
return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("failed to create cache directory: %v", err)
}
if err := os.Chmod(cacheFileHostDirectory, 0777); err != nil {
PodAllocationFailed(nodename, current, NodeLockNvidia)
return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("failed to chmod cache directory: %v", err)
}
if err := os.MkdirAll("/tmp/vgpulock", 0777); err != nil {
PodAllocationFailed(nodename, current, NodeLockNvidia)
return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("failed to create lock directory: %v", err)
}
Comment thread
shinigami-777 marked this conversation as resolved.
if err := os.Chmod("/tmp/vgpulock", 0777); err != nil {
PodAllocationFailed(nodename, current, NodeLockNvidia)
return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("failed to chmod lock directory: %v", err)
}
response.Mounts = append(response.Mounts,
&kubeletdevicepluginv1beta1.Mount{ContainerPath: fmt.Sprintf("%s/vgpu/libvgpu.so", hostHookPath),
HostPath: GetLibPath(),
Expand Down
8 changes: 8 additions & 0 deletions pkg/device-plugin/nvidiadevice/nvinternal/plugin/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -964,6 +964,10 @@ func TestAlignContainerDevicesWithAllocatedIDsRejectsLengthMismatch(t *testing.T
}

func TestAllocateUsesKubeletSelectedUUIDsForVGPUResponse(t *testing.T) {
previousHostHookPath := hostHookPath
hostHookPath = t.TempDir()
defer func() { hostHookPath = previousHostHookPath }()

deviceListStrategies, _ := v1.NewDeviceListStrategies([]string{"envvar"})
deviceIDStrategy := v1.DeviceIDStrategyUUID
memScale := 1.0
Expand Down Expand Up @@ -1036,6 +1040,10 @@ func TestAllocateUsesKubeletSelectedUUIDsForVGPUResponse(t *testing.T) {
}

func TestAllocatePreservesContainerOrderWhenOneContainerFallsBack(t *testing.T) {
previousHostHookPath := hostHookPath
hostHookPath = t.TempDir()
defer func() { hostHookPath = previousHostHookPath }()

deviceListStrategies, _ := v1.NewDeviceListStrategies([]string{"envvar"})
deviceIDStrategy := v1.DeviceIDStrategyUUID
memScale := 1.0
Expand Down
16 changes: 12 additions & 4 deletions pkg/scheduler/routes/route.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,15 @@ func PredicateRoute(s *scheduler.Scheduler) httprouter.Handle {
Error: fmt.Sprintf("Failed to marshal extender filter result: %s", err.Error()),
}
resultBody, _ = json.Marshal(extenderFilterResult)
w.Write(resultBody)
if _, err := w.Write(resultBody); err != nil {
klog.ErrorS(err, "Failed to write response")
}
} else {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(resultBody)
if _, err := w.Write(resultBody); err != nil {
klog.ErrorS(err, "Failed to write response")
}
}
}
}
Expand Down Expand Up @@ -128,12 +132,16 @@ func Bind(s *scheduler.Scheduler) httprouter.Handle {
Error: fmt.Sprintf("Failed to marshal binding result: %s", err.Error()),
}
response, _ := json.Marshal(extenderBindingResult)
w.Write(response)
if _, err := w.Write(response); err != nil {
klog.ErrorS(err, "Failed to write response")
}
} else {
klog.V(5).InfoS("Returning bind response", "result", extenderBindingResult)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(response)
if _, err := w.Write(response); err != nil {
klog.ErrorS(err, "Failed to write response")
}
}
}
}
Expand Down
Loading