Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions charts/hami/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ This document provides detailed descriptions of all configurable values paramete
|-----------|-------------|---------------|
| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` |
| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` |
| `devicePlugin.hostPID` | Use the host PID namespace for the device plugin | `true` |
| `devicePlugin.hostPIDBroker.enabled` | Let HAMi core ask the device plugin for its host PID. This requires `devicePlugin.hostPID`. See [Host PID broker](../../docs/develop/hostpid-broker.md) | `false` |
| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` |
| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` |
| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` |
Expand Down
17 changes: 17 additions & 0 deletions charts/hami/templates/device-plugin/daemonsetnvidia.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.hostPIDBroker.enabled (not .Values.devicePlugin.hostPID) }}
{{- fail "devicePlugin.hostPIDBroker requires devicePlugin.hostPID" }}
{{- end }}
{{- if .Values.devicePlugin.enabled }}
apiVersion: apps/v1
kind: DaemonSet
Expand Down Expand Up @@ -94,6 +97,10 @@ spec:
value: {{ .Values.devicePlugin.deviceListStrategy }}
- name: HOOK_PATH
value: {{ .Values.global.gpuHookPath }}
{{- if .Values.devicePlugin.hostPIDBroker.enabled }}
- name: LIBVGPU_HOSTPID_BROKER
value: "1"
{{- end }}
{{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }}
- name: PASS_DEVICE_SPECS
value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }}
Expand Down Expand Up @@ -169,6 +176,10 @@ spec:
mountPath: /etc/hami/numa-refit-ca
readOnly: true
{{- end }}
{{- if .Values.devicePlugin.hostPIDBroker.enabled }}
- name: hostpid-broker
mountPath: /var/run/hami/hostpid
{{- end }}
{{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }}
# We always mount the driver root at /driver-root in the container.
# This is required for CDI detection to work correctly.
Expand Down Expand Up @@ -266,6 +277,12 @@ spec:
hostPath:
path: /var/run/cdi
type: DirectoryOrCreate
{{- if .Values.devicePlugin.hostPIDBroker.enabled }}
- name: hostpid-broker
hostPath:
path: /var/run/hami/hostpid
type: DirectoryOrCreate
{{- end }}
- name: usrbin
hostPath:
path: /usr/bin
Expand Down
4 changes: 4 additions & 0 deletions charts/hami/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,10 @@ devicePlugin:

podAnnotations: {}
hostPID: true
# Let HAMi core ask the device plugin for its host PID.
# This requires hostPID to be true.
hostPIDBroker:
enabled: false
hostNetwork: false
securityContext:
privileged: true
Expand Down
78 changes: 78 additions & 0 deletions cmd/device-plugin/nvidia/hostpid_broker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Copyright (c) 2026 The HAMi Authors.
*/

package main

import (
"errors"
"fmt"
"os"

"k8s.io/klog/v2"

"github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/hostpid"
)

type runningHostPIDBroker struct {
broker hostPIDBroker
done chan struct{}
serveErr error
}

type hostPIDBroker interface {
Serve() error
Close() error
}

type hostPIDBrokerListener func() (hostPIDBroker, error)

func startHostPIDBroker() (*runningHostPIDBroker, error) {
return startHostPIDBrokerWithListener(func() (hostPIDBroker, error) {
return hostpid.ListenDefault()
})
}

func startHostPIDBrokerWithListener(
listen hostPIDBrokerListener) (*runningHostPIDBroker, error) {
if !hostpid.Enabled(os.Getenv(hostpid.EnvironmentVariable)) {
return nil, nil
}
broker, err := listen()
if err != nil {
return nil, err
}
running := &runningHostPIDBroker{
broker: broker,
done: make(chan struct{}),
}
go func() {
running.serveErr = broker.Serve()
close(running.done)
}()
klog.Infof("Host PID broker is listening on %s", hostpid.ServerSocketPath)
return running, nil
}

func (running *runningHostPIDBroker) stop() error {
closeErr := running.broker.Close()
<-running.done
return closeErr
}

func (running *runningHostPIDBroker) failure() error {
if running == nil {
return nil
}
select {
case <-running.done:
if running.serveErr != nil {
return fmt.Errorf("host PID broker stopped: %w", running.serveErr)
}
return errors.New("host PID broker stopped unexpectedly")
default:
return nil
}
}
130 changes: 130 additions & 0 deletions cmd/device-plugin/nvidia/hostpid_broker_lifecycle_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Copyright (c) 2026 The HAMi Authors.
*/

package main

import (
"errors"
"testing"

"github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/plugin"
"github.com/Project-HAMi/HAMi/pkg/device-plugin/nvidiadevice/nvinternal/rm"
)

type fakeDevicePlugin struct {
devices rm.Devices
start func(string) error
stopErr error
startCalls int
stopCalls int
}

func (p *fakeDevicePlugin) Devices() rm.Devices {
return p.devices
}

func (p *fakeDevicePlugin) Start(socket string) error {
p.startCalls++
if p.start != nil {
return p.start(socket)
}
return nil
}

func (p *fakeDevicePlugin) Stop() error {
p.stopCalls++
return p.stopErr
}

func devicePluginWithDevice() *fakeDevicePlugin {
return &fakeDevicePlugin{
devices: rm.Devices{"GPU-0": &rm.Device{}},
}
}

func TestStartPluginServersDetectsBrokerFailureBeforeStart(t *testing.T) {
wantErr := errors.New("broker failed")
done := make(chan struct{})
close(done)
running := &runningHostPIDBroker{
done: done,
serveErr: wantErr,
}
p := devicePluginWithDevice()

started, restart, err := startPluginServers(
[]plugin.Interface{p}, "/tmp/kubelet.sock", running)
if started != 0 || restart || !errors.Is(err, wantErr) {
t.Fatalf("started=%d restart=%v err=%v, want broker failure",
started, restart, err)
}
if p.startCalls != 0 || p.stopCalls != 0 {
t.Fatalf("start calls=%d stop calls=%d, want 0 and 0",
p.startCalls, p.stopCalls)
}
}

func TestStartPluginServersCleansUpAfterBrokerFailure(t *testing.T) {
wantServeErr := errors.New("broker failed")
wantStopErr := errors.New("plugin stop failed")
done := make(chan struct{})
running := &runningHostPIDBroker{done: done}
p := devicePluginWithDevice()
p.stopErr = wantStopErr
p.start = func(string) error {
running.serveErr = wantServeErr
close(done)
return nil
}

started, restart, err := startPluginServers(
[]plugin.Interface{p}, "/tmp/kubelet.sock", running)
if started != 0 || restart || !errors.Is(err, wantServeErr) ||
!errors.Is(err, wantStopErr) {
t.Fatalf("started=%d restart=%v err=%v, want joined failures",
started, restart, err)
}
if p.startCalls != 1 || p.stopCalls != 1 {
t.Fatalf("start calls=%d stop calls=%d, want 1 and 1",
p.startCalls, p.stopCalls)
}
}

func TestStartPluginServersRequestsRestartAfterStartFailure(t *testing.T) {
wantErr := errors.New("plugin start failed")
p := devicePluginWithDevice()
p.start = func(socket string) error {
if socket != "/tmp/kubelet.sock" {
t.Fatalf("socket=%q", socket)
}
return wantErr
}

started, restart, err := startPluginServers(
[]plugin.Interface{p}, "/tmp/kubelet.sock", nil)
if started != 0 || !restart || err != nil {
t.Fatalf("started=%d restart=%v err=%v", started, restart, err)
}
if p.startCalls != 1 || p.stopCalls != 0 {
t.Fatalf("start calls=%d stop calls=%d, want 1 and 0",
p.startCalls, p.stopCalls)
}
}

func TestStartPluginServersSkipsEmptyPlugins(t *testing.T) {
empty := &fakeDevicePlugin{}
ready := devicePluginWithDevice()

started, restart, err := startPluginServers(
[]plugin.Interface{empty, ready}, "/tmp/kubelet.sock", nil)
if started != 1 || restart || err != nil {
t.Fatalf("started=%d restart=%v err=%v", started, restart, err)
}
if empty.startCalls != 0 || ready.startCalls != 1 {
t.Fatalf("empty starts=%d ready starts=%d, want 0 and 1",
empty.startCalls, ready.startCalls)
}
}
Loading
Loading