diff --git a/charts/hami/templates/scheduler/device-configmap.yaml b/charts/hami/templates/scheduler/device-configmap.yaml index 1335115bc6..9493783215 100644 --- a/charts/hami/templates/scheduler/device-configmap.yaml +++ b/charts/hami/templates/scheduler/device-configmap.yaml @@ -22,6 +22,20 @@ data: defaultCores: 0 defaultGPUNum: 1 preConfiguredDeviceMemory: {{ .Values.devicePlugin.preConfiguredDeviceMemory | default 0 }} + # memoryFactor controls the granularity of GPU memory requests. + # The device plugin advertises ceil(totalMemMiB / memoryFactor) virtual + # entries per GPU. If total entries across all GPUs exceeds ~60 000 + # (the kubelet gRPC 4 MB message limit), kubelet rejects the + # ListAndWatch response and vgpu-memory shows 0 or a stale value. + # + # Minimum safe value: + # memoryFactor >= ceil(totalMemMiB / floor(60000 / gpuCount)) + # + # Examples: + # A100/A800 80 GB × 1 GPU → factor >= 2 + # A100/A800 80 GB × 8 GPUs → factor >= 11 + # + # See docs/gpu-memory-factor.md for the full sizing table and formula. memoryFactor: 1 deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} diff --git a/docs/gpu-memory-factor.md b/docs/gpu-memory-factor.md new file mode 100644 index 0000000000..fc09fce600 --- /dev/null +++ b/docs/gpu-memory-factor.md @@ -0,0 +1,231 @@ +# gpuMemoryFactor: Configuration and Sizing Guide + +## Overview + +`memoryFactor` is an integer multiplier that controls how memory requests are +interpreted by the HAMi scheduler and device plugin. When a pod requests +`N MiB` of GPU memory, the scheduler multiplies that value by `memoryFactor` +before comparing it against the device's available memory. This lets operators +express memory requests in coarser granularities (e.g. multiples of 2 MiB +instead of 1 MiB) without changing every pod spec. + +**Default value:** `1` (no scaling; requests are taken at face value). + +### Where to set it + +`memoryFactor` lives in the scheduler's `device-config.yaml`, under the +`nvidia:` section: + +```yaml +# charts/hami/templates/scheduler/device-configmap.yaml (or your own override) +nvidia: + memoryFactor: 1 # <- change this value +``` + +It can also be set directly in a `device-config.yaml` file mounted into the +scheduler pod. + +--- + +## The kubelet gRPC 4 MB limit + +The Kubernetes device plugin protocol requires the device plugin to stream a +list of all virtual device IDs to kubelet via the `ListAndWatch` gRPC call. +For NVIDIA GPUs, HAMi advertises one entry per MiB of GPU memory divided by +`memoryFactor`: + +```text +entries_per_gpu = ceil(totalMemMiB / memoryFactor) +total_entries = entries_per_gpu × gpuCount +``` + +The kubelet gRPC server enforces a hard message-size limit of roughly **4 MB**. +Each device-plugin entry is approximately 64 bytes, so the effective ceiling is +around **60 000 entries** in a single `ListAndWatch` response. + +When `total_entries > 60 000`, kubelet silently rejects the response. The +device plugin does **not** receive an explicit error from kubelet in the +protocol stream; the connection drops and is re-established, but the next +message is also too large. The observable symptoms are: + +- `volcano.sh/vgpu-memory` (or `nvidia.com/gpumem`) shows **0** or a **stale** + value in node allocatable resources. +- Pods that request GPU memory are never scheduled. + +Starting from the fix for issue #2187, HAMi now: + +1. **Logs a Warning at runtime** from `GetPluginDevices` whenever the generated + entry count exceeds 60 000, including the exact minimum safe `memoryFactor` + computed from the actual GPU memory and count detected on the node. +2. **Checks and returns the `s.Send` error** in `ListAndWatch`. If kubelet + rejects the response (e.g. `ResourceExhausted: received message larger than + max`), the error is logged with an actionable message and the stream is + closed gracefully so kubelet can reconnect. + +--- + +## Sizing formula + +The **runtime check** computes total entries directly: + +```text +estimatedEntries = ceil(totalMemMiB / memoryFactor) × gpuCount +``` + +The **minimum safe factor** is derived by rearranging the constraint +`estimatedEntries ≤ 60 000`: + +```text +minFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) +``` + +Use this formula to find the smallest `memoryFactor` that keeps `total_entries` +within the kubelet limit. + +### Reference table — common data-centre GPUs + +| GPU model | Memory (GiB) | totalMemMiB | 1 GPU min factor | 8 GPU min factor | +| :----------------- | :----------: | :---------: | :--------------: | :--------------: | +| NVIDIA T4 | 16 | 16 384 | 1 | 3 | +| NVIDIA A10 | 24 | 24 576 | 1 | 4 | +| NVIDIA A30 | 24 | 24 576 | 1 | 4 | +| NVIDIA A100 40 GB | 40 | 40 960 | 1 | 6 | +| NVIDIA A100 80 GB | 80 | 81 920 | 2 | 11 | +| NVIDIA A800 80 GB | 80 | 81 920 | 2 | 11 | +| NVIDIA H100 80 GB | 80 | 81 920 | 2 | 11 | +| NVIDIA H100 94 GB | 94 | 96 256 | 2 | 13 | +| NVIDIA H200 141 GB | 141 | 144 384 | 3 | 20 | +| NVIDIA B200 180 GB | 180 | 184 320 | 4 | 25 | + +> Min factor values are `ceil(totalMemMiB / floor(60000 / gpuCount))`. +> Round up to the next integer when your cluster has more GPUs per node. + +### Single-GPU examples + +**A800 (80 GiB), 1 GPU, factor=1 — broken:** + +```text +entries = ceil(81920 / 1) × 1 = 81920 # exceeds 60000 → kubelet rejects +``` + +**A800 (80 GiB), 1 GPU, factor=2 — correct:** + +```text +entries = ceil(81920 / 2) × 1 = 40960 # safe ✓ +minFactor = ceil(81920 / floor(60000 / 1)) = ceil(81920 / 60000) = 2 +``` + +### Multi-GPU examples + +**A800 (80 GiB), 8 GPUs, factor=2 — broken:** + +```text +entries = ceil(81920 / 2) × 8 = 40960 × 8 = 327680 # exceeds 60000 → rejects +``` + +**A800 (80 GiB), 8 GPUs, factor=11 — correct:** + +```text +entries = ceil(81920 / 11) × 8 = 7448 × 8 = 59584 # safe ✓ +minFactor = ceil(81920 / floor(60000 / 8)) + = ceil(81920 / 7500) + = ceil(10.92) = 11 +``` + +--- + +## Runtime validation + +Validation runs at **device plugin startup**, inside `GetPluginDevices`, once +the plugin has queried the hardware. This means the warning is based on actual +GPU memory rather than a configured estimate. Look for these log lines in the +device plugin pod: + +```sh +kubectl logs -n +``` + +**Warning logged when entry count exceeds 60 000:** + +```text +W ... GetPluginDevices: entry count 81920 exceeds the kubelet gRPC message + limit of ~60000 entries (~4 MB). kubelet will reject the ListAndWatch + response and vgpu-memory will show 0 or a stale value. + gpuCount=1, splitCount=81920, maxGPUMemMiB=81920. + Minimum safe memoryFactor = ceil(81920 / floor(60000 / 1)) = 2. +``` + +**Error logged when kubelet actually rejects the send:** + +```text +E ... ListAndWatch: failed to send initial device list (81920 entries) for + resource 'nvidia.com/gpumem': rpc error: code = ResourceExhausted ... + If the error is 'ResourceExhausted' or 'grpc: received message larger + than max', increase memoryFactor to at least + ceil(totalMemMiB / floor(60000 / gpuCount)); + see the preceding GetPluginDevices warning for the exact value. +``` + +If you see either message, increase `memoryFactor` according to the table +above and restart the scheduler pod. + +--- + +## Side-effects of a non-unity factor + +When `memoryFactor > 1`, pod memory requests are interpreted as **logical MiB** +that are multiplied by `memoryFactor` before scheduling: + +```text +scheduled_memory_MiB = request_MiB × memoryFactor +``` + +For example, if a pod requests `1024 MiB` and `memoryFactor=2`, HAMi allocates +`2048 MiB` on the physical GPU. This means: + +- Memory requests in pod specs are expressed in units of `memoryFactor` MiB. + Document this convention for your users. +- The minimum allocatable memory per container is `memoryFactor` MiB (one + virtual entry). +- `deviceMemoryScaling` is applied **before** the factor and operates on the + physical device's reported memory. + +--- + +## Choosing the right value + +1. Find your GPU's total memory in MiB: `totalMemMiB = GiB × 1024`. +2. Count the maximum number of GPUs per node: `gpuCount`. +3. Apply the formula: + + ```text + memoryFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) + ``` + +4. If the result is 1, keep the default. Otherwise set `memoryFactor` in + `device-config.yaml` and redeploy the scheduler. + +--- + +## Helm configuration + +`memoryFactor` is not exposed as a top-level Helm value because it depends on +the GPU model deployed in the cluster. Override it via a `device-config.yaml` +file or by patching the scheduler ConfigMap directly: + +```yaml +# Example: values override snippet +scheduler: + # Mount a custom device-config.yaml that sets memoryFactor: 2 + extraVolumes: + - name: device-config + configMap: + name: my-device-config + extraVolumeMounts: + - name: device-config + mountPath: /config +``` + +Or, if you supply a `files/device-config.yaml` in your Helm chart overlay, the +template will use it automatically (see the comment in +`charts/hami/templates/scheduler/device-configmap.yaml`). diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go index 8a3890ead5..8e57edc3d4 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/plugin/server.go @@ -454,7 +454,22 @@ func (plugin *NvidiaDevicePlugin) GetDevicePluginOptions(context.Context, *kubel // ListAndWatch lists devices and update that list according to the health status func (plugin *NvidiaDevicePlugin) ListAndWatch(e *kubeletdevicepluginv1beta1.Empty, s kubeletdevicepluginv1beta1.DevicePlugin_ListAndWatchServer) error { - s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: plugin.apiDevices()}) + devices := plugin.apiDevices() + klog.Infof("ListAndWatch: sending initial device list with %d entries for resource '%s'", len(devices), plugin.rm.Resource()) + if err := s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: devices}); err != nil { + // A gRPC ResourceExhausted error here almost always means the response + // exceeded the kubelet ~4 MB message limit. This happens when: + // ceil(totalMemMiB / memoryFactor) × gpuCount > 60 000 + // Increase memoryFactor to at least: + // ceil(totalMemMiB / floor(60000 / gpuCount)) + // GetPluginDevices logs the exact minFactor value for this node. + klog.Errorf("ListAndWatch: failed to send initial device list (%d entries) for resource '%s': %v. "+ + "If the error is 'ResourceExhausted' or 'grpc: received message larger than max', "+ + "increase memoryFactor to at least ceil(totalMemMiB / floor(60000 / gpuCount)); "+ + "see the preceding GetPluginDevices warning for the exact value.", + len(devices), plugin.rm.Resource(), err) + return err + } for { select { @@ -464,7 +479,15 @@ func (plugin *NvidiaDevicePlugin) ListAndWatch(e *kubeletdevicepluginv1beta1.Emp // FIXME: there is no way to recover from the Unhealthy state. d.Health = kubeletdevicepluginv1beta1.Unhealthy klog.Infof("'%s' device marked unhealthy: %s", plugin.rm.Resource(), d.ID) - s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: plugin.apiDevices()}) + updated := plugin.apiDevices() + klog.V(4).Infof("ListAndWatch: sending updated device list with %d entries for resource '%s'", len(updated), plugin.rm.Resource()) + if err := s.Send(&kubeletdevicepluginv1beta1.ListAndWatchResponse{Devices: updated}); err != nil { + klog.Errorf("ListAndWatch: failed to send updated device list (%d entries) for resource '%s': %v. "+ + "If the error is 'ResourceExhausted', increase memoryFactor; "+ + "see docs/gpu-memory-factor.md for the sizing formula.", + len(updated), plugin.rm.Resource(), err) + return err + } } } } diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go index 2c9c3b1447..72186675de 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices.go @@ -191,6 +191,11 @@ func (ds Devices) GetUUIDs() []string { return res } +// kubeletListAndWatchMaxEntries is the conservative upper bound on the number +// of kubelet Device entries that fit within the gRPC ~4 MB message limit. +// Each Device proto is approximately 64 bytes; 60 000 × 64 B ≈ 3.84 MB. +const kubeletListAndWatchMaxEntries = 60_000 + // GetPluginDevices returns the plugin Devices from all devices in the Devices func (ds Devices) GetPluginDevices(count uint, numaTopology bool) []*kubeletdevicepluginv1beta1.Device { var res []*kubeletdevicepluginv1beta1.Device @@ -199,7 +204,8 @@ func (ds Devices) GetPluginDevices(count uint, numaTopology bool) []*kubeletdevi return res } - if !strings.Contains(ds.GetIDs()[0], "MIG") { + isMIG := strings.Contains(ds.GetIDs()[0], "MIG") + if !isMIG { for _, dev := range ds { topology := dev.Topology if !numaTopology { @@ -218,7 +224,44 @@ func (ds Devices) GetPluginDevices(count uint, numaTopology bool) []*kubeletdevi for _, d := range ds { res = append(res, &d.Device) } + } + + total := len(res) + gpuCount := len(ds) + klog.V(4).Infof("GetPluginDevices: generated %d ListAndWatch entries (gpuCount=%d, splitCount=%d)", total, gpuCount, count) + + // Runtime validation: warn when entry count exceeds the kubelet gRPC limit. + // Only applies to non-MIG devices whose entry count scales with splitCount. + if !isMIG && total > kubeletListAndWatchMaxEntries { + // Compute per-GPU memory in MiB from actual device data (TotalMemory is bytes). + var maxMemBytes uint64 + for _, dev := range ds { + if dev.TotalMemory > maxMemBytes { + maxMemBytes = dev.TotalMemory + } + } + totalMemMiB := int64(maxMemBytes >> 20) // bytes → MiB + + // Minimum safe factor: ceil(totalMemMiB / floor(60000 / gpuCount)) + // Derived from: ceil(totalMemMiB / factor) × gpuCount ≤ 60000 + var minFactor int64 + if totalMemMiB > 0 && gpuCount > 0 { + entriesPerGPULimit := int64(kubeletListAndWatchMaxEntries) / int64(gpuCount) + if entriesPerGPULimit < 1 { + entriesPerGPULimit = 1 + } + minFactor = (totalMemMiB + entriesPerGPULimit - 1) / entriesPerGPULimit + } + klog.Warningf( + "GetPluginDevices: entry count %d exceeds the kubelet gRPC message limit of ~%d entries (~4 MB). "+ + "kubelet will reject the ListAndWatch response and vgpu-memory will show 0 or a stale value. "+ + "gpuCount=%d, splitCount=%d, maxGPUMemMiB=%d. "+ + "Minimum safe memoryFactor = ceil(%d / floor(%d / %d)) = %d.", + total, kubeletListAndWatchMaxEntries, + gpuCount, count, totalMemMiB, + totalMemMiB, kubeletListAndWatchMaxEntries, gpuCount, minFactor, + ) } return res diff --git a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices_test.go b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices_test.go index 59a540d823..2b51391769 100644 --- a/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices_test.go +++ b/pkg/device-plugin/nvidiadevice/nvinternal/rm/devices_test.go @@ -17,6 +17,7 @@ limitations under the License. package rm import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -116,3 +117,226 @@ func TestGetPluginDevicesTopology(t *testing.T) { }) } } + +// makeGPUs builds a Devices map with n non-MIG entries, each with the given +// health status. The IDs are "GPU-uuid-" so they do NOT contain "MIG". +// TotalMemory is left at zero; use makeGPUsWithMem when the validation path +// that reads TotalMemory matters. +func makeGPUs(n int, health string) Devices { + return makeGPUsWithMem(n, health, 0) +} + +// makeGPUsWithMem is like makeGPUs but also sets TotalMemory (in bytes) on +// each device. This exercises the runtime validation path in GetPluginDevices +// that computes minFactor from actual GPU memory. +func makeGPUsWithMem(n int, health string, totalMemBytes uint64) Devices { + ds := make(Devices, n) + for i := 0; i < n; i++ { + id := fmt.Sprintf("GPU-uuid-%d", i) + ds[id] = &Device{ + Device: kubeletdevicepluginv1beta1.Device{ + ID: id, + Health: health, + }, + TotalMemory: totalMemBytes, + } + } + return ds +} + +func TestGetPluginDevices_EntryCountLogging(t *testing.T) { + // kubeletListAndWatchMaxEntries = 60 000. + // Each non-MIG GPU generates `count` entries. + // total_entries = gpuCount × splitCount. + + tests := []struct { + name string + gpuCount int + splitCount uint + expectEntries int + expectOverMax bool + }{ + { + name: "below limit: 1 GPU × 100 splits = 100 entries", + gpuCount: 1, + splitCount: 100, + expectEntries: 100, + expectOverMax: false, + }, + { + name: "at limit: 1 GPU × 60000 splits = 60000 entries", + gpuCount: 1, + splitCount: 60_000, + expectEntries: 60_000, + expectOverMax: false, + }, + { + name: "over limit: 1 GPU × 60001 splits = 60001 entries", + gpuCount: 1, + splitCount: 60_001, + expectEntries: 60_001, + expectOverMax: true, + }, + { + // 8 GPUs × 7500 splits = 60000: exactly at the per-GPU budget + // derived from floor(60000/8)=7500, so total=60000 — safe. + name: "at limit: 8 GPUs × 7500 splits = 60000 entries", + gpuCount: 8, + splitCount: 7_500, + expectEntries: 60_000, + expectOverMax: false, + }, + { + // 8 GPUs × 7501 splits = 60008: one step over the safe per-GPU budget. + name: "over limit: 8 GPUs × 7501 splits = 60008 entries", + gpuCount: 8, + splitCount: 7_501, + expectEntries: 60_008, + expectOverMax: true, + }, + { + // Classic A800 failure case: splitCount = totalMemMiB/factor = 81920/1. + name: "over limit: 1 GPU × 81920 splits (A800 factor=1)", + gpuCount: 1, + splitCount: 81_920, + expectEntries: 81_920, + expectOverMax: true, + }, + { + // A800 with factor=2: splitCount = 81920/2 = 40960 — safe. + name: "below limit: 1 GPU × 40960 splits (A800 factor=2)", + gpuCount: 1, + splitCount: 40_960, + expectEntries: 40_960, + expectOverMax: false, + }, + { + name: "empty device list returns nothing", + gpuCount: 0, + splitCount: 100, + expectEntries: 0, + expectOverMax: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ds := makeGPUs(tc.gpuCount, kubeletdevicepluginv1beta1.Healthy) + result := ds.GetPluginDevices(tc.splitCount, false) + + require.Len(t, result, tc.expectEntries, + "expected %d entries, got %d", tc.expectEntries, len(result)) + + // Verify that entries beyond the limit are still returned (we warn but + // do not truncate — truncation would hide the problem from the caller). + if tc.expectOverMax { + require.Greater(t, len(result), kubeletListAndWatchMaxEntries, + "expected entry count to exceed kubeletListAndWatchMaxEntries") + } + }) + } +} + +// TestGetPluginDevices_RuntimeValidation verifies that GetPluginDevices +// computes the correct minFactor from actual TotalMemory when the entry count +// exceeds the kubelet gRPC limit. +// +// Formula under test: +// +// minFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) +func TestGetPluginDevices_RuntimeValidation(t *testing.T) { + const mibBytes = 1 << 20 // bytes per MiB + + tests := []struct { + name string + gpuCount int + totalMemGiB uint64 // memory per GPU in GiB + splitCount uint // = totalMemMiB / memoryFactor in the real plugin + expectOverMax bool + // wantMinFactor is what GetPluginDevices should compute and log. + // We verify it independently here. + wantMinFactor int64 + }{ + { + // A800 80 GiB, 1 GPU, factor=1 → 81920 entries, over limit. + // minFactor = ceil(81920 / floor(60000/1)) = ceil(81920/60000) = 2. + name: "A800 80 GiB × 1 GPU, factor=1 — over limit, minFactor=2", + gpuCount: 1, + totalMemGiB: 80, + splitCount: 81_920, // totalMemMiB / factor = 81920 / 1 + expectOverMax: true, + wantMinFactor: 2, + }, + { + // A800 80 GiB, 8 GPUs, factor=2 → ceil(81920/2)×8 = 327680, over limit. + // minFactor = ceil(81920 / floor(60000/8)) = ceil(81920/7500) = 11. + name: "A800 80 GiB × 8 GPUs, factor=2 — over limit, minFactor=11", + gpuCount: 8, + totalMemGiB: 80, + splitCount: 40_960, // totalMemMiB / factor = 81920 / 2 + expectOverMax: true, + wantMinFactor: 11, + }, + { + // A800 80 GiB, 8 GPUs, factor=11 → ceil(81920/11)×8 = 7448×8 = 59584 ≤ 60000. + // Safe — no warning, minFactor not needed. + name: "A800 80 GiB × 8 GPUs, factor=11 — under limit", + gpuCount: 8, + totalMemGiB: 80, + splitCount: 7_448, // ceil(81920 / 11) + expectOverMax: false, + wantMinFactor: 0, // irrelevant when under limit + }, + { + // T4 16 GiB, 1 GPU, factor=1 → 16384 entries — well under limit. + name: "T4 16 GiB × 1 GPU, factor=1 — under limit", + gpuCount: 1, + totalMemGiB: 16, + splitCount: 16_384, + expectOverMax: false, + wantMinFactor: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + totalMemBytes := tc.totalMemGiB * 1024 * mibBytes + ds := makeGPUsWithMem(tc.gpuCount, kubeletdevicepluginv1beta1.Healthy, totalMemBytes) + result := ds.GetPluginDevices(tc.splitCount, false) + + wantEntries := tc.gpuCount * int(tc.splitCount) + require.Len(t, result, wantEntries) + + if tc.expectOverMax { + require.Greater(t, len(result), kubeletListAndWatchMaxEntries) + + // Independently verify the minFactor the code should compute. + totalMemMiB := int64(tc.totalMemGiB) * 1024 + entriesPerGPULimit := int64(kubeletListAndWatchMaxEntries) / int64(tc.gpuCount) + if entriesPerGPULimit < 1 { + entriesPerGPULimit = 1 + } + computedMinFactor := (totalMemMiB + entriesPerGPULimit - 1) / entriesPerGPULimit + require.Equal(t, tc.wantMinFactor, computedMinFactor, + "minFactor formula mismatch for %s", tc.name) + } + }) + } +} + +// TestGetPluginDevices_UniqueIDs verifies that every generated entry has a +// unique ID of the form "-", which is what the gRPC ListAndWatch +// protocol requires. +func TestGetPluginDevices_UniqueIDs(t *testing.T) { + ds := makeGPUs(3, kubeletdevicepluginv1beta1.Healthy) + result := ds.GetPluginDevices(4, false) + + require.Len(t, result, 12) // 3 GPUs × 4 splits + + seen := make(map[string]struct{}, len(result)) + for _, d := range result { + _, dup := seen[d.ID] + require.False(t, dup, "duplicate device ID %q", d.ID) + seen[d.ID] = struct{}{} + } +} diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 6d95c4d31f..3c3a6ea753 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -168,6 +168,65 @@ type NvidiaGPUDevices struct { mu sync.Mutex // protects concurrent access to reported node state } +// kubeletListAndWatchMaxEntries is the conservative upper bound on the number +// of kubelet Device entries that fit within the gRPC ~4 MB message limit. +// Each Device proto is approximately 64 bytes; 60 000 × 64 B ≈ 3.84 MB, +// leaving a safe margin below the hard 4 MB ceiling. +const kubeletListAndWatchMaxEntries = 60_000 + +// ValidateMemoryFactor checks whether the combination of per-GPU memory, +// GPU count, and memoryFactor would produce a ListAndWatch response that +// exceeds the kubelet gRPC 4 MB message limit, and logs a warning if so. +// +// The device plugin advertises ceil(totalMemMiB / memoryFactor) virtual +// entries per GPU. The total entry count across all GPUs must not exceed +// kubeletListAndWatchMaxEntries (~60 000). +// +// Runtime check: +// +// estimatedEntries = ceil(totalMemMiB / memoryFactor) × gpuCount +// +// Minimum safe factor (derived by rearranging the above inequality): +// +// minFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) +// +// This function is called at runtime by GetPluginDevices with the actual GPU +// memory and count obtained from the device plugin. It is a warn-only +// function; callers are responsible for any remediation. +func ValidateMemoryFactor(memoryFactor int32, totalMemMiB int64, gpuCount int) { + if memoryFactor <= 0 { + klog.Warningf("memoryFactor must be a positive integer (got %d); defaulting behaviour may be unexpected", memoryFactor) + return + } + if totalMemMiB <= 0 || gpuCount <= 0 { + // Not enough information to estimate — skip. + return + } + // Runtime check: ceil(totalMemMiB / memoryFactor) × gpuCount + entriesPerGPU := (totalMemMiB + int64(memoryFactor) - 1) / int64(memoryFactor) + estimatedEntries := entriesPerGPU * int64(gpuCount) + if estimatedEntries > kubeletListAndWatchMaxEntries { + // Compute minimum safe factor using the canonical form: + // minFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) + entriesPerGPULimit := int64(kubeletListAndWatchMaxEntries) / int64(gpuCount) + minFactor := (totalMemMiB + entriesPerGPULimit - 1) / entriesPerGPULimit + klog.Warningf( + "gpuMemoryFactor=%d with totalMemMiB=%d and gpuCount=%d would generate ~%d ListAndWatch entries, "+ + "exceeding the kubelet gRPC message limit of ~%d entries (~4 MB). "+ + "kubelet will reject the response and volcano.sh/vgpu-memory will show 0 or a stale value. "+ + "Minimum safe memoryFactor = ceil(%d / floor(%d / %d)) = %d.", + memoryFactor, totalMemMiB, gpuCount, estimatedEntries, + kubeletListAndWatchMaxEntries, + totalMemMiB, kubeletListAndWatchMaxEntries, gpuCount, minFactor, + ) + } else { + klog.V(4).Infof( + "memoryFactor=%d validation passed: estimated ListAndWatch entries=%d (limit=%d)", + memoryFactor, estimatedEntries, kubeletListAndWatchMaxEntries, + ) + } +} + func InitNvidiaDevice(nvconfig NvidiaConfig) *NvidiaGPUDevices { klog.InfoS("initializing nvidia device", "resourceName", nvconfig.ResourceCountName, "resourceMem", nvconfig.ResourceMemoryName, "DefaultGPUNum", nvconfig.DefaultGPUNum) _, ok := device.InRequestDevices[NvidiaGPUDevice] @@ -177,6 +236,12 @@ func InitNvidiaDevice(nvconfig NvidiaConfig) *NvidiaGPUDevices { util.HandshakeAnnos[NvidiaGPUDevice] = HandshakeAnnos } MemoryFactor = nvconfig.MemoryFactor + if nvconfig.MemoryFactor <= 0 { + klog.Warningf("memoryFactor is %d (must be >= 1); defaulting to 1 to avoid divide-by-zero", nvconfig.MemoryFactor) + MemoryFactor = 1 + } + // Validation with actual GPU memory and count happens at runtime inside + // GetPluginDevices, once the device plugin has queried the hardware. return &NvidiaGPUDevices{ config: nvconfig, ReportedGPUNum: make(map[string]int64), diff --git a/pkg/device/nvidia/device_test.go b/pkg/device/nvidia/device_test.go index 3f71933ad3..ed7d2f957f 100644 --- a/pkg/device/nvidia/device_test.go +++ b/pkg/device/nvidia/device_test.go @@ -2783,3 +2783,242 @@ func TestFit_TopologyBestCombination(t *testing.T) { assert.Assert(t, uuids["dev-0"]) assert.Assert(t, uuids["dev-2"]) } + +// TestValidateMemoryFactor verifies that ValidateMemoryFactor correctly +// identifies configurations that would exceed the kubelet gRPC 4 MB message +// limit (~60 000 entries) and does not warn for safe configurations. +// +// Runtime check: ceil(totalMemMiB / factor) × gpuCount > 60 000 +// Min safe factor: ceil(totalMemMiB / floor(60000 / gpuCount)) +// +// The function only logs warnings so we verify indirectly: confirm no panic, +// and independently verify the estimated entry count against expectOverMax. +func TestValidateMemoryFactor(t *testing.T) { + tests := []struct { + name string + factor int32 + totalMemMiB int64 + gpuCount int + expectOverMax bool + }{ + { + // A800 80 GiB, factor=1: ceil(81920/1)×1 = 81920 → over limit. + name: "A800 factor=1 single GPU — over limit", + factor: 1, + totalMemMiB: 81_920, + gpuCount: 1, + expectOverMax: true, + }, + { + // A800 80 GiB, factor=2: ceil(81920/2)×1 = 40960 → under limit. + name: "A800 factor=2 single GPU — under limit", + factor: 2, + totalMemMiB: 81_920, + gpuCount: 1, + expectOverMax: false, + }, + { + // A800 80 GiB, factor=2, 8 GPUs: ceil(81920/2)×8 = 327680 → over limit. + name: "A800 factor=2 eight GPUs — over limit", + factor: 2, + totalMemMiB: 81_920, + gpuCount: 8, + expectOverMax: true, + }, + { + // A800 80 GiB, factor=11, 8 GPUs: ceil(81920/11)×8 = 7448×8 = 59584 → under limit. + name: "A800 factor=11 eight GPUs — under limit", + factor: 11, + totalMemMiB: 81_920, + gpuCount: 8, + expectOverMax: false, + }, + { + // 8 GiB GPU, factor=1, 1 GPU: 8192 entries → well under limit. + name: "8 GiB GPU factor=1 single GPU — under limit", + factor: 1, + totalMemMiB: 8_192, + gpuCount: 1, + expectOverMax: false, + }, + { + // Exactly at the boundary: totalMemMiB=60000, factor=1, 1 GPU → not over. + name: "exactly at limit boundary — at limit, not over", + factor: 1, + totalMemMiB: 60_000, + gpuCount: 1, + expectOverMax: false, + }, + { + // One over the boundary. + name: "one over limit boundary — over limit", + factor: 1, + totalMemMiB: 60_001, + gpuCount: 1, + expectOverMax: true, + }, + { + // Zero totalMemMiB → skip (not enough info). + name: "zero totalMemMiB — skip", + factor: 1, + totalMemMiB: 0, + gpuCount: 1, + expectOverMax: false, + }, + { + // Zero gpuCount → skip. + name: "zero gpuCount — skip", + factor: 1, + totalMemMiB: 81_920, + gpuCount: 0, + expectOverMax: false, + }, + { + // Negative factor → warns and returns early, no panic. + name: "negative factor — warns and returns early", + factor: -1, + totalMemMiB: 81_920, + gpuCount: 1, + expectOverMax: false, + }, + { + // Zero factor → same early-return path. + name: "zero factor — warns and returns early", + factor: 0, + totalMemMiB: 81_920, + gpuCount: 1, + expectOverMax: false, + }, + { + // Large factor: ceil(81920/1000)×1 = 83 → safe. + name: "large factor — very few entries, safe", + factor: 1000, + totalMemMiB: 81_920, + gpuCount: 1, + expectOverMax: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Must not panic under any input. + func() { + defer func() { + if r := recover(); r != nil { + t.Errorf("ValidateMemoryFactor panicked: %v", r) + } + }() + ValidateMemoryFactor(tt.factor, tt.totalMemMiB, tt.gpuCount) + }() + + // Independently verify the runtime check formula so the test stays in + // sync with the implementation. + if tt.factor > 0 && tt.totalMemMiB > 0 && tt.gpuCount > 0 { + entriesPerGPU := (tt.totalMemMiB + int64(tt.factor) - 1) / int64(tt.factor) + estimated := entriesPerGPU * int64(tt.gpuCount) + isOver := estimated > int64(kubeletListAndWatchMaxEntries) + assert.Equal(t, tt.expectOverMax, isOver, + "entry estimate mismatch: estimated=%d, expectOverMax=%v", estimated, tt.expectOverMax) + } + }) + } +} + +// TestValidateMemoryFactor_MinFactor verifies that the minimum safe +// memoryFactor formula produces the correct result for representative GPUs. +// +// Formula: minFactor = ceil(totalMemMiB / floor(60000 / gpuCount)) +func TestValidateMemoryFactor_MinFactor(t *testing.T) { + const limit = kubeletListAndWatchMaxEntries + + tests := []struct { + name string + totalMemMiB int64 + gpuCount int + wantMinFactor int64 + }{ + { + // A800 80 GiB, 1 GPU: + // floor(60000/1)=60000; ceil(81920/60000)=ceil(1.365)=2 + name: "A800 80 GiB × 1 GPU", + totalMemMiB: 81_920, + gpuCount: 1, + wantMinFactor: 2, + }, + { + // A800 80 GiB, 8 GPUs: + // floor(60000/8)=7500; ceil(81920/7500)=ceil(10.923)=11 + name: "A800 80 GiB × 8 GPUs", + totalMemMiB: 81_920, + gpuCount: 8, + wantMinFactor: 11, + }, + { + // H200 141 GiB, 8 GPUs: + // floor(60000/8)=7500; ceil(144384/7500)=ceil(19.25)=20 + name: "H200 141 GiB × 8 GPUs", + totalMemMiB: 144_384, + gpuCount: 8, + wantMinFactor: 20, + }, + { + // T4 16 GiB, 8 GPUs: + // floor(60000/8)=7500; ceil(16384/7500)=ceil(2.18)=3 + name: "T4 16 GiB × 8 GPUs", + totalMemMiB: 16_384, + gpuCount: 8, + wantMinFactor: 3, + }, + { + // A100 40 GiB, 1 GPU: + // floor(60000/1)=60000; ceil(40960/60000)=ceil(0.682)=1 + name: "A100 40 GiB × 1 GPU — factor 1 is safe", + totalMemMiB: 40_960, + gpuCount: 1, + wantMinFactor: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entriesPerGPULimit := int64(limit) / int64(tt.gpuCount) + if entriesPerGPULimit < 1 { + entriesPerGPULimit = 1 + } + minFactor := (tt.totalMemMiB + entriesPerGPULimit - 1) / entriesPerGPULimit + assert.Equal(t, tt.wantMinFactor, minFactor, + "minFactor mismatch for %s: got %d, want %d", tt.name, minFactor, tt.wantMinFactor) + + // Confirm minFactor is actually safe: ceil(totalMemMiB/minFactor)×gpuCount ≤ limit. + entriesPerGPU := (tt.totalMemMiB + minFactor - 1) / minFactor + totalEntries := entriesPerGPU * int64(tt.gpuCount) + assert.Assert(t, totalEntries <= int64(limit), + "minFactor=%d still produces %d entries > %d for %s", + minFactor, totalEntries, limit, tt.name) + }) + } +} + +// TestInitNvidiaDevice_NonPositiveMemoryFactor confirms that a zero or negative +// memoryFactor is corrected to 1 inside InitNvidiaDevice, and that no startup +// call to ValidateMemoryFactor with zero inputs occurs (the removed no-op). +func TestInitNvidiaDevice_NonPositiveMemoryFactor(t *testing.T) { + tests := []struct { + name string + factor int32 + wantGlobalFact int32 + }{ + {name: "factor=0 defaults to 1", factor: 0, wantGlobalFact: 1}, + {name: "factor=-5 defaults to 1", factor: -5, wantGlobalFact: 1}, + {name: "factor=3 kept as-is", factor: 3, wantGlobalFact: 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + InitNvidiaDevice(NvidiaConfig{ + ResourceCountName: "nvidia.com/gpu", + MemoryFactor: tt.factor, + }) + assert.Equal(t, tt.wantGlobalFact, MemoryFactor) + }) + } +}