fix: warn when device list exceeds kubelet gRPC limit

doc/design.md

@@ -15,6 +15,8 @@ For example:
  - args: ["--gpu-strategy=number"] will let device plugin using the gpu-number strategy
  - args: ["--gpu-strategy=share","--gpu-memory-factor=10"] will let device plugin using the gpu-share strategy, and memory factor is 10MB
 
+Every memory unit is registered as a separate device, so keep the node total, `sum of GPU memory in MB / gpu-memory-factor`, at most 60000. Kubelet drops a larger list and the resource stays at 0. A single 80GB card needs a factor of 2, a node with eight of them needs 11.
+
 ### As a configuration file
 ```
 version: v1

pkg/plugin/server.go

@@ -53,8 +53,27 @@ const (
 	deviceListEnvVar                          = "NVIDIA_VISIBLE_DEVICES"
 	deviceListAsVolumeMountsHostPath          = "/dev/null"
 	deviceListAsVolumeMountsContainerPathRoot = "/var/run/nvidia-container-devices"
+
+	// deviceEntryLimit is how many devices fit in one ListAndWatch response.
+	// Kubelet dials the plugin without overriding grpc's
+	// defaultClientMaxReceiveMessageSize, so 4MB applies. A memory device entry
+	// costs about 66 bytes, which leaves room for ~63500, rounded down here to
+	// keep a margin if the ID format grows.
+	deviceEntryLimit = 60000
 )
 
+// checkDeviceEntries reports a device list too large for kubelet to receive.
+// Kubelet drops such a response, so the resource stays at 0 or keeps the value
+// from the last accepted one.
+func checkDeviceEntries(count int, factor uint) error {
+	if count <= deviceEntryLimit {
+		return nil
+	}
+	needed := (factor*uint(count) + deviceEntryLimit - 1) / deviceEntryLimit
+	return fmt.Errorf("%d memory devices exceed the %d kubelet can receive, set gpuMemoryFactor to at least %d",
+		count, deviceEntryLimit, needed)
+}
+
 // nvidiaDevicePlugin implements the Kubernetes device plugin API
 type nvidiaDevicePlugin struct {
 	pluginapi.UnimplementedDevicePluginServer
@@ -76,6 +95,11 @@ type nvidiaDevicePlugin struct {
 	mps mpsOptions
 
 	migCurrent config.MigPartedSpec
+
+	// entryLimitWarned keeps the device count warning to one line per plugin,
+	// since apiDevices runs again on every health event. Only ListAndWatch
+	// reaches it, so no synchronisation is needed.
+	entryLimitWarned bool
 }
 
 // devicePluginForResource creates a device plugin for the specified resource.
@@ -655,6 +679,12 @@ func (plugin *nvidiaDevicePlugin) apiDevices() []*pluginapi.Device {
 			}
 		}
 		klog.Infoln("res length=", len(res))
+		if !plugin.entryLimitWarned {
+			plugin.entryLimitWarned = true
+			if err := checkDeviceEntries(len(res), config.GPUMemoryFactor); err != nil {
+				klog.Warning(err)
+			}
+		}
 		return res
 	} else if plugin.rm.Resource() == spec.ResourceName(util.ResourceCores) {
 		for _, dev := range devs {

pkg/plugin/server_test.go

@@ -251,6 +251,16 @@ func TestCDIAllocateResponse(t *testing.T) {
 	}
 }
 
+func TestCheckDeviceEntries(t *testing.T) {
+	require.NoError(t, checkDeviceEntries(deviceEntryLimit, 1))
+	// two 80GB cards at factor 1
+	require.ErrorContains(t, checkDeviceEntries(163840, 1), "at least 3")
+	// same node at factor 4
+	require.NoError(t, checkDeviceEntries(40960, 4))
+	// an exact multiple of the limit needs that factor, not one more
+	require.ErrorContains(t, checkDeviceEntries(2*deviceEntryLimit, 1), "at least 2")
+}
+
 func ptr[T any](x T) *T {
 	return &x
 }