feat(monitor): add DeviceProcessCount metric to expose tracked GPU process count

cmd/vGPUmonitor/feedback_test.go

@@ -73,6 +73,7 @@ func (s *stubInfo) GetRecentKernel() int32               { return 1 }
 func (s *stubInfo) SetRecentKernel(int32)                {}
 func (s *stubInfo) GetUtilizationSwitch() int32          { return 0 }
 func (s *stubInfo) SetUtilizationSwitch(int32)           {}
+func (s *stubInfo) DeviceProcessCount(int) int           { return 0 }
 
 func TestCheckFunctionsHighPriority(t *testing.T) {
 	sw := map[string]UtilizationPerDevice{"gpu-0": {0, 1}}

cmd/vGPUmonitor/metrics.go

@@ -118,6 +118,12 @@ var (
 		"Container device memory buffer size in bytes",
 		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
 	)
+
+	ctrDeviceProcessCountDesc = prometheus.NewDesc(
+		"hami_container_device_process_count",
+		"Number of processes tracked in the container's shared-memory region for this device",
+		[]string{"namespace", "pod", "container", "vdevice_index", "device_uuid"}, nil,
+	)
 )
 
 // Legacy metric descriptors (populated only when --legacy-metrics is enabled).
@@ -198,6 +204,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- ctrDeviceMemoryContextDesc
 	ch <- ctrDeviceMemoryModuleDesc
 	ch <- ctrDeviceMemoryBufferDesc
+	ch <- ctrDeviceProcessCountDesc
 
 	if cc.ClusterManager.LegacyMetrics {
 		ch <- legacyHostGPUdesc
@@ -489,6 +496,12 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M
 			return err
 		}
 
+		processCount := c.Info.DeviceProcessCount(i)
+		if err := sendMetric(ch, ctrDeviceProcessCountDesc, prometheus.GaugeValue, float64(processCount), labels...); err != nil {
+			klog.Errorf("Failed to send device process count metric: %v", err)
+			return err
+		}
+
 		if lastKernelTime > 0 {
 			lastSec := max(nowSec-lastKernelTime, 0)
 			if err := sendMetric(ch, ctrDeviceLastKernelDesc, prometheus.GaugeValue, float64(lastSec), labels...); err != nil {

cmd/vGPUmonitor/metrics_container_test.go

@@ -83,6 +83,7 @@ func TestCollectContainerMetrics(t *testing.T) {
 		ctrDeviceMemoryContextDesc: 1000,
 		ctrDeviceMemoryModuleDesc:  500,
 		ctrDeviceMemoryBufferDesc:  300,
+		ctrDeviceProcessCountDesc:  0,
 		ctrDeviceLastKernelDesc:    60, // nowSec - lastKernel = 160 - 100
 	}
 	if len(metrics) != len(want) {
@@ -121,8 +122,8 @@ func TestCollectContainerMetricsNoKernelActivity(t *testing.T) {
 			t.Error("last-kernel metric should not be emitted when no kernel has run")
 		}
 	}
-	if len(metrics) != 7 {
-		t.Errorf("got %d metrics, want 7", len(metrics))
+	if len(metrics) != 8 {
+		t.Errorf("got %d metrics, want 8", len(metrics))
 	}
 }
 

pkg/monitor/nvidia/cudevshr.go

@@ -65,7 +65,7 @@ type UsageInfo interface {
 	DeviceMemoryLimit(idx int) uint64
 	SetDeviceMemoryLimit(l uint64)
 	LastKernelTime() int64
-	//UsedMemory(idx int) (uint64, error)
+	DeviceProcessCount(idx int) int
 	GetPriority() int
 	GetRecentKernel() int32
 	SetRecentKernel(v int32)

pkg/monitor/nvidia/v0/spec.go

@@ -162,6 +162,16 @@ func (s Spec) SetDeviceSmLimit(l uint64) {
 	}
 }
 
+func (s Spec) DeviceProcessCount(idx int) int {
+	n := 0
+	for _, p := range s.activeProcs() {
+		if p.status != 0 && (p.used[idx].total > 0 || p.used[idx].contextSize > 0 || p.used[idx].moduleSize > 0) {
+			n++
+		}
+	}
+	return n
+}
+
 func (s Spec) IsValidUUID(idx int) bool {
 	return s.sr.uuids[idx].uuid[0] != 0
 }

pkg/monitor/nvidia/v0/spec_test.go

@@ -489,6 +489,110 @@ func TestSpec_CorruptProcnumIsClamped(t *testing.T) {
 	}
 }
 
+func TestSpec_DeviceProcessCount(t *testing.T) {
+	tests := []struct {
+		name      string
+		spec      *Spec
+		deviceIdx int
+		want      int
+	}{
+		{
+			name: "no active processes",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 2,
+				procs: [1024]shrregProcSlotT{
+					{status: 0},
+					{status: 0},
+				},
+			}},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "one active process with memory on device 0",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{total: 512 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "process active on device 1 only is not counted for device 0",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{}, {total: 256 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "dead slot within procnum is excluded",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 2,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{total: 128 * 1024 * 1024}}},
+					{status: 0, used: [16]deviceMemory{{total: 999 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "corrupt procnum is clamped and does not panic",
+			spec: &Spec{sr: func() *sharedRegionT {
+				sr := &sharedRegionT{num: 1, procnum: -99}
+				sr.procs[0].status = 1
+				sr.procs[0].used[0].total = 64 * 1024 * 1024
+				return sr
+			}()},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "context-only slot is counted (total not yet written)",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     1,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{contextSize: 32 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "module-only slot is counted (total not yet written)",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     1,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{moduleSize: 16 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.spec.DeviceProcessCount(tt.deviceIdx)
+			if got != tt.want {
+				t.Errorf("DeviceProcessCount(%d) = %d, want %d", tt.deviceIdx, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestDeviceMemoryLimit(t *testing.T) {
 	testCases := []struct {
 		name          string

pkg/monitor/nvidia/v1/spec.go

@@ -176,6 +176,16 @@ func (s Spec) SetDeviceSmLimit(l uint64) {
 	}
 }
 
+func (s Spec) DeviceProcessCount(idx int) int {
+	n := 0
+	for _, p := range s.activeProcs() {
+		if p.status != 0 && (p.used[idx].total > 0 || p.used[idx].contextSize > 0 || p.used[idx].moduleSize > 0) {
+			n++
+		}
+	}
+	return n
+}
+
 func (s Spec) IsValidUUID(idx int) bool {
 	return s.sr.uuids[idx].uuid[0] != 0
 }

pkg/monitor/nvidia/v1/spec_test.go

@@ -1388,3 +1388,107 @@ func TestSpec_CorruptProcnumIsClamped(t *testing.T) {
 		})
 	}
 }
+
+func TestSpec_DeviceProcessCount(t *testing.T) {
+	tests := []struct {
+		name      string
+		spec      *Spec
+		deviceIdx int
+		want      int
+	}{
+		{
+			name: "no active processes",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 2,
+				procs: [1024]shrregProcSlotT{
+					{status: 0},
+					{status: 0},
+				},
+			}},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "one active process with memory on device 0",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{total: 512 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "process active on device 1 only is not counted for device 0",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{}, {total: 256 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "dead slot within procnum is excluded",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     2,
+				procnum: 2,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{total: 128 * 1024 * 1024}}},
+					{status: 0, used: [16]deviceMemory{{total: 999 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "corrupt procnum is clamped and does not panic",
+			spec: &Spec{sr: func() *sharedRegionT {
+				sr := &sharedRegionT{num: 1, procnum: -99}
+				sr.procs[0].status = 1
+				sr.procs[0].used[0].total = 64 * 1024 * 1024
+				return sr
+			}()},
+			deviceIdx: 0,
+			want:      0,
+		},
+		{
+			name: "context-only slot is counted (total not yet written)",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     1,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{contextSize: 32 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+		{
+			name: "module-only slot is counted (total not yet written)",
+			spec: &Spec{sr: &sharedRegionT{
+				num:     1,
+				procnum: 1,
+				procs: [1024]shrregProcSlotT{
+					{status: 1, used: [16]deviceMemory{{moduleSize: 16 * 1024 * 1024}}},
+				},
+			}},
+			deviceIdx: 0,
+			want:      1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.spec.DeviceProcessCount(tt.deviceIdx)
+			if got != tt.want {
+				t.Errorf("DeviceProcessCount(%d) = %d, want %d", tt.deviceIdx, got, tt.want)
+			}
+		})
+	}
+}