From 43067d5111646aa78516f420591c1d10828571d4 Mon Sep 17 00:00:00 2001 From: Nakshatra Sharma Date: Mon, 3 Aug 2026 22:42:02 +0530 Subject: [PATCH] fix(vGPUmonitor): skip dead proc slots in v0 and v1 spec metric sums Both v0 (old 1197897-byte cache) and v1 aggregation functions iterated activeProcs() without checking p.status, so slots whose CUDA process had already exited were still included in memory and utilization totals. Add a p.status == 0 guard inside every loop in both specs so only live process slots contribute to the reported metrics. Signed-off-by: Nakshatra Sharma --- pkg/monitor/nvidia/v0/spec.go | 18 +++ pkg/monitor/nvidia/v0/spec_test.go | 151 ++++++++++++++++------ pkg/monitor/nvidia/v1/spec.go | 18 +++ pkg/monitor/nvidia/v1/spec_test.go | 197 ++++++++++++++++++++++++++++- 4 files changed, 347 insertions(+), 37 deletions(-) diff --git a/pkg/monitor/nvidia/v0/spec.go b/pkg/monitor/nvidia/v0/spec.go index 444bddf827..cf2ebc9f4c 100644 --- a/pkg/monitor/nvidia/v0/spec.go +++ b/pkg/monitor/nvidia/v0/spec.go @@ -92,6 +92,9 @@ func (s Spec) activeProcs() []shrregProcSlotT { func (s Spec) DeviceMemoryContextSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].contextSize } return v @@ -100,6 +103,9 @@ func (s Spec) DeviceMemoryContextSize(idx int) uint64 { func (s Spec) DeviceMemoryModuleSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].moduleSize } return v @@ -108,6 +114,9 @@ func (s Spec) DeviceMemoryModuleSize(idx int) uint64 { func (s Spec) DeviceMemoryBufferSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].bufferSize } return v @@ -116,6 +125,9 @@ func (s Spec) DeviceMemoryBufferSize(idx int) uint64 { func (s Spec) DeviceMemoryOffset(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].offset } return v @@ -124,6 +136,9 @@ func (s Spec) DeviceMemoryOffset(idx int) uint64 { func (s Spec) DeviceMemoryTotal(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].total } return v @@ -132,6 +147,9 @@ func (s Spec) DeviceMemoryTotal(idx int) uint64 { func (s Spec) DeviceSmUtil(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.deviceUtil[idx].smUtil } return v diff --git a/pkg/monitor/nvidia/v0/spec_test.go b/pkg/monitor/nvidia/v0/spec_test.go index 0e68e78250..fed9794411 100644 --- a/pkg/monitor/nvidia/v0/spec_test.go +++ b/pkg/monitor/nvidia/v0/spec_test.go @@ -68,8 +68,8 @@ func TestSpec_DeviceMemoryContextSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, - {used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, }, }}, input: 1, @@ -81,8 +81,8 @@ func TestSpec_DeviceMemoryContextSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, - {used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, }, }}, input: 0, @@ -94,13 +94,26 @@ func TestSpec_DeviceMemoryContextSize(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, - {used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 100}, {contextSize: 200}}}, + {status: 1, used: [16]deviceMemory{{contextSize: 300}, {contextSize: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{contextSize: 200}}}, + {status: 0, used: [16]deviceMemory{{contextSize: 999}}}, + }, + }}, + input: 0, + expected: uint64(200), + }, } for _, tt := range tests { @@ -121,8 +134,8 @@ func TestSpec_DeviceMemoryModuleSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, - {used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, }, }}, input: 1, @@ -134,8 +147,8 @@ func TestSpec_DeviceMemoryModuleSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, - {used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, }, }}, input: 0, @@ -147,13 +160,26 @@ func TestSpec_DeviceMemoryModuleSize(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, - {used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 100}, {moduleSize: 200}}}, + {status: 1, used: [16]deviceMemory{{moduleSize: 300}, {moduleSize: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{moduleSize: 150}}}, + {status: 0, used: [16]deviceMemory{{moduleSize: 999}}}, + }, + }}, + input: 0, + expected: uint64(150), + }, } for _, tt := range tests { @@ -174,8 +200,8 @@ func TestSpec_DeviceMemoryBufferSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, - {used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, }, }}, input: 1, @@ -187,8 +213,8 @@ func TestSpec_DeviceMemoryBufferSize(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, - {used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, }, }}, input: 0, @@ -200,13 +226,26 @@ func TestSpec_DeviceMemoryBufferSize(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, - {used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 100}, {bufferSize: 200}}}, + {status: 1, used: [16]deviceMemory{{bufferSize: 300}, {bufferSize: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{bufferSize: 400}}}, + {status: 0, used: [16]deviceMemory{{bufferSize: 999}}}, + }, + }}, + input: 0, + expected: uint64(400), + }, } for _, tt := range tests { @@ -227,8 +266,8 @@ func TestSpec_DeviceMemoryOffset(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, - {used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, + {status: 1, used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, + {status: 1, used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, }, }}, input: 1, @@ -240,8 +279,8 @@ func TestSpec_DeviceMemoryOffset(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, - {used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, + {status: 1, used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, + {status: 1, used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, }, }}, input: 0, @@ -253,13 +292,26 @@ func TestSpec_DeviceMemoryOffset(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, - {used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, + {status: 1, used: [16]deviceMemory{{offset: 100}, {offset: 200}}}, + {status: 1, used: [16]deviceMemory{{offset: 300}, {offset: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{offset: 100}}}, + {status: 0, used: [16]deviceMemory{{offset: 999}}}, + }, + }}, + input: 0, + expected: uint64(100), + }, } for _, tt := range tests { @@ -280,8 +332,8 @@ func TestSpec_DeviceMemoryTotal(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{total: 100}, {total: 200}}}, - {used: [16]deviceMemory{{total: 300}, {total: 400}}}, + {status: 1, used: [16]deviceMemory{{total: 100}, {total: 200}}}, + {status: 1, used: [16]deviceMemory{{total: 300}, {total: 400}}}, }, }}, input: 1, @@ -293,8 +345,8 @@ func TestSpec_DeviceMemoryTotal(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{total: 100}, {total: 200}}}, - {used: [16]deviceMemory{{total: 300}, {total: 400}}}, + {status: 1, used: [16]deviceMemory{{total: 100}, {total: 200}}}, + {status: 1, used: [16]deviceMemory{{total: 300}, {total: 400}}}, }, }}, input: 0, @@ -306,13 +358,26 @@ func TestSpec_DeviceMemoryTotal(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {used: [16]deviceMemory{{total: 100}, {total: 200}}}, - {used: [16]deviceMemory{{total: 300}, {total: 400}}}, + {status: 1, used: [16]deviceMemory{{total: 100}, {total: 200}}}, + {status: 1, used: [16]deviceMemory{{total: 300}, {total: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{total: 512}}}, + {status: 0, used: [16]deviceMemory{{total: 999}}}, + }, + }}, + input: 0, + expected: uint64(512), + }, } for _, tt := range tests { @@ -333,8 +398,8 @@ func TestSpec_DeviceSmUtil(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, - {deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, }, }}, input: 1, @@ -346,8 +411,8 @@ func TestSpec_DeviceSmUtil(t *testing.T) { num: 2, procnum: 2, procs: [1024]shrregProcSlotT{ - {deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, - {deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, }, }}, input: 0, @@ -359,13 +424,26 @@ func TestSpec_DeviceSmUtil(t *testing.T) { num: 2, procnum: 1, procs: [1024]shrregProcSlotT{ - {deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, - {deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 100}, {smUtil: 200}}}, + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 300}, {smUtil: 400}}}, }, }}, input: 1, expected: uint64(200), }, + { + name: "dead slot within procnum is excluded", + spec: &Spec{sr: &sharedRegionT{ + num: 2, + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 60}}}, + {status: 0, deviceUtil: [16]deviceUtilization{{smUtil: 999}}}, + }, + }}, + input: 0, + expected: uint64(60), + }, } for _, tt := range tests { @@ -394,6 +472,7 @@ func TestSpec_CorruptProcnumIsClamped(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sr := &sharedRegionT{num: 2, procnum: tt.procnum} + sr.procs[0].status = 1 sr.procs[0].used[0].total = 100 sr.procs[0].deviceUtil[0].smUtil = 100 s := Spec{sr: sr} diff --git a/pkg/monitor/nvidia/v1/spec.go b/pkg/monitor/nvidia/v1/spec.go index 49a12aac11..b2c52bc2bd 100644 --- a/pkg/monitor/nvidia/v1/spec.go +++ b/pkg/monitor/nvidia/v1/spec.go @@ -106,6 +106,9 @@ func (s Spec) activeProcs() []shrregProcSlotT { func (s Spec) DeviceMemoryContextSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].contextSize } return v @@ -114,6 +117,9 @@ func (s Spec) DeviceMemoryContextSize(idx int) uint64 { func (s Spec) DeviceMemoryModuleSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].moduleSize } return v @@ -122,6 +128,9 @@ func (s Spec) DeviceMemoryModuleSize(idx int) uint64 { func (s Spec) DeviceMemoryBufferSize(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].bufferSize } return v @@ -130,6 +139,9 @@ func (s Spec) DeviceMemoryBufferSize(idx int) uint64 { func (s Spec) DeviceMemoryOffset(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].offset } return v @@ -138,6 +150,9 @@ func (s Spec) DeviceMemoryOffset(idx int) uint64 { func (s Spec) DeviceMemoryTotal(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.used[idx].total } return v @@ -146,6 +161,9 @@ func (s Spec) DeviceMemoryTotal(idx int) uint64 { func (s Spec) DeviceSmUtil(idx int) uint64 { v := uint64(0) for _, p := range s.activeProcs() { + if p.status == 0 { + continue + } v += p.deviceUtil[idx].smUtil } return v diff --git a/pkg/monitor/nvidia/v1/spec_test.go b/pkg/monitor/nvidia/v1/spec_test.go index 59f93c0362..5bde938bb1 100644 --- a/pkg/monitor/nvidia/v1/spec_test.go +++ b/pkg/monitor/nvidia/v1/spec_test.go @@ -169,6 +169,7 @@ func Test_DeviceMemoryContextSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { contextSize: 100, @@ -179,6 +180,7 @@ func Test_DeviceMemoryContextSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { contextSize: 100, @@ -206,6 +208,7 @@ func Test_DeviceMemoryContextSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { contextSize: 100, @@ -216,6 +219,7 @@ func Test_DeviceMemoryContextSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { contextSize: 100, @@ -231,6 +235,81 @@ func Test_DeviceMemoryContextSize(t *testing.T) { }, want: uint64(400), }, + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + { + status: 1, + used: [16]deviceMemory{{ + contextSize: 200, + }}, + }, + { + status: 0, + used: [16]deviceMemory{{ + contextSize: 999, + }}, + }, + }, + }, + }, + }, + want: uint64(200), + }, + { + name: "negative procnum is clamped to zero", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: -1, + procs: [1024]shrregProcSlotT{ + { + status: 1, + used: [16]deviceMemory{{ + contextSize: 500, + }}, + }, + }, + }, + }, + }, + want: uint64(0), + }, + { + name: "oversized procnum is clamped to len(procs)", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 9999, + procs: [1024]shrregProcSlotT{ + { + status: 1, + used: [16]deviceMemory{{ + contextSize: 300, + }}, + }, + }, + }, + }, + }, + want: uint64(300), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -262,6 +341,7 @@ func Test_DeviceMemoryModuleSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { moduleSize: 100, @@ -272,6 +352,7 @@ func Test_DeviceMemoryModuleSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { moduleSize: 100, @@ -299,6 +380,7 @@ func Test_DeviceMemoryModuleSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { moduleSize: 100, @@ -309,6 +391,7 @@ func Test_DeviceMemoryModuleSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { moduleSize: 100, @@ -324,6 +407,25 @@ func Test_DeviceMemoryModuleSize(t *testing.T) { }, want: uint64(400), }, + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{moduleSize: 150}}}, + {status: 0, used: [16]deviceMemory{{moduleSize: 999}}}, + }, + }, + }, + }, + want: uint64(150), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -343,6 +445,25 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { } want uint64 }{ + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{bufferSize: 400}}}, + {status: 0, used: [16]deviceMemory{{bufferSize: 999}}}, + }, + }, + }, + }, + want: uint64(400), + }, { name: "device memory buffer size for idx 0", args: struct { @@ -355,6 +476,7 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { bufferSize: 100, @@ -365,6 +487,7 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { bufferSize: 100, @@ -381,7 +504,7 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { want: uint64(200), }, { - name: "device memory module size for idx 1", + name: "device memory buffer size for idx 1", args: struct { idx int spec *Spec @@ -392,6 +515,7 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { bufferSize: 100, @@ -402,6 +526,7 @@ func Test_DeviceMemoryBufferSize(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { bufferSize: 100, @@ -448,6 +573,7 @@ func Test_DeviceMemoryOffset(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { offset: 100, @@ -458,6 +584,7 @@ func Test_DeviceMemoryOffset(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { offset: 100, @@ -485,6 +612,7 @@ func Test_DeviceMemoryOffset(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { offset: 100, @@ -495,6 +623,7 @@ func Test_DeviceMemoryOffset(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { offset: 100, @@ -510,6 +639,25 @@ func Test_DeviceMemoryOffset(t *testing.T) { }, want: uint64(400), }, + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{offset: 100}}}, + {status: 0, used: [16]deviceMemory{{offset: 999}}}, + }, + }, + }, + }, + want: uint64(100), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -541,6 +689,7 @@ func Test_DeviceMemoryTotal(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { total: 100, @@ -551,6 +700,7 @@ func Test_DeviceMemoryTotal(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { total: 100, @@ -578,6 +728,7 @@ func Test_DeviceMemoryTotal(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, used: [16]deviceMemory{ { total: 100, @@ -588,6 +739,7 @@ func Test_DeviceMemoryTotal(t *testing.T) { }, }, { + status: 1, used: [16]deviceMemory{ { total: 100, @@ -603,6 +755,25 @@ func Test_DeviceMemoryTotal(t *testing.T) { }, want: uint64(400), }, + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, used: [16]deviceMemory{{total: 512}}}, + {status: 0, used: [16]deviceMemory{{total: 999}}}, + }, + }, + }, + }, + want: uint64(512), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -634,6 +805,7 @@ func Test_DeviceSmUtil(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, deviceUtil: [16]deviceUtilization{ { smUtil: 100, @@ -644,6 +816,7 @@ func Test_DeviceSmUtil(t *testing.T) { }, }, { + status: 1, deviceUtil: [16]deviceUtilization{ { smUtil: 100, @@ -671,6 +844,7 @@ func Test_DeviceSmUtil(t *testing.T) { procnum: 2, procs: [1024]shrregProcSlotT{ { + status: 1, deviceUtil: [16]deviceUtilization{ { smUtil: 100, @@ -681,6 +855,7 @@ func Test_DeviceSmUtil(t *testing.T) { }, }, { + status: 1, deviceUtil: [16]deviceUtilization{ { smUtil: 100, @@ -696,6 +871,25 @@ func Test_DeviceSmUtil(t *testing.T) { }, want: uint64(400), }, + { + name: "dead slot within procnum is excluded", + args: struct { + idx int + spec *Spec + }{ + idx: int(0), + spec: &Spec{ + sr: &sharedRegionT{ + procnum: 2, + procs: [1024]shrregProcSlotT{ + {status: 1, deviceUtil: [16]deviceUtilization{{smUtil: 60}}}, + {status: 0, deviceUtil: [16]deviceUtilization{{smUtil: 999}}}, + }, + }, + }, + }, + want: uint64(60), + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -1152,6 +1346,7 @@ func TestSpec_CorruptProcnumIsClamped(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sr := &sharedRegionT{num: 2, procnum: tt.procnum} + sr.procs[0].status = 1 sr.procs[0].used[0].total = 100 sr.procs[0].deviceUtil[0].smUtil = 100 s := Spec{sr: sr}