Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
df233e6
feat: account for CUDA context memory
iemAnshuman Aug 26, 2026
44e65f0
test: cover the context accounting unit
iemAnshuman Aug 26, 2026
849cf11
fix: defer the charge when the context size is unknown
iemAnshuman Aug 26, 2026
695429f
fix: fall back to the probed context size on every device
iemAnshuman Aug 26, 2026
ba096f9
fix: fall back to the probed size only after measuring
iemAnshuman Aug 26, 2026
652d9fc
fix: drop the charge when the primary context is reset
iemAnshuman Aug 26, 2026
a34c7d2
fix: keep the probed context size across fork
iemAnshuman Aug 26, 2026
3d2f7cf
fix: lock accounting after the driver call in release
iemAnshuman Aug 26, 2026
2dba259
test: cover rollback mismatch and restore while retained
iemAnshuman Aug 26, 2026
cadf987
resolve conflict
iemAnshuman Aug 27, 2026
cf4b6ea
test: drive the fork handlers through pthread_atfork
iemAnshuman Aug 28, 2026
037c8e7
chore: remove the ctx_activate flag the retain count replaces
iemAnshuman Aug 28, 2026
b8f289a
fix: keep the caller working when the shared region refuses the charge
iemAnshuman Aug 28, 2026
5080bc8
docs: note what the context measurement window can catch
iemAnshuman Aug 28, 2026
967f17c
chore: drop the license headers the repo does not use
iemAnshuman Aug 28, 2026
c5d2c13
resolve conflict
iemAnshuman Aug 31, 2026
43198a0
fix: reject an implausible context measurement
iemAnshuman Aug 31, 2026
24ee1b4
fix: preserve context references and use the probe charge
iemAnshuman Sep 9, 2026
2fa5d3f
test: run accounting regressions inside the build container
iemAnshuman Sep 9, 2026
7f99005
test: cover a refused context memory charge
iemAnshuman Sep 13, 2026
fdac406
chore: drop the pidfound check in the retain hook
iemAnshuman Sep 14, 2026
0c70a7e
chore: remove the accounted retain variant
iemAnshuman Sep 14, 2026
a780672
chore: clear a refused charge directly
iemAnshuman Sep 14, 2026
9c3ab20
chore: remove the unused rollback helper
iemAnshuman Sep 14, 2026
988b025
fix: don't fail a retain we cannot count
iemAnshuman Sep 14, 2026
21d0e51
test: drop nvml fakes that nothing calls
iemAnshuman Sep 14, 2026
76437f0
test: remove two checks that cannot fail
iemAnshuman Sep 14, 2026
746a64a
chore: drop the makefile ctest filter
iemAnshuman Sep 14, 2026
4b3f2d2
chore: fold the fork handlers into context.c
iemAnshuman Sep 14, 2026
e09aecc
resolve conflict
iemAnshuman Sep 14, 2026
a468ff3
chore: drop an include context.c does not use
iemAnshuman Sep 14, 2026
01caa28
chore: drop the charge local and its stale comment
iemAnshuman Sep 14, 2026
e2a41b3
chore: clear fork child accounting with memset
iemAnshuman Sep 14, 2026
b21970c
chore: remove unused stdint includes
iemAnshuman Sep 14, 2026
b0b880d
test: cover a failed removal on release
iemAnshuman Sep 14, 2026
18920ba
chore: drop the restore helper for failed removals
iemAnshuman Sep 14, 2026
d26193d
test: drop an include the fixture does not use
iemAnshuman Sep 14, 2026
c7f4c59
test: drop the unused signal.h include
iemAnshuman Sep 14, 2026
2a9242a
test: charge a context retained after its final release
iemAnshuman Sep 14, 2026
53004c6
test: drop a duplicate nested retain check
iemAnshuman Sep 14, 2026
bca9b6f
test: reject zero-byte shared region calls in the fixture
iemAnshuman Sep 14, 2026
eacad26
test: cover devices outside the accounting range
iemAnshuman Sep 14, 2026
bdcfe04
chore: declare the fork handler registration once
iemAnshuman Sep 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cuda/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
add_library(cuda_mod OBJECT context.c device.c hook.c event.c hook.c memory.c stream.c graph.c)
add_library(cuda_mod OBJECT context.c context_accounting.c device.c hook.c event.c hook.c memory.c stream.c graph.c)
target_compile_options(cuda_mod PUBLIC ${LIBRARY_COMPILE_FLAGS})
target_link_libraries(cuda_mod PUBLIC nvidia-ml -lcuda)
203 changes: 191 additions & 12 deletions src/cuda/context.c
Original file line number Diff line number Diff line change
@@ -1,8 +1,51 @@
#include <errno.h>

#include "include/libcuda_hook.h"
#include "include/libvgpu.h"
#include "cuda/context_accounting.h"
#include "multiprocess/multiprocess_memory_limit.h"

extern size_t context_size;
extern int ctx_activate[16];
extern int ctx_activate[CUDA_DEVICE_MAX_COUNT];
extern int pidfound;

static size_t device_context_size[CUDA_DEVICE_MAX_COUNT];
static primary_context_accounting_t
context_accounting[CUDA_DEVICE_MAX_COUNT];
static pthread_mutex_t context_accounting_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t context_device_locks[CUDA_DEVICE_MAX_COUNT] = {
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER,
};

void context_accounting_fork_prepare() {
pthread_mutex_lock(&context_accounting_lock);
}

void context_accounting_fork_parent() {
pthread_mutex_unlock(&context_accounting_lock);
}

void context_accounting_fork_child() {
int dev;

context_size = 0;
primary_context_accounting_reset(context_accounting,
CUDA_DEVICE_MAX_COUNT);
for (dev = 0; dev < CUDA_DEVICE_MAX_COUNT; dev++) {
device_context_size[dev] = 0;
ctx_activate[dev] = 0;
context_device_locks[dev] =
(pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
}
pthread_mutex_unlock(&context_accounting_lock);
}


CUresult cuDevicePrimaryCtxGetState( CUdevice dev, unsigned int* flags, int* active ){
Expand All @@ -11,16 +54,134 @@ CUresult cuDevicePrimaryCtxGetState( CUdevice dev, unsigned int* flags, int* act
return res;
}

static size_t context_charge_for_device(CUdevice dev) {
if (dev >= 0 && dev < CUDA_DEVICE_MAX_COUNT &&
device_context_size[dev] > 0) {
return device_context_size[dev];
}
return dev == 0 ? context_size : 0;
}

static CUresult rollback_unaccounted_retain(CUdevice dev,
int retain_recorded,
size_t bytes_to_add) {
CUresult release_result;

if (retain_recorded &&
primary_context_rollback_retain(&context_accounting[dev],
bytes_to_add) != 0) {
LOG_ERROR("Failed to roll back local context accounting on device %d",
dev);
}
release_result = CUDA_OVERRIDE_CALL(cuda_library_entry,
cuDevicePrimaryCtxRelease_v2, dev);
if (release_result != CUDA_SUCCESS) {
LOG_ERROR("Failed to release an unaccounted primary context on "
"device %d: %d", dev, release_result);
}
ctx_activate[dev] = (int)context_accounting[dev].retain_count;
return CUDA_ERROR_OUT_OF_MEMORY;
}

CUresult cuDevicePrimaryCtxRetain(CUcontext *pctx, CUdevice dev){
LOG_INFO("dev=%d context_size=%ld",dev,context_size);
//for Initialization only
CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,cuDevicePrimaryCtxRetain,pctx,dev);
if (ctx_activate[dev] == 0) {
add_gpu_device_memory_usage(getpid(),dev,context_size,0);
uint64_t before = 0;
uint64_t after = 0;
int hostpid;
int measure_context = 0;
size_t charge;
size_t measured_charge = 0;
size_t bytes_to_add = 0;

if (dev < 0 || dev >= CUDA_DEVICE_MAX_COUNT) {
return CUDA_OVERRIDE_CALL(cuda_library_entry,
cuDevicePrimaryCtxRetain, pctx, dev);
}
if (context_size>0) {
ctx_activate[dev] = 1;

pthread_mutex_lock(&context_device_locks[dev]);
pthread_mutex_lock(&context_accounting_lock);
charge = context_charge_for_device(dev);
hostpid = get_current_host_pid();
if (charge == 0 && pidfound == 1 && hostpid > 0 &&
context_accounting[dev].charged_bytes == 0) {
measure_context = 1;
}
pthread_mutex_unlock(&context_accounting_lock);

if (measure_context) {
nvmlReturn_t result = get_used_gpu_memory_by_pid(
(unsigned int)hostpid, dev, &before);
if (result == NVML_SUCCESS || result == NVML_ERROR_NOT_FOUND) {
if (result == NVML_ERROR_NOT_FOUND) {
before = 0;
}
} else {
measure_context = 0;
}
}

CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,
cuDevicePrimaryCtxRetain, pctx, dev);
if (res != CUDA_SUCCESS) {
pthread_mutex_unlock(&context_device_locks[dev]);
return res;
}

if (measure_context) {
int attempt;

for (attempt = 0; attempt < 10; attempt++) {
if (get_used_gpu_memory_by_pid((unsigned int)hostpid, dev,
&after) == NVML_SUCCESS &&
after > before) {
measured_charge = after - before;
break;
}
usleep(1000);
}
}
pthread_mutex_lock(&context_accounting_lock);
if (measured_charge > 0) {
device_context_size[dev] = measured_charge;
charge = measured_charge;
LOG_INFO("Measured primary context size lazily: "
"dev=%d size=%lu", dev, charge);
}
errno = 0;
int record_result =
(pidfound == 1)
Comment thread
iemAnshuman marked this conversation as resolved.
Resolved
? primary_context_record_accounted_retain(
&context_accounting[dev], charge, &bytes_to_add)
: primary_context_record_retain(&context_accounting[dev], charge,
&bytes_to_add);
/* The driver retain already succeeded. An unknown context size must not
* fail the caller, so defer the charge to a later retain that knows it. */
if (record_result != 0 && errno == ENODATA) {
LOG_WARN("Primary context size unknown on device %d; charge is "
"deferred to a later retain", dev);
record_result = primary_context_record_retain(
&context_accounting[dev], charge, &bytes_to_add);
}
Comment thread
iemAnshuman marked this conversation as resolved.
Resolved
if (record_result != 0) {
LOG_ERROR("Cannot account primary context retain on device %d",
dev);
res = rollback_unaccounted_retain(dev, 0, 0);
pthread_mutex_unlock(&context_accounting_lock);
pthread_mutex_unlock(&context_device_locks[dev]);
return res;
}
if (bytes_to_add > 0) {
if (add_gpu_device_memory_usage(getpid(), dev, bytes_to_add, 0) != 0) {
LOG_ERROR("Failed to charge primary context memory on device %d",
dev);
res = rollback_unaccounted_retain(dev, 1, bytes_to_add);
pthread_mutex_unlock(&context_accounting_lock);
pthread_mutex_unlock(&context_device_locks[dev]);
return res;
}
}
ctx_activate[dev] = (int)context_accounting[dev].retain_count;
pthread_mutex_unlock(&context_accounting_lock);
pthread_mutex_unlock(&context_device_locks[dev]);
return res;
}

Expand All @@ -31,11 +192,30 @@ CUresult cuDevicePrimaryCtxSetFlags_v2( CUdevice dev, unsigned int flags ){
}

CUresult cuDevicePrimaryCtxRelease_v2( CUdevice dev ){
if (ctx_activate[dev] == 1) {
rm_gpu_device_memory_usage(getpid(),dev,context_size,0);
size_t bytes_to_remove = 0;

if (dev < 0 || dev >= CUDA_DEVICE_MAX_COUNT) {
return CUDA_OVERRIDE_CALL(cuda_library_entry,
cuDevicePrimaryCtxRelease_v2, dev);
}
ctx_activate[dev] = 0;
pthread_mutex_lock(&context_device_locks[dev]);
pthread_mutex_lock(&context_accounting_lock);
CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,cuDevicePrimaryCtxRelease_v2,dev);
if (res == CUDA_SUCCESS) {
if (primary_context_record_release(&context_accounting[dev],
&bytes_to_remove) != 0) {
LOG_WARN("Unbalanced primary context release on device %d", dev);
} else if (bytes_to_remove > 0) {
if (rm_gpu_device_memory_usage(getpid(), dev, bytes_to_remove,
0) != 0) {
primary_context_restore_charge(&context_accounting[dev],
bytes_to_remove);
}
}
ctx_activate[dev] = (int)context_accounting[dev].retain_count;
}
pthread_mutex_unlock(&context_accounting_lock);
pthread_mutex_unlock(&context_device_locks[dev]);
return res;
}

Expand Down Expand Up @@ -149,4 +329,3 @@ CUresult cuCtxSynchronize ( void ){
CUresult res = CUDA_OVERRIDE_CALL(cuda_library_entry,cuCtxSynchronize);
return res;
}

105 changes: 105 additions & 0 deletions src/cuda/context_accounting.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Copyright (c) 2026 The HAMi Authors.
*/

#include "cuda/context_accounting.h"

#include <errno.h>
#include <limits.h>

static int record_retain(primary_context_accounting_t *state,
size_t context_bytes, size_t *bytes_to_add,
int require_charge) {
if (state == NULL || bytes_to_add == NULL) {
errno = EINVAL;
return -1;
}
if (state->retain_count == UINT_MAX) {
errno = EOVERFLOW;
return -1;
}
if (require_charge && state->charged_bytes == 0 && context_bytes == 0) {
errno = ENODATA;
return -1;
}

*bytes_to_add = 0;
if (state->charged_bytes == 0 && context_bytes > 0) {
state->charged_bytes = context_bytes;
*bytes_to_add = context_bytes;
}
state->retain_count++;
return 0;
}

int primary_context_record_retain(primary_context_accounting_t *state,
size_t context_bytes,
size_t *bytes_to_add) {
return record_retain(state, context_bytes, bytes_to_add, 0);
}

int primary_context_record_accounted_retain(
primary_context_accounting_t *state, size_t context_bytes,
size_t *bytes_to_add) {
return record_retain(state, context_bytes, bytes_to_add, 1);
}

int primary_context_record_release(primary_context_accounting_t *state,
size_t *bytes_to_remove) {
if (state == NULL || bytes_to_remove == NULL) {
errno = EINVAL;
return -1;
}
if (state->retain_count == 0) {
errno = EINVAL;
return -1;
}

*bytes_to_remove = 0;
state->retain_count--;
if (state->retain_count == 0) {
*bytes_to_remove = state->charged_bytes;
state->charged_bytes = 0;
}
return 0;
}

int primary_context_rollback_retain(primary_context_accounting_t *state,
size_t bytes_to_add) {
if (state == NULL || state->retain_count == 0) {
errno = EINVAL;
return -1;
}
if (bytes_to_add > 0 && state->charged_bytes != bytes_to_add) {
errno = EINVAL;
return -1;
}

state->retain_count--;
if (bytes_to_add > 0) {
state->charged_bytes = 0;
}
return 0;
}

void primary_context_restore_charge(primary_context_accounting_t *state,
size_t context_bytes) {
if (state != NULL && state->retain_count == 0 && context_bytes > 0) {
state->charged_bytes = context_bytes;
}
}

void primary_context_accounting_reset(primary_context_accounting_t *states,
size_t state_count) {
size_t i;

if (states == NULL) {
return;
}
for (i = 0; i < state_count; i++) {
states[i].retain_count = 0;
states[i].charged_bytes = 0;
}
}
47 changes: 47 additions & 0 deletions src/cuda/context_accounting.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* Copyright (c) 2026 The HAMi Authors.
*/

#ifndef SRC_CUDA_CONTEXT_ACCOUNTING_H_
#define SRC_CUDA_CONTEXT_ACCOUNTING_H_

#include <stddef.h>

typedef struct {
unsigned int retain_count;
size_t charged_bytes;
} primary_context_accounting_t;

/*
* Record a successful retain or release. A known size is charged once and
* removed when the final retain is released.
*/
int primary_context_record_retain(primary_context_accounting_t *state,
size_t context_bytes,
size_t *bytes_to_add);

/*
* Record a retain only when its context memory is already charged or a
* nonzero charge can be applied before the caller reports success.
*/
int primary_context_record_accounted_retain(
primary_context_accounting_t *state, size_t context_bytes,
size_t *bytes_to_add);
int primary_context_record_release(primary_context_accounting_t *state,
size_t *bytes_to_remove);

/* Undo the most recent retain after its external accounting step fails. */
int primary_context_rollback_retain(primary_context_accounting_t *state,
size_t bytes_to_add);

/* Restore a charge that could not be removed from shared accounting. */
void primary_context_restore_charge(primary_context_accounting_t *state,
size_t context_bytes);

/* Clear process-local accounting inherited across fork(). */
void primary_context_accounting_reset(primary_context_accounting_t *states,
size_t state_count);

#endif // SRC_CUDA_CONTEXT_ACCOUNTING_H_
Loading
Loading