Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ cmake_minimum_required (VERSION 2.8.12)

project(libvgpu)

enable_testing()

# Set CUDA_HOME if not defined
if (NOT DEFINED ENV{CUDA_HOME})
set(CUDA_HOME /usr/local/cuda)
Expand Down
8 changes: 2 additions & 6 deletions src/libvgpu.c
Original file line number Diff line number Diff line change
Expand Up @@ -894,19 +894,15 @@ void postInit(){
allocator_init();
map_cuda_visible_devices();

// Use shared memory semaphore to serialize host PID detection
// Returns 1 if lock acquired, 0 if timeout (skip detection)
// Use the process-death-safe shared file lock to serialize host PID detection
int lock_acquired = lock_postinit();
nvmlReturn_t res = NVML_SUCCESS;

if (lock_acquired) {
// Lock acquired - safe to call set_task_pid()
res = set_task_pid();
unlock_postinit();
} else {
// Timeout - another process likely crashed holding the lock
// Skip host PID detection for this process
LOG_WARN("Skipped host PID detection due to lock timeout");
LOG_WARN("Skipped host PID detection because the postinit lock failed");
res = NVML_ERROR_TIMEOUT;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

Expand Down
136 changes: 94 additions & 42 deletions src/multiprocess/multiprocess_memory_limit.c
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,18 @@
#define SEM_WAIT_RETRY_TIMES 30
#endif

// Longer timeout for postinit since set_task_pid() with adaptive polling can take several seconds
#ifndef SEM_WAIT_TIME_POSTINIT
#define SEM_WAIT_TIME_POSTINIT 30
#endif
#define POSTINIT_FILE_LOCK_OFFSET ((off_t)UINT64_C(0x40000000))

#ifndef SEM_WAIT_RETRY_TIMES_POSTINIT
#define SEM_WAIT_RETRY_TIMES_POSTINIT 10
#endif
_Static_assert(POSTINIT_FILE_LOCK_OFFSET > (off_t)sizeof(shared_region_t),
"postinit file lock must remain outside the shared region");

int pidfound;

int ctx_activate[32];

static shared_region_info_t region_info = {0, -1, PTHREAD_ONCE_INIT, NULL, 0, NULL};
static atomic_flag postinit_local_lock = ATOMIC_FLAG_INIT;
static _Thread_local int postinit_lock_held;
// size_t initial_offset=117440512;
int env_utilization_switch;
int enable_active_oom_killer;
Expand Down Expand Up @@ -600,6 +598,69 @@ void get_timespec(int seconds, struct timespec* spec) {
spec->tv_nsec = 0;
}

/*
* sem_postinit remains in shared_region_t to preserve its layout, but an
* unnamed semaphore cannot recover when its holder dies. Serialize postInit
* with a POSIX record lock instead: the kernel releases it when the process
* exits, including on SIGKILL, and does not inherit it across fork.
*
* The byte is outside the mapped shared-region layout. Its fixed offset does
* not depend on sizeof(shared_region_t), so future layouts address the same
* byte. Processes sharing one cache must all use this protocol; a sem-only
* binary does not participate in the record lock.
*/
static int postinit_file_lock(int lock_type) {
struct flock lock = {
.l_type = lock_type,
.l_whence = SEEK_SET,
.l_start = POSTINIT_FILE_LOCK_OFFSET,
.l_len = 1,
};
int status;

if (lock_type == F_WRLCK) {
do {
status = fcntl(region_info.fd, F_SETLK, &lock);
} while (status != 0 && errno == EINTR);
if (status == 0) {
return 1;
}
if (errno != EACCES && errno != EAGAIN) {
LOG_ERROR("Failed to acquire postinit file lock: errno=%d", errno);
return 0;
}
LOG_MSG("Waiting for postinit file lock (PID %d)", getpid());
/*
* A live owner is never bypassed: skipping host PID detection also
* skips SM rate limiting for the process.
*/
do {
status = fcntl(region_info.fd, F_SETLKW, &lock);
} while (status != 0 && errno == EINTR);
} else {
do {
status = fcntl(region_info.fd, F_SETLK, &lock);
} while (status != 0 && errno == EINTR);
}
if (status != 0) {
LOG_ERROR("Failed to %s postinit file lock: errno=%d",
lock_type == F_UNLCK ? "release" : "acquire", errno);
return 0;
}
return 1;
}

static void lock_postinit_local(void) {
while (atomic_flag_test_and_set_explicit(&postinit_local_lock,
memory_order_acquire)) {
usleep(1000);
}
}

static void unlock_postinit_local(void) {
atomic_flag_clear_explicit(&postinit_local_lock, memory_order_release);
}

int fix_lock_shrreg() {
int res = 1;
if (region_info.fd == -1) {
Expand Down Expand Up @@ -818,45 +879,34 @@ void unlock_shrreg() {
}

int lock_postinit() {
shared_region_t* region = region_info.shared_region;
int trials = 0;
while (1) {
// CRITICAL: Create fresh timeout for each iteration!
// If created outside loop, timestamp becomes stale after first timeout
// Use longer timeout for postinit since set_task_pid() can take several seconds
struct timespec sem_ts;
get_timespec(SEM_WAIT_TIME_POSTINIT, &sem_ts);

int status = sem_timedwait(&region->sem_postinit, &sem_ts);
if (status == 0) {
// Lock acquired successfully
LOG_DEBUG("Acquired postinit lock after %d waits (PID %d)", trials, getpid());
return 1; // Success
} else if (errno == ETIMEDOUT) {
trials++;
LOG_MSG("Waiting for postinit lock (trial %d/%d, waited %ds, PID %d)",
trials, SEM_WAIT_RETRY_TIMES_POSTINIT, trials * SEM_WAIT_TIME_POSTINIT, getpid());

// After many retries, give up
if (trials > SEM_WAIT_RETRY_TIMES_POSTINIT) {
LOG_ERROR("Postinit lock timeout after %d seconds - another process may have crashed",
SEM_WAIT_RETRY_TIMES_POSTINIT * SEM_WAIT_TIME_POSTINIT);
LOG_ERROR("Skipping host PID detection for this process (will use container PID)");
return 0; // Timeout - didn't acquire lock
}
continue;
} else {
LOG_ERROR("Failed to lock postinit semaphore: errno=%d", errno);
// Don't give up - keep retrying
trials++;
continue;
}
if (postinit_lock_held) {
LOG_ERROR("Postinit lock cannot be acquired recursively");
return 0;
}
lock_postinit_local();
if (!postinit_file_lock(F_WRLCK)) {
unlock_postinit_local();
return 0;
}
postinit_lock_held = 1;
LOG_DEBUG("Acquired postinit file lock (PID %d)", getpid());
return 1;
}

void unlock_postinit() {
shared_region_t* region = region_info.shared_region;
sem_post(&region->sem_postinit);
if (!postinit_lock_held) {
LOG_ERROR("Postinit unlock attempted by a thread that does not hold the lock");
return;
}
if (!postinit_file_lock(F_UNLCK)) {
/*
* Keep the local guard held. POSIX record locks are process-wide, so
* letting another thread proceed could overlap the still-held lock.
*/
return;
}
postinit_lock_held = 0;
unlock_postinit_local();
}


Expand Down Expand Up @@ -1024,6 +1074,8 @@ void print_all() {
void child_reinit_flag() {
LOG_DEBUG("Detect child pid: %d -> %d", region_info.pid, getpid());
region_info.init_status = PTHREAD_ONCE_INIT;
postinit_lock_held = 0;
atomic_flag_clear_explicit(&postinit_local_lock, memory_order_release);
}

int set_active_oom_killer() {
Expand Down
4 changes: 2 additions & 2 deletions src/multiprocess/multiprocess_memory_limit.h
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ typedef struct {
_Atomic int recent_kernel;
int priority;
_Atomic uint64_t last_kernel_time;
sem_t sem_postinit; // For serializing postInit() host PID detection
sem_t sem_postinit; // Retained for shared-region layout compatibility
} shared_region_t;

typedef struct {
Expand Down Expand Up @@ -166,7 +166,7 @@ int init_device_info();
void lock_shrreg();
void unlock_shrreg();

int lock_postinit(); // Returns 1 on success, 0 on timeout
int lock_postinit(); // Returns 1 on success, 0 on lock error
void unlock_postinit();

//Setspec of the corresponding device
Expand Down
30 changes: 27 additions & 3 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,34 @@ foreach(TEST_SCRIPT ${TEST_SCRIPTS})
get_filename_component(TEST_TARGET_NAME ${RELATIVE_TEST_PATH} NAME_WE)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${TEST_TARGET_DIR})

if (TEST_SCRIPT MATCHES ".*cu")
if (TEST_TARGET_NAME STREQUAL "test_postinit_owner_death")
# Build this focused regression test with the production shared-region
# implementation. It does not invoke any CUDA/NVML entry point at
# runtime. Section garbage collection drops unrelated GPU-facing
# production functions.
add_executable(${TEST_TARGET_NAME}
${TEST_SCRIPT}
${CMAKE_CURRENT_SOURCE_DIR}/../src/multiprocess/multiprocess_memory_limit.c
${CMAKE_CURRENT_SOURCE_DIR}/../src/log_utils.c)
target_compile_definitions(${TEST_TARGET_NAME} PRIVATE
_GNU_SOURCE)
target_compile_options(${TEST_TARGET_NAME} PRIVATE
-ffunction-sections -fdata-sections)
set_target_properties(${TEST_TARGET_NAME} PROPERTIES
LINK_FLAGS "-Wl,--no-export-dynamic,--gc-sections")
elseif (TEST_SCRIPT MATCHES ".*cu")
cuda_add_executable(${TEST_TARGET_NAME} ${TEST_SCRIPT})
else()
add_executable(${TEST_TARGET_NAME} ${TEST_SCRIPT})
endif()

list(APPEND TEST_TARGET_NAMES_LIST ${TEST_TARGET_NAME})
target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread -lnvidia-ml -lcuda -lcudart -L${CUDA_HOME}/lib64)
if (TEST_TARGET_NAME STREQUAL "test_postinit_owner_death")
target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread)
else()
target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread
-lnvidia-ml -lcuda -lcudart -L${CUDA_HOME}/lib64)
endif()
target_compile_options(${TEST_TARGET_NAME} PUBLIC ${TEST_COMPILE_FLAGS})

if (DEFINED TEST_DEVICE_ID)
Expand All @@ -28,10 +48,14 @@ foreach(TEST_SCRIPT ${TEST_SCRIPTS})
endforeach()
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

add_test(NAME postinit_owner_death
COMMAND test_postinit_owner_death)
set_tests_properties(postinit_owner_death PROPERTIES TIMEOUT 20)


add_custom_target(python_test ALL
COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR})

# add_custom_target(multiprocess_test_scripts ALL
# COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/multiprocess/scripts
# ${CMAKE_CURRENT_BINARY_DIR}/multiprocess)
# ${CMAKE_CURRENT_BINARY_DIR}/multiprocess)
Loading
Loading