From 2b77f8925911f94aee84a7f89d756126feb81a2b Mon Sep 17 00:00:00 2001 From: iemAnshuman Date: Sun, 2 Aug 2026 15:00:18 +0530 Subject: [PATCH 1/3] recover the post init locking after owner death. Signed-off-by: iemAnshuman --- CMakeLists.txt | 2 + src/libvgpu.c | 8 +- src/multiprocess/multiprocess_memory_limit.c | 136 +++++-- src/multiprocess/multiprocess_memory_limit.h | 4 +- test/CMakeLists.txt | 30 +- test/test_postinit_owner_death.c | 406 +++++++++++++++++++ 6 files changed, 533 insertions(+), 53 deletions(-) create mode 100644 test/test_postinit_owner_death.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 553c6582..479f634f 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required (VERSION 2.8.12) project(libvgpu) +enable_testing() + # Set CUDA_HOME if not defined if (NOT DEFINED ENV{CUDA_HOME}) set(CUDA_HOME /usr/local/cuda) diff --git a/src/libvgpu.c b/src/libvgpu.c index 49ef9878..c2f56680 100644 --- a/src/libvgpu.c +++ b/src/libvgpu.c @@ -894,19 +894,15 @@ void postInit(){ allocator_init(); map_cuda_visible_devices(); - // Use shared memory semaphore to serialize host PID detection - // Returns 1 if lock acquired, 0 if timeout (skip detection) + // Use the process-death-safe shared file lock to serialize host PID detection int lock_acquired = lock_postinit(); nvmlReturn_t res = NVML_SUCCESS; if (lock_acquired) { - // Lock acquired - safe to call set_task_pid() res = set_task_pid(); unlock_postinit(); } else { - // Timeout - another process likely crashed holding the lock - // Skip host PID detection for this process - LOG_WARN("Skipped host PID detection due to lock timeout"); + LOG_WARN("Skipped host PID detection because the postinit lock failed"); res = NVML_ERROR_TIMEOUT; } diff --git a/src/multiprocess/multiprocess_memory_limit.c b/src/multiprocess/multiprocess_memory_limit.c index ff6cea1f..fdb5648b 100755 --- a/src/multiprocess/multiprocess_memory_limit.c +++ b/src/multiprocess/multiprocess_memory_limit.c @@ -34,20 +34,18 @@ #define SEM_WAIT_RETRY_TIMES 30 #endif -// Longer timeout for postinit since set_task_pid() with adaptive polling can take several seconds -#ifndef SEM_WAIT_TIME_POSTINIT -#define SEM_WAIT_TIME_POSTINIT 30 -#endif +#define POSTINIT_FILE_LOCK_OFFSET ((off_t)UINT64_C(0x40000000)) -#ifndef SEM_WAIT_RETRY_TIMES_POSTINIT -#define SEM_WAIT_RETRY_TIMES_POSTINIT 10 -#endif +_Static_assert(POSTINIT_FILE_LOCK_OFFSET > (off_t)sizeof(shared_region_t), + "postinit file lock must remain outside the shared region"); int pidfound; int ctx_activate[32]; static shared_region_info_t region_info = {0, -1, PTHREAD_ONCE_INIT, NULL, 0, NULL}; +static atomic_flag postinit_local_lock = ATOMIC_FLAG_INIT; +static _Thread_local int postinit_lock_held; // size_t initial_offset=117440512; int env_utilization_switch; int enable_active_oom_killer; @@ -600,6 +598,69 @@ void get_timespec(int seconds, struct timespec* spec) { spec->tv_nsec = 0; } +/* + * sem_postinit remains in shared_region_t to preserve its layout, but an + * unnamed semaphore cannot recover when its holder dies. Serialize postInit + * with a POSIX record lock instead: the kernel releases it when the process + * exits, including on SIGKILL, and does not inherit it across fork. + * + * The byte is outside the mapped shared-region layout. Its fixed offset does + * not depend on sizeof(shared_region_t), so future layouts address the same + * byte. Processes sharing one cache must all use this protocol; a sem-only + * binary does not participate in the record lock. + */ +static int postinit_file_lock(int lock_type) { + struct flock lock = { + .l_type = lock_type, + .l_whence = SEEK_SET, + .l_start = POSTINIT_FILE_LOCK_OFFSET, + .l_len = 1, + }; + int status; + + if (lock_type == F_WRLCK) { + do { + status = fcntl(region_info.fd, F_SETLK, &lock); + } while (status != 0 && errno == EINTR); + if (status == 0) { + return 1; + } + if (errno != EACCES && errno != EAGAIN) { + LOG_ERROR("Failed to acquire postinit file lock: errno=%d", errno); + return 0; + } + LOG_MSG("Waiting for postinit file lock (PID %d)", getpid()); + /* + * A live owner is never bypassed: skipping host PID detection also + * skips SM rate limiting for the process. + */ + do { + status = fcntl(region_info.fd, F_SETLKW, &lock); + } while (status != 0 && errno == EINTR); + } else { + do { + status = fcntl(region_info.fd, F_SETLK, &lock); + } while (status != 0 && errno == EINTR); + } + if (status != 0) { + LOG_ERROR("Failed to %s postinit file lock: errno=%d", + lock_type == F_UNLCK ? "release" : "acquire", errno); + return 0; + } + return 1; +} + +static void lock_postinit_local(void) { + while (atomic_flag_test_and_set_explicit(&postinit_local_lock, + memory_order_acquire)) { + usleep(1000); + } +} + +static void unlock_postinit_local(void) { + atomic_flag_clear_explicit(&postinit_local_lock, memory_order_release); +} + int fix_lock_shrreg() { int res = 1; if (region_info.fd == -1) { @@ -818,45 +879,34 @@ void unlock_shrreg() { } int lock_postinit() { - shared_region_t* region = region_info.shared_region; - int trials = 0; - while (1) { - // CRITICAL: Create fresh timeout for each iteration! - // If created outside loop, timestamp becomes stale after first timeout - // Use longer timeout for postinit since set_task_pid() can take several seconds - struct timespec sem_ts; - get_timespec(SEM_WAIT_TIME_POSTINIT, &sem_ts); - - int status = sem_timedwait(®ion->sem_postinit, &sem_ts); - if (status == 0) { - // Lock acquired successfully - LOG_DEBUG("Acquired postinit lock after %d waits (PID %d)", trials, getpid()); - return 1; // Success - } else if (errno == ETIMEDOUT) { - trials++; - LOG_MSG("Waiting for postinit lock (trial %d/%d, waited %ds, PID %d)", - trials, SEM_WAIT_RETRY_TIMES_POSTINIT, trials * SEM_WAIT_TIME_POSTINIT, getpid()); - - // After many retries, give up - if (trials > SEM_WAIT_RETRY_TIMES_POSTINIT) { - LOG_ERROR("Postinit lock timeout after %d seconds - another process may have crashed", - SEM_WAIT_RETRY_TIMES_POSTINIT * SEM_WAIT_TIME_POSTINIT); - LOG_ERROR("Skipping host PID detection for this process (will use container PID)"); - return 0; // Timeout - didn't acquire lock - } - continue; - } else { - LOG_ERROR("Failed to lock postinit semaphore: errno=%d", errno); - // Don't give up - keep retrying - trials++; - continue; - } + if (postinit_lock_held) { + LOG_ERROR("Postinit lock cannot be acquired recursively"); + return 0; + } + lock_postinit_local(); + if (!postinit_file_lock(F_WRLCK)) { + unlock_postinit_local(); + return 0; } + postinit_lock_held = 1; + LOG_DEBUG("Acquired postinit file lock (PID %d)", getpid()); + return 1; } void unlock_postinit() { - shared_region_t* region = region_info.shared_region; - sem_post(®ion->sem_postinit); + if (!postinit_lock_held) { + LOG_ERROR("Postinit unlock attempted by a thread that does not hold the lock"); + return; + } + if (!postinit_file_lock(F_UNLCK)) { + /* + * Keep the local guard held. POSIX record locks are process-wide, so + * letting another thread proceed could overlap the still-held lock. + */ + return; + } + postinit_lock_held = 0; + unlock_postinit_local(); } @@ -1024,6 +1074,8 @@ void print_all() { void child_reinit_flag() { LOG_DEBUG("Detect child pid: %d -> %d", region_info.pid, getpid()); region_info.init_status = PTHREAD_ONCE_INIT; + postinit_lock_held = 0; + atomic_flag_clear_explicit(&postinit_local_lock, memory_order_release); } int set_active_oom_killer() { diff --git a/src/multiprocess/multiprocess_memory_limit.h b/src/multiprocess/multiprocess_memory_limit.h index 39bcbbf2..5981bbda 100755 --- a/src/multiprocess/multiprocess_memory_limit.h +++ b/src/multiprocess/multiprocess_memory_limit.h @@ -105,7 +105,7 @@ typedef struct { _Atomic int recent_kernel; int priority; _Atomic uint64_t last_kernel_time; - sem_t sem_postinit; // For serializing postInit() host PID detection + sem_t sem_postinit; // Retained for shared-region layout compatibility } shared_region_t; typedef struct { @@ -166,7 +166,7 @@ int init_device_info(); void lock_shrreg(); void unlock_shrreg(); -int lock_postinit(); // Returns 1 on success, 0 on timeout +int lock_postinit(); // Returns 1 on success, 0 on lock error void unlock_postinit(); //Setspec of the corresponding device diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 86575d0e..ae39f322 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -12,14 +12,34 @@ foreach(TEST_SCRIPT ${TEST_SCRIPTS}) get_filename_component(TEST_TARGET_NAME ${RELATIVE_TEST_PATH} NAME_WE) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${TEST_TARGET_DIR}) - if (TEST_SCRIPT MATCHES ".*cu") + if (TEST_TARGET_NAME STREQUAL "test_postinit_owner_death") + # Build this focused regression test with the production shared-region + # implementation. It does not invoke any CUDA/NVML entry point at + # runtime. Section garbage collection drops unrelated GPU-facing + # production functions. + add_executable(${TEST_TARGET_NAME} + ${TEST_SCRIPT} + ${CMAKE_CURRENT_SOURCE_DIR}/../src/multiprocess/multiprocess_memory_limit.c + ${CMAKE_CURRENT_SOURCE_DIR}/../src/log_utils.c) + target_compile_definitions(${TEST_TARGET_NAME} PRIVATE + _GNU_SOURCE) + target_compile_options(${TEST_TARGET_NAME} PRIVATE + -ffunction-sections -fdata-sections) + set_target_properties(${TEST_TARGET_NAME} PROPERTIES + LINK_FLAGS "-Wl,--no-export-dynamic,--gc-sections") + elseif (TEST_SCRIPT MATCHES ".*cu") cuda_add_executable(${TEST_TARGET_NAME} ${TEST_SCRIPT}) else() add_executable(${TEST_TARGET_NAME} ${TEST_SCRIPT}) endif() list(APPEND TEST_TARGET_NAMES_LIST ${TEST_TARGET_NAME}) - target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread -lnvidia-ml -lcuda -lcudart -L${CUDA_HOME}/lib64) + if (TEST_TARGET_NAME STREQUAL "test_postinit_owner_death") + target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread) + else() + target_link_libraries(${TEST_TARGET_NAME} -lrt -lpthread + -lnvidia-ml -lcuda -lcudart -L${CUDA_HOME}/lib64) + endif() target_compile_options(${TEST_TARGET_NAME} PUBLIC ${TEST_COMPILE_FLAGS}) if (DEFINED TEST_DEVICE_ID) @@ -28,10 +48,14 @@ foreach(TEST_SCRIPT ${TEST_SCRIPTS}) endforeach() set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +add_test(NAME postinit_owner_death + COMMAND test_postinit_owner_death) +set_tests_properties(postinit_owner_death PROPERTIES TIMEOUT 20) + add_custom_target(python_test ALL COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/python ${CMAKE_CURRENT_BINARY_DIR}) # add_custom_target(multiprocess_test_scripts ALL # COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/multiprocess/scripts -# ${CMAKE_CURRENT_BINARY_DIR}/multiprocess) \ No newline at end of file +# ${CMAKE_CURRENT_BINARY_DIR}/multiprocess) diff --git a/test/test_postinit_owner_death.c b/test/test_postinit_owner_death.c new file mode 100644 index 00000000..645cef0f --- /dev/null +++ b/test/test_postinit_owner_death.c @@ -0,0 +1,406 @@ +/* + * GPU-free regression tests for the postInit record lock. + * + * The process test kills both an initial owner and the first waiter that + * acquires after it. A second waiter must remain blocked while either owner + * is alive and must acquire promptly after each SIGKILL. The thread test + * verifies the process-local guard required by POSIX record-lock semantics. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "multiprocess/multiprocess_memory_limit.h" + +#define PROCESS_COUNT 3 +#define HOLDER_ID 0 +#define FIRST_WAITER_ID 1 +#define SECOND_WAITER_ID 2 +#define TEST_TIMEOUT_MS 5000.0 +#define OBSERVATION_MS 200 +#define THREAD_COUNT 2 +#define THREAD_HOLD_MS 100 + +typedef struct { + _Atomic int process_started; + _Atomic int process_acquired; + _Atomic int process_completed; + _Atomic int process_failures; + _Atomic int current_owner; + _Atomic int release_process[PROCESS_COUNT]; + + _Atomic int threads_started; + _Atomic int threads_go; + _Atomic int threads_acquired; + _Atomic int threads_completed; + _Atomic int threads_active; + _Atomic int max_threads_active; + _Atomic int thread_failures; +} test_state_t; + +static test_state_t *state; + +static double now_ms(void) { + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0.0; + } + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; +} + +static void sleep_ms(int milliseconds) { + struct timespec ts; + + ts.tv_sec = milliseconds / 1000; + ts.tv_nsec = (milliseconds % 1000) * 1000000L; + while (nanosleep(&ts, &ts) != 0 && errno == EINTR) { + } +} + +static int wait_for_counter(_Atomic int *counter, int expected, + double timeout_ms) { + double deadline = now_ms() + timeout_ms; + + while (atomic_load_explicit(counter, memory_order_acquire) < expected) { + if (now_ms() >= deadline) { + return -1; + } + sleep_ms(1); + } + return 0; +} + +static int wait_child_bounded(pid_t child, double timeout_ms, int *status_out) { + double deadline = now_ms() + timeout_ms; + int status; + pid_t waited; + + for (;;) { + do { + waited = waitpid(child, &status, WNOHANG); + } while (waited < 0 && errno == EINTR); + if (waited == child) { + if (status_out != NULL) { + *status_out = status; + } + return 0; + } + if (waited < 0) { + return -1; + } + if (now_ms() >= deadline) { + return -1; + } + sleep_ms(1); + } +} + +static void kill_and_reap(pid_t child) { + int status; + pid_t waited; + + if (child <= 0) { + return; + } + do { + waited = waitpid(child, &status, WNOHANG); + } while (waited < 0 && errno == EINTR); + if (waited == child || (waited < 0 && errno == ECHILD)) { + return; + } + kill(child, SIGKILL); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) { + } +} + +static int kill_and_expect_sigkill(pid_t child) { + int status; + + if (kill(child, SIGKILL) != 0 || + wait_child_bounded(child, TEST_TIMEOUT_MS, &status) != 0) { + return -1; + } + return WIFSIGNALED(status) && WTERMSIG(status) == SIGKILL ? 0 : -1; +} + +static void initialize_state(void) { + int i; + + atomic_init(&state->process_started, 0); + atomic_init(&state->process_acquired, 0); + atomic_init(&state->process_completed, 0); + atomic_init(&state->process_failures, 0); + atomic_init(&state->current_owner, -1); + for (i = 0; i < PROCESS_COUNT; i++) { + atomic_init(&state->release_process[i], 0); + } + atomic_init(&state->threads_started, 0); + atomic_init(&state->threads_go, 0); + atomic_init(&state->threads_acquired, 0); + atomic_init(&state->threads_completed, 0); + atomic_init(&state->threads_active, 0); + atomic_init(&state->max_threads_active, 0); + atomic_init(&state->thread_failures, 0); +} + +static void process_worker(int id) { + ensure_initialized(); + atomic_fetch_add_explicit(&state->process_started, 1, + memory_order_release); + if (lock_postinit() != 1) { + atomic_fetch_add_explicit(&state->process_failures, 1, + memory_order_release); + _exit(2); + } + + atomic_store_explicit(&state->current_owner, id, memory_order_release); + atomic_fetch_add_explicit(&state->process_acquired, 1, + memory_order_release); + while (atomic_load_explicit(&state->release_process[id], + memory_order_acquire) == 0) { + sleep_ms(1); + } + unlock_postinit(); + atomic_fetch_add_explicit(&state->process_completed, 1, + memory_order_release); + _exit(0); +} + +static pid_t spawn_process_worker(int id) { + pid_t child = fork(); + + if (child == 0) { + process_worker(id); + } + return child; +} + +static int test_process_owner_death(void) { + pid_t children[PROCESS_COUNT] = {0}; + int first_recovery; + int survivor; + int status; + int result = -1; + + children[HOLDER_ID] = spawn_process_worker(HOLDER_ID); + if (children[HOLDER_ID] < 0 || + wait_for_counter(&state->process_acquired, 1, TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "initial postinit owner did not acquire\n"); + goto cleanup; + } + + children[FIRST_WAITER_ID] = spawn_process_worker(FIRST_WAITER_ID); + children[SECOND_WAITER_ID] = spawn_process_worker(SECOND_WAITER_ID); + if (children[FIRST_WAITER_ID] < 0 || children[SECOND_WAITER_ID] < 0 || + wait_for_counter(&state->process_started, PROCESS_COUNT, + TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "postinit waiters did not start\n"); + goto cleanup; + } + + sleep_ms(OBSERVATION_MS); + if (atomic_load_explicit(&state->process_acquired, + memory_order_acquire) != 1) { + fprintf(stderr, "a waiter overlapped the live initial owner\n"); + goto cleanup; + } + + if (kill_and_expect_sigkill(children[HOLDER_ID]) != 0) { + fprintf(stderr, "could not SIGKILL the initial owner\n"); + goto cleanup; + } + children[HOLDER_ID] = 0; + if (wait_for_counter(&state->process_acquired, 2, TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "no waiter recovered after initial owner death\n"); + goto cleanup; + } + first_recovery = atomic_load_explicit(&state->current_owner, + memory_order_acquire); + if (first_recovery != FIRST_WAITER_ID && + first_recovery != SECOND_WAITER_ID) { + fprintf(stderr, "recovery owner id was invalid\n"); + goto cleanup; + } + + sleep_ms(OBSERVATION_MS); + if (atomic_load_explicit(&state->process_acquired, + memory_order_acquire) != 2) { + fprintf(stderr, "both waiters entered the critical section\n"); + goto cleanup; + } + + if (kill_and_expect_sigkill(children[first_recovery]) != 0) { + fprintf(stderr, "could not SIGKILL the first recovery owner\n"); + goto cleanup; + } + children[first_recovery] = 0; + if (wait_for_counter(&state->process_acquired, 3, TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "second waiter did not recover after repeated owner death\n"); + goto cleanup; + } + survivor = atomic_load_explicit(&state->current_owner, + memory_order_acquire); + if ((survivor != FIRST_WAITER_ID && survivor != SECOND_WAITER_ID) || + survivor == first_recovery) { + fprintf(stderr, "surviving recovery owner id was invalid\n"); + goto cleanup; + } + + atomic_store_explicit(&state->release_process[survivor], 1, + memory_order_release); + if (wait_for_counter(&state->process_completed, 1, TEST_TIMEOUT_MS) != 0 || + wait_child_bounded(children[survivor], TEST_TIMEOUT_MS, &status) != 0 || + !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "surviving recovery waiter did not unlock cleanly\n"); + goto cleanup; + } + children[survivor] = 0; + if (atomic_load_explicit(&state->process_failures, + memory_order_acquire) != 0) { + fprintf(stderr, "a process failed to acquire the postinit lock\n"); + goto cleanup; + } + + if (lock_postinit() != 1) { + fprintf(stderr, "postinit lock was not reusable after recovery\n"); + goto cleanup; + } + unlock_postinit(); + result = 0; + +cleanup: + for (int i = 0; i < PROCESS_COUNT; i++) { + kill_and_reap(children[i]); + } + return result; +} + +static void record_thread_entry(void) { + int active = atomic_fetch_add_explicit(&state->threads_active, 1, + memory_order_acq_rel) + 1; + int observed = atomic_load_explicit(&state->max_threads_active, + memory_order_acquire); + + while (active > observed && + !atomic_compare_exchange_weak_explicit( + &state->max_threads_active, &observed, active, + memory_order_release, memory_order_acquire)) { + } +} + +static void *thread_worker(void *unused) { + (void)unused; + atomic_fetch_add_explicit(&state->threads_started, 1, + memory_order_release); + while (atomic_load_explicit(&state->threads_go, memory_order_acquire) == 0) { + sleep_ms(1); + } + if (lock_postinit() != 1) { + atomic_fetch_add_explicit(&state->thread_failures, 1, + memory_order_release); + return NULL; + } + record_thread_entry(); + atomic_fetch_add_explicit(&state->threads_acquired, 1, + memory_order_release); + sleep_ms(THREAD_HOLD_MS); + atomic_fetch_sub_explicit(&state->threads_active, 1, + memory_order_acq_rel); + unlock_postinit(); + atomic_fetch_add_explicit(&state->threads_completed, 1, + memory_order_release); + return NULL; +} + +static int test_same_process_threads(void) { + pthread_t threads[THREAD_COUNT]; + int created = 0; + + for (int i = 0; i < THREAD_COUNT; i++) { + if (pthread_create(&threads[i], NULL, thread_worker, NULL) != 0) { + fprintf(stderr, "pthread_create failed\n"); + return -1; + } + created++; + } + if (wait_for_counter(&state->threads_started, THREAD_COUNT, + TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "postinit lock threads did not start\n"); + return -1; + } + atomic_store_explicit(&state->threads_go, 1, memory_order_release); + if (wait_for_counter(&state->threads_completed, THREAD_COUNT, + TEST_TIMEOUT_MS) != 0) { + fprintf(stderr, "postinit lock threads did not finish\n"); + return -1; + } + for (int i = 0; i < created; i++) { + pthread_join(threads[i], NULL); + } + if (atomic_load_explicit(&state->thread_failures, + memory_order_acquire) != 0 || + atomic_load_explicit(&state->threads_acquired, + memory_order_acquire) != THREAD_COUNT || + atomic_load_explicit(&state->max_threads_active, + memory_order_acquire) != 1) { + fprintf(stderr, "same-process postinit calls were not serialized\n"); + return -1; + } + return 0; +} + +int main(void) { + char cache_path[] = "/tmp/hami-postinit-ownerdeath.XXXXXX"; + int cache_fd; + int failures = 0; + + cache_fd = mkstemp(cache_path); + if (cache_fd < 0) { + perror("mkstemp(shared-region cache)"); + return 1; + } + close(cache_fd); + unlink(cache_path); + if (setenv(MULTIPROCESS_SHARED_REGION_CACHE_ENV, cache_path, 1) != 0 || + setenv("LIBCUDA_LOG_LEVEL", "0", 1) != 0) { + perror("setenv"); + return 1; + } + + state = mmap(NULL, sizeof(*state), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (state == MAP_FAILED) { + perror("mmap(test state)"); + return 1; + } + memset(state, 0, sizeof(*state)); + initialize_state(); + log_utils_init(); + ensure_initialized(); + + if (test_process_owner_death() != 0) { + failures++; + } + if (test_same_process_threads() != 0) { + failures++; + } + + unlink(cache_path); + if (failures != 0) { + fprintf(stderr, "%d postinit record-lock test group(s) failed\n", + failures); + return 1; + } + munmap(state, sizeof(*state)); + puts("postinit owner-death tests passed"); + return 0; +} From 148e92cd987e5c615abdabad5f62415c49095ae5 Mon Sep 17 00:00:00 2001 From: iemAnshuman Date: Sun, 2 Aug 2026 19:12:16 +0530 Subject: [PATCH 2/3] report postinit lock failures as unknown Signed-off-by: iemAnshuman --- src/libvgpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libvgpu.c b/src/libvgpu.c index c2f56680..89a29553 100644 --- a/src/libvgpu.c +++ b/src/libvgpu.c @@ -903,7 +903,7 @@ void postInit(){ unlock_postinit(); } else { LOG_WARN("Skipped host PID detection because the postinit lock failed"); - res = NVML_ERROR_TIMEOUT; + res = NVML_ERROR_UNKNOWN; } LOG_MSG("Initialized"); From baec1490615df8ec9e03941f41f096db054d6e52 Mon Sep 17 00:00:00 2001 From: iemAnshuman Date: Sun, 2 Aug 2026 19:13:00 +0530 Subject: [PATCH 3/3] retry postinit file locks with jitter Signed-off-by: iemAnshuman --- src/multiprocess/multiprocess_memory_limit.c | 76 ++++++++++++++++---- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/multiprocess/multiprocess_memory_limit.c b/src/multiprocess/multiprocess_memory_limit.c index fdb5648b..dc45c5d8 100755 --- a/src/multiprocess/multiprocess_memory_limit.c +++ b/src/multiprocess/multiprocess_memory_limit.c @@ -35,6 +35,8 @@ #endif #define POSTINIT_FILE_LOCK_OFFSET ((off_t)UINT64_C(0x40000000)) +#define POSTINIT_FILE_LOCK_INITIAL_RETRY_US 10000U +#define POSTINIT_FILE_LOCK_MAX_RETRY_US 1000000U _Static_assert(POSTINIT_FILE_LOCK_OFFSET > (off_t)sizeof(shared_region_t), "postinit file lock must remain outside the shared region"); @@ -609,7 +611,33 @@ void get_timespec(int seconds, struct timespec* spec) { * byte. Processes sharing one cache must all use this protocol; a sem-only * binary does not participate in the record lock. */ -static int postinit_file_lock(int lock_type) { +static void postinit_file_lock_backoff(unsigned int backoff_us, + uint32_t* jitter_state) { + struct timespec delay; + unsigned int minimum_us = backoff_us / 2; + unsigned int jitter_range_us = backoff_us - minimum_us; + + *jitter_state = *jitter_state * UINT32_C(1103515245) + UINT32_C(12345); + unsigned int delay_us = minimum_us + + *jitter_state % (jitter_range_us + 1); + delay.tv_sec = delay_us / 1000000U; + delay.tv_nsec = (long)(delay_us % 1000000U) * 1000L; + while (nanosleep(&delay, &delay) != 0 && errno == EINTR) { + } +} + +static uint32_t postinit_file_lock_jitter_seed(void) { + struct timespec now; + uint32_t seed = (uint32_t)getpid(); + + if (clock_gettime(CLOCK_MONOTONIC, &now) == 0) { + seed ^= (uint32_t)now.tv_sec; + seed ^= (uint32_t)now.tv_nsec; + } + return seed; +} + +static int postinit_file_lock(short lock_type) { struct flock lock = { .l_type = lock_type, .l_whence = SEEK_SET, @@ -619,24 +647,42 @@ static int postinit_file_lock(int lock_type) { int status; if (lock_type == F_WRLCK) { - do { - status = fcntl(region_info.fd, F_SETLK, &lock); - } while (status != 0 && errno == EINTR); - if (status == 0) { - return 1; - } - if (errno != EACCES && errno != EAGAIN) { - LOG_ERROR("Failed to acquire postinit file lock: errno=%d", errno); - return 0; - } - LOG_MSG("Waiting for postinit file lock (PID %d)", getpid()); + unsigned int retry_delay_us = POSTINIT_FILE_LOCK_INITIAL_RETRY_US; + uint32_t retry_jitter = postinit_file_lock_jitter_seed(); + int waiting_logged = 0; + /* * A live owner is never bypassed: skipping host PID detection also * skips SM rate limiting for the process. + * + * Use nonblocking requests instead of F_SETLKW. Some NFS lock + * managers defer grants to blocked requests for tens of seconds + * after the owner exits even though fresh requests can acquire the + * released lock immediately. */ - do { - status = fcntl(region_info.fd, F_SETLKW, &lock); - } while (status != 0 && errno == EINTR); + for (;;) { + do { + status = fcntl(region_info.fd, F_SETLK, &lock); + } while (status != 0 && errno == EINTR); + if (status == 0) { + return 1; + } + if (errno != EACCES && errno != EAGAIN) { + LOG_ERROR("Failed to acquire postinit file lock: errno=%d", + errno); + return 0; + } + if (!waiting_logged) { + LOG_MSG("Waiting for postinit file lock (PID %d)", getpid()); + waiting_logged = 1; + } + postinit_file_lock_backoff(retry_delay_us, &retry_jitter); + if (retry_delay_us < POSTINIT_FILE_LOCK_MAX_RETRY_US / 2) { + retry_delay_us *= 2; + } else { + retry_delay_us = POSTINIT_FILE_LOCK_MAX_RETRY_US; + } + } } else { do { status = fcntl(region_info.fd, F_SETLK, &lock);