From 0301b880ed684df80f5ec6c7fe15e856dd582953 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Thu, 15 Oct 2026 20:07:13 +0000 Subject: [PATCH 01/16] UCT/ROCM: Add removal of remote mapping when owning buffer is freed to drop IPC --- src/uct/rocm/ipc/rocm_ipc_cache.c | 612 ++++++++++++++++++++++++++---- src/uct/rocm/ipc/rocm_ipc_cache.h | 24 +- src/uct/rocm/ipc/rocm_ipc_iface.c | 3 + src/uct/rocm/ipc/rocm_ipc_md.c | 7 + src/uct/rocm/ipc/rocm_ipc_md.h | 2 + 5 files changed, 573 insertions(+), 75 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 0b3f12afef1..d6a05e21383 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -11,6 +11,8 @@ #include "rocm_ipc_cache.h" +#include +#include #include #include #include @@ -18,7 +20,11 @@ #include #include #include +#include #include +#include + +#include typedef struct uct_rocm_ipc_cache_hash_key { @@ -55,6 +61,204 @@ typedef struct uct_rocm_ipc_remote_cache { static uct_rocm_ipc_remote_cache_t uct_rocm_ipc_remote_cache; +/* Runs before the registration cache (priority 1000). */ +#define UCT_ROCM_IPC_MEM_EVENT_PRIO 100 + + +typedef struct uct_rocm_ipc_export { + ucs_list_link_t list; + uintptr_t address; + size_t length; + hsa_signal_t signal; +} uct_rocm_ipc_export_t; + + +static ucs_list_link_t uct_rocm_ipc_export_active; +static ucs_list_link_t uct_rocm_ipc_export_retired; +static ucs_spinlock_t uct_rocm_ipc_export_lock; +static uint32_t uct_rocm_ipc_export_retired_count; +static ucs_init_once_t uct_rocm_ipc_export_handler_once = + UCS_INIT_ONCE_INITIALIZER; + + +static int +uct_rocm_ipc_signal_is_empty(const hsa_amd_ipc_signal_t *ipc_signal) +{ + size_t i; + + for (i = 0; i < ucs_static_array_size(ipc_signal->handle); ++i) { + if (ipc_signal->handle[i] != 0) { + return 0; + } + } + + return 1; +} + +static void uct_rocm_ipc_export_destroy(uct_rocm_ipc_export_t *exp) +{ + if (exp->signal.handle != 0) { + if (hsa_signal_destroy(exp->signal) != HSA_STATUS_SUCCESS) { + ucs_warn("failed to destroy rocm ipc export signal for %p", + (void*)exp->address); + } + } + + ucs_free(exp); +} + +/* An importer attaches the IPC signal before it maps memory. Therefore, after + * the free event has been dispatched, every mapping that can delay hipFree + * already holds its own signal reference and any thread may reap the export. */ +static void uct_rocm_ipc_export_reap(void) +{ + uct_rocm_ipc_export_t *exp, *tmp; + ucs_list_link_t destroy_list; + + ucs_list_head_init(&destroy_list); + + ucs_spin_lock(&uct_rocm_ipc_export_lock); + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { + ucs_list_del(&exp->list); + ucs_list_add_tail(&destroy_list, &exp->list); + uct_rocm_ipc_export_retired_count--; + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { + uct_rocm_ipc_export_destroy(exp); + } +} + +static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, + ucm_event_t *event, void *arg) +{ + uct_rocm_ipc_export_t *exp, *tmp; + uintptr_t start, end; + + if ((event_type != UCM_EVENT_MEM_TYPE_FREE) || + ((event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM) && + (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM_MANAGED))) { + return; + } + + start = (uintptr_t)event->mem_type.address; + end = start + event->mem_type.size; + + /* Importers acquire a signal reference before mapping memory, so the + * exported signal can be destroyed after this callback on any thread. */ + ucs_spin_lock(&uct_rocm_ipc_export_lock); + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { + if ((exp->address >= end) || + ((exp->address + exp->length) <= start)) { + continue; + } + + hsa_signal_store_screlease(exp->signal, 0); + ucs_list_del(&exp->list); + ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); + uct_rocm_ipc_export_retired_count++; + ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, + exp->length); + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); +} + +static void uct_rocm_ipc_export_install_handler(void) +{ + ucs_status_t status; + + status = ucm_set_event_handler(UCM_EVENT_MEM_TYPE_FREE, + UCT_ROCM_IPC_MEM_EVENT_PRIO, + uct_rocm_ipc_export_mem_free_cb, NULL); + if (status != UCS_OK) { + ucs_diag("rocm ipc free handler not installed: %s", + ucs_status_string(status)); + } +} + +ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, + hsa_amd_ipc_signal_t *ipc_signal_p) +{ + uct_rocm_ipc_export_t *exp; + hsa_signal_t signal; + hsa_status_t hsa_status; + + UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { + uct_rocm_ipc_export_install_handler(); + } + + uct_rocm_ipc_export_reap(); + + hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, &signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("failed to create rocm ipc signal for %p length %zu: %d", + (void*)address, length, hsa_status); + return UCS_ERR_IO_ERROR; + } + + hsa_status = hsa_amd_ipc_signal_create(signal, ipc_signal_p); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("failed to export rocm ipc signal for %p length %zu: %d", + (void*)address, length, hsa_status); + hsa_signal_destroy(signal); + return UCS_ERR_IO_ERROR; + } + + exp = ucs_malloc(sizeof(*exp), "uct_rocm_ipc_export_t"); + if (exp == NULL) { + hsa_signal_destroy(signal); + return UCS_ERR_NO_MEMORY; + } + + exp->address = address; + exp->length = length; + exp->signal = signal; + + ucs_spin_lock(&uct_rocm_ipc_export_lock); + ucs_list_add_tail(&uct_rocm_ipc_export_active, &exp->list); + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + return UCS_OK; +} + +void uct_rocm_ipc_export_progress(void) +{ + /* Writers hold the export lock. A stale zero only delays reaping. */ + if (ucs_likely(uct_rocm_ipc_export_retired_count == 0)) { + return; + } + + uct_rocm_ipc_export_reap(); +} + +static void uct_rocm_ipc_export_cleanup(void) +{ + uct_rocm_ipc_export_t *exp, *tmp; + ucs_list_link_t destroy_list; + + UCS_CLEANUP_ONCE(&uct_rocm_ipc_export_handler_once) { + ucm_unset_event_handler(UCM_EVENT_MEM_TYPE_FREE, + uct_rocm_ipc_export_mem_free_cb, NULL); + } + + ucs_list_head_init(&destroy_list); + ucs_spin_lock(&uct_rocm_ipc_export_lock); + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { + ucs_list_del(&exp->list); + ucs_list_add_tail(&destroy_list, &exp->list); + } + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { + ucs_list_del(&exp->list); + ucs_list_add_tail(&destroy_list, &exp->list); + } + uct_rocm_ipc_export_retired_count = 0; + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { + uct_rocm_ipc_export_destroy(exp); + } +} + static ucs_pgt_dir_t *uct_rocm_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) { void *ptr; @@ -84,31 +288,157 @@ uct_rocm_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, ucs_list_add_tail(list, ®ion->list); } +static void +uct_rocm_ipc_cache_region_unref(uct_rocm_ipc_cache_region_t *region) +{ + if (ucs_atomic_fadd32(®ion->refcount, (uint32_t)-1) == 1) { + ucs_free(region); + } +} + +/* Caller holds the remote-cache write lock. The claim winner detaches the + * mapping immediately so the exporter's hipFree can finish. It destroys the + * IPC signal only when no handler has taken ownership of it. */ +static int +uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, + ucs_list_link_t *deferred) +{ + void *mapped_addr; + ucs_status_t status; + + if (ucs_atomic_cswap32(®ion->deactivate, 0, 1) != 0) { + return 0; + } + + mapped_addr = region->mapped_addr; + region->mapped_addr = NULL; + /* A registered handler owns the signal and destroys it when it runs. */ + if (ucs_atomic_cswap32(®ion->signal_owner, 0, 1) == 0) { + region->retired_signal = region->signal; + region->signal.handle = 0; + } + + if (region->in_pgtable) { + status = ucs_pgtable_remove(®ion->cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_fatal("%s: failed to remove address:%p from cache (%s)", + region->cache->name, (void*)region->key.address, + ucs_status_string(status)); + } + region->in_pgtable = 0; + } + + if ((mapped_addr != NULL) && + (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS)) { + ucs_warn("failed to unmap addr:%p", mapped_addr); + } + + ucs_trace("rocm ipc cache drop addr:%p mapped:%p", + (void*)region->key.address, mapped_addr); + ucs_list_add_tail(deferred, ®ion->list); + return 1; +} + +static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) +{ + uct_rocm_ipc_cache_region_t *region, *tmp; + + ucs_list_for_each_safe(region, tmp, deferred, list) { + ucs_list_del(®ion->list); + if (region->retired_signal.handle != 0) { + if (hsa_signal_destroy(region->retired_signal) != + HSA_STATUS_SUCCESS) { + ucs_warn("failed to destroy rocm ipc cache signal for %p", + (void*)region->key.address); + } + region->retired_signal.handle = 0; + } + uct_rocm_ipc_cache_region_unref(region); + } +} + +static void +uct_rocm_ipc_cache_region_signal_destroy(uct_rocm_ipc_cache_region_t *region, + hsa_signal_t signal) +{ + if (signal.handle == 0) { + return; + } + + if (hsa_signal_destroy(signal) != HSA_STATUS_SUCCESS) { + ucs_warn("failed to destroy rocm ipc cache signal for %p", + (void*)region->key.address); + } +} + +static bool uct_rocm_ipc_cache_signal_cb(hsa_signal_value_t value, void *arg) +{ + uct_rocm_ipc_cache_region_t *region = arg; + ucs_list_link_t deferred; + hsa_signal_t signal = {.handle = 0}; + int claimed; + + (void)value; + if (ucs_atomic_cswap32(®ion->handler_done, 0, 1) != 0) { + return false; + } + + ucs_list_head_init(&deferred); + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + claimed = uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + if (region->signal.handle != 0) { + signal = region->signal; + region->signal.handle = 0; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + + uct_rocm_ipc_cache_region_signal_destroy(region, signal); + if (claimed) { + uct_rocm_ipc_cache_region_finalize(&deferred); + } + uct_rocm_ipc_cache_region_unref(region); + return false; +} + +static void +uct_rocm_ipc_cache_purge_collect_cb(const ucs_pgtable_t *pgtable, + ucs_pgt_region_t *pgt_region, void *arg) +{ + ucs_list_link_t *list = arg; + uct_rocm_ipc_cache_region_t *region; + + /* pgtable_purge has already unlinked the region. */ + region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); + region->in_pgtable = 0; + ucs_list_add_tail(list, ®ion->list); +} + static void uct_rocm_ipc_cache_purge(uct_rocm_ipc_cache_t *cache) { uct_rocm_ipc_cache_region_t *region, *tmp; - ucs_list_link_t region_list; + ucs_list_link_t region_list, deferred; ucs_list_head_init(®ion_list); - ucs_pgtable_purge(&cache->pgtable, uct_rocm_ipc_cache_region_collect_callback, - ®ion_list); + ucs_list_head_init(&deferred); + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + ucs_pgtable_purge(&cache->pgtable, uct_rocm_ipc_cache_purge_collect_cb, + ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_fatal("failed to unmap addr:%p", region->mapped_addr); - } - - ucs_free(region); + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_drop_locked(region, &deferred); } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_finalize(&deferred); ucs_trace("%s: rocm ipc cache purged", cache->name); } -static void uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, - void *from, void *to) +static void +uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, void *from, + void *to, ucs_list_link_t *deferred) { ucs_list_link_t region_list; - ucs_status_t status; uct_rocm_ipc_cache_region_t *region, *tmp; ucs_list_head_init(®ion_list); @@ -117,112 +447,151 @@ static void uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, uct_rocm_ipc_cache_region_collect_callback, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { - status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); - if (status != UCS_OK) { - ucs_error("failed to remove address:%p from cache (%s)", - (void *)region->key.address, ucs_status_string(status)); - } - - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_fatal("failed to unmap addr:%p", region->mapped_addr); - } - ucs_free(region); + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_drop_locked(region, deferred); } - ucs_trace("%s: closed memhandles in the range [%p..%p]", - cache->name, from, to); + ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, + to); } -static ucs_status_t uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, - uct_rocm_ipc_key_t *key, - void **mapped_addr) +/* Caller holds the remote-cache write lock. A new region is returned through + * @a watch_region_p so the caller can arm its IPC signal after releasing the + * lock. @a deferred receives regions whose signals must be destroyed. */ +static ucs_status_t +uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, + uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **watch_region_p, + ucs_list_link_t *deferred) { ucs_status_t status; ucs_pgt_region_t *pgt_region; uct_rocm_ipc_cache_region_t *region; + hsa_signal_t local_signal = {.handle = 0}; hsa_status_t hsa_status; int ret; + *watch_region_p = NULL; + /* A per-cache lock would be needed only if the caller holds no lock or * just a read lock on the global remote-cache lock, allowing concurrent * access to this page table. */ - pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, - &cache->pgtable, key->address); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, + key->address); if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); if (memcmp(&key->ipc, ®ion->key.ipc, sizeof(key->ipc)) == 0) { - /*cache hit */ ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, cache->name, (void *)key->address, - key->length, UCS_PGT_REGION_ARG(®ion->super)); + UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); *mapped_addr = region->mapped_addr; return UCS_OK; - } else { - ucs_trace("%s: rocm_ipc cache remove stale region:" - UCS_PGT_REGION_FMT " new_addr:%p new_size:%lu", - cache->name, UCS_PGT_REGION_ARG(®ion->super), - (void *)key->address, key->length); - - status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); - if (status != UCS_OK) { - ucs_error("%s: failed to remove address:%p from cache", - cache->name, (void *)key->address); - goto err; - } - - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_fatal("failed to unmap addr:%p", region->mapped_addr); - } + } - ucs_free(region); + ucs_trace("%s: rocm_ipc cache remove stale region:" UCS_PGT_REGION_FMT + " new_addr:%p new_size:%lu", + cache->name, UCS_PGT_REGION_ARG(®ion->super), + (void*)key->address, key->length); + if (!uct_rocm_ipc_cache_region_drop_locked(region, deferred)) { + status = UCS_ERR_IO_ERROR; + goto err; } } - hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, mapped_addr); + if (uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { + ucs_error("%s: rocm ipc key %p has no ipc signal", cache->name, + (void*)key->address); + status = UCS_ERR_INVALID_PARAM; + goto err; + } + + /* Acquire the signal reference before mapping memory. Once the mapping + * exists, the exporter may be blocked in hipFree and may destroy its + * signal reference immediately after dispatching the free event. */ + hsa_status = hsa_amd_ipc_signal_attach(&key->ipc_signal, &local_signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("%s: failed to attach rocm ipc signal for %p: %d", + cache->name, (void*)key->address, hsa_status); + status = UCS_ERR_IO_ERROR; + goto err; + } + + if (hsa_signal_load_scacquire(local_signal) <= 0) { + status = UCS_ERR_INVALID_ADDR; + goto err_destroy_signal; + } + + hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, + mapped_addr); if (ucs_unlikely(hsa_status != HSA_STATUS_SUCCESS)) { - ucs_fatal("%s: failed to open ipc mem handle. addr:%p len:%lu", - cache->name, (void *)key->address, key->length); + ucs_error("%s: failed to open ipc mem handle. addr:%p len:%lu", + cache->name, (void*)key->address, key->length); + status = UCS_ERR_INVALID_ADDR; + goto err_destroy_signal; } - /*create new cache entry */ - ret = ucs_posix_memalign((void **)®ion, - ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + ret = ucs_posix_memalign((void**)®ion, + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(uct_rocm_ipc_cache_region_t), "uct_rocm_ipc_cache_region"); if (ret != 0) { ucs_warn("failed to allocate uct_rocm_ipc_cache region"); + if (hsa_amd_ipc_memory_detach(*mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", *mapped_addr); + } status = UCS_ERR_NO_MEMORY; - goto err; + goto err_destroy_signal; } - region->super.start = ucs_align_down_pow2(key->address, UCS_PGT_ADDR_ALIGN); - region->super.end = ucs_align_up_pow2(key->address + key->length, UCS_PGT_ADDR_ALIGN); - region->key = *key; - region->mapped_addr = *mapped_addr; - - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + region->super.start = + ucs_align_down_pow2(key->address, UCS_PGT_ADDR_ALIGN); + region->super.end = + ucs_align_up_pow2(key->address + key->length, UCS_PGT_ADDR_ALIGN); + region->key = *key; + region->mapped_addr = *mapped_addr; + region->cache = cache; + region->signal = local_signal; + region->retired_signal.handle = 0; + region->refcount = 1; + region->deactivate = 0; + region->handler_done = 0; + region->signal_owner = 0; + region->in_pgtable = 0; + + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); if (status == UCS_ERR_ALREADY_EXISTS) { - /* overlapped region means memory freed at source. remove and try insert */ - uct_rocm_ipc_cache_invalidate_regions(cache, - (void *)region->super.start, - (void *)region->super.end); - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + /* Overlap means the source freed the memory. Drop it and retry. */ + uct_rocm_ipc_cache_invalidate_regions(cache, (void*)region->super.start, + (void*)region->super.end, + deferred); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); } if (status != UCS_OK) { - - ucs_error("%s: failed to insert region:"UCS_PGT_REGION_FMT" size:%lu :%s", + ucs_error("%s: failed to insert region:" UCS_PGT_REGION_FMT + " size:%lu :%s", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, ucs_status_string(status)); + if (hsa_amd_ipc_memory_detach(*mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", *mapped_addr); + } ucs_free(region); - goto err; + goto err_destroy_signal; } - ucs_trace("%s: rocm_ipc cache new region:"UCS_PGT_REGION_FMT" size:%lu", + region->in_pgtable = 1; + *watch_region_p = region; + ucs_trace("%s: rocm_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); - return UCS_OK; + +err_destroy_signal: + if (hsa_signal_destroy(local_signal) != HSA_STATUS_SUCCESS) { + ucs_warn("%s: failed to destroy rocm ipc signal for %p", cache->name, + (void*)key->address); + } err: return status; } @@ -302,21 +671,105 @@ uct_rocm_ipc_remote_cache_get(uct_rocm_ipc_cache_hash_key_t key, return UCS_OK; } +static ucs_status_t +uct_rocm_ipc_cache_region_watch(uct_rocm_ipc_cache_region_t *region) +{ + hsa_signal_t signal = {.handle = 0}; + hsa_signal_value_t value; + hsa_status_t hsa_status; + ucs_list_link_t deferred; + ucs_status_t status; + + ucs_list_head_init(&deferred); + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + if ((ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) || + (region->signal.handle == 0) || + (ucs_atomic_cswap32(®ion->signal_owner, 0, 2) != 0)) { + /* The mapping was claimed before the handler existed. */ + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + status = UCS_ERR_INVALID_ADDR; + goto out_unref; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + + ucs_atomic_add32(®ion->refcount, 1); + hsa_status = hsa_amd_signal_async_handler(region->signal, + HSA_SIGNAL_CONDITION_EQ, 0, + uct_rocm_ipc_cache_signal_cb, + region); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("%s: failed to arm rocm ipc signal for %p: %d", + region->cache->name, (void*)region->key.address, hsa_status); + ucs_atomic_fadd32(®ion->refcount, (uint32_t)-1); + status = UCS_ERR_IO_ERROR; + goto drop_mapping; + } + + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + if (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) { + /* The handler owns both its region reference and the signal. */ + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + status = UCS_ERR_INVALID_ADDR; + goto out_unref; + } + + value = hsa_signal_load_scacquire(region->signal); + if (value <= 0) { + uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + status = UCS_ERR_INVALID_ADDR; + } else { + status = UCS_OK; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_finalize(&deferred); + goto out_unref; + +drop_mapping: + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + signal = region->signal; + region->signal.handle = 0; + uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_signal_destroy(region, signal); + uct_rocm_ipc_cache_region_finalize(&deferred); +out_unref: + /* Drops the reference taken before the watcher was called. The handler + * reference, once arming succeeds, belongs to the signal callback. */ + uct_rocm_ipc_cache_region_unref(region); + return status; +} + ucs_status_t uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) { uct_rocm_ipc_cache_hash_key_t hash_key = {key->pid, key->pid_ns, key->dev_num}; + uct_rocm_ipc_cache_region_t *watch_region = NULL; uct_rocm_ipc_cache_t *cache; + ucs_list_link_t deferred; ucs_status_t status; + void *mapped; + ucs_list_head_init(&deferred); ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); status = uct_rocm_ipc_remote_cache_get(hash_key, &cache); if (status == UCS_OK) { - status = uct_rocm_ipc_cache_map_region(cache, key, mapped_addr); + status = uct_rocm_ipc_cache_map_region(cache, key, &mapped, + &watch_region, &deferred); + } + if ((status == UCS_OK) && (watch_region != NULL)) { + /* Keep the region alive until its IPC signal is armed. */ + ucs_atomic_add32(&watch_region->refcount, 1); } ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_finalize(&deferred); + if ((status == UCS_OK) && (watch_region != NULL)) { + status = uct_rocm_ipc_cache_region_watch(watch_region); + } + if (status == UCS_OK) { + *mapped_addr = mapped; + } return status; } @@ -330,16 +783,27 @@ void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache) UCS_STATIC_INIT { + ucs_status_t status; + ucs_rw_spinlock_init(&uct_rocm_ipc_remote_cache.lock); kh_init_inplace(rocm_ipc_rem_cache, &uct_rocm_ipc_remote_cache.hash); + + status = ucs_spinlock_init(&uct_rocm_ipc_export_lock, 0); + if (status != UCS_OK) { + ucs_fatal("failed to initialize rocm ipc export lock"); + } + ucs_list_head_init(&uct_rocm_ipc_export_active); + ucs_list_head_init(&uct_rocm_ipc_export_retired); } UCS_STATIC_CLEANUP { uct_rocm_ipc_cache_t *cache; + uct_rocm_ipc_export_cleanup(); kh_foreach_value(&uct_rocm_ipc_remote_cache.hash, cache, { uct_rocm_ipc_destroy_cache(cache); }) kh_destroy_inplace(rocm_ipc_rem_cache, &uct_rocm_ipc_remote_cache.hash); ucs_rw_spinlock_cleanup(&uct_rocm_ipc_remote_cache.lock); + ucs_spinlock_destroy(&uct_rocm_ipc_export_lock); } diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 01751c36d15..eedd2370cf8 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -13,11 +13,19 @@ #include "rocm_ipc_md.h" -typedef struct uct_cuda_ipc_cache_region { +typedef struct uct_rocm_ipc_cache_region { ucs_pgt_region_t super; /**< Base class - page table region */ ucs_list_link_t list; /**< List element */ uct_rocm_ipc_key_t key; /**< Remote memory key */ void *mapped_addr; /**< Local mapped address */ + uct_rocm_ipc_cache_t *cache; /**< Cache that owns the region */ + hsa_signal_t signal; /**< Attached IPC signal */ + hsa_signal_t retired_signal; /**< Signal owned by claim winner */ + uint32_t refcount; /**< Pgtable, handler, and watcher */ + uint32_t deactivate; /**< 1 after unmap is claimed */ + uint32_t handler_done; /**< Async handler entered once */ + uint32_t signal_owner; /**< 0 open, 1 claim, 2 handler */ + int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; typedef struct uct_rocm_ipc_cache { @@ -32,4 +40,18 @@ void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache); ucs_status_t uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); + +/** + * Create an IPC signal for an exported buffer and remember it until free. + * The packed @a ipc_signal is the same for every peer that imports this key. + */ +ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, + hsa_amd_ipc_signal_t *ipc_signal_p); + +/** + * Destroy retired export signals. Importers attach the signal before mapping + * memory, so any thread may safely reap them. + */ +void uct_rocm_ipc_export_progress(void); + #endif diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.c b/src/uct/rocm/ipc/rocm_ipc_iface.c index 89c82d1c996..37829f553da 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.c +++ b/src/uct/rocm/ipc/rocm_ipc_iface.c @@ -11,6 +11,7 @@ #include "rocm_ipc_iface.h" #include "rocm_ipc_md.h" #include "rocm_ipc_ep.h" +#include "rocm_ipc_cache.h" #include #include @@ -170,6 +171,8 @@ static unsigned uct_rocm_ipc_iface_progress(uct_iface_h tl_iface) { uct_rocm_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_rocm_ipc_iface_t); + /* Destroy export signals whose hipFree has already returned. */ + uct_rocm_ipc_export_progress(); return uct_rocm_base_progress(&iface->signal_queue); } diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index 02178861add..3a2198b012f 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -84,6 +84,12 @@ static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, key->pid = getpid(); key->pid_ns = ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID); + /* One IPC signal per export. Every importer of this key watches it. */ + if (uct_rocm_ipc_export_add(key->address, key->length, &key->ipc_signal) != + UCS_OK) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + return HSA_STATUS_SUCCESS; } @@ -119,6 +125,7 @@ uct_rocm_ipc_mem_dereg(uct_md_h md, UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); + /* The IPC signal is owned by the export table and stays until free. */ mem_hndl = params->memh; ucs_free(mem_hndl); return UCS_OK; diff --git a/src/uct/rocm/ipc/rocm_ipc_md.h b/src/uct/rocm/ipc/rocm_ipc_md.h index 3f0f70b39db..7c2fc58293c 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.h +++ b/src/uct/rocm/ipc/rocm_ipc_md.h @@ -30,6 +30,8 @@ typedef struct uct_rocm_ipc_md_config { typedef struct uct_rocm_ipc_key { hsa_amd_ipc_memory_t ipc; + /* Owner sets this signal to 0 when the allocation is released */ + hsa_amd_ipc_signal_t ipc_signal; uintptr_t address; size_t length; int dev_num; From e8f9f8ea7bad0c124aef32c41536f40bd617bf44 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Mon, 26 Oct 2026 19:05:02 +0000 Subject: [PATCH 02/16] UCT/ROCM: review fixes --- src/uct/rocm/ipc/rocm_ipc_cache.c | 48 +++++++++++++++++++------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index d6a05e21383..2df5f429eaf 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -121,7 +121,8 @@ static void uct_rocm_ipc_export_reap(void) ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); - uct_rocm_ipc_export_retired_count--; + __atomic_fetch_sub(&uct_rocm_ipc_export_retired_count, 1, + __ATOMIC_RELAXED); } ucs_spin_unlock(&uct_rocm_ipc_export_lock); @@ -157,7 +158,8 @@ static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, hsa_signal_store_screlease(exp->signal, 0); ucs_list_del(&exp->list); ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); - uct_rocm_ipc_export_retired_count++; + __atomic_fetch_add(&uct_rocm_ipc_export_retired_count, 1, + __ATOMIC_RELAXED); ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, exp->length); } @@ -223,8 +225,8 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, void uct_rocm_ipc_export_progress(void) { - /* Writers hold the export lock. A stale zero only delays reaping. */ - if (ucs_likely(uct_rocm_ipc_export_retired_count == 0)) { + if (ucs_likely(__atomic_load_n(&uct_rocm_ipc_export_retired_count, + __ATOMIC_RELAXED) == 0)) { return; } @@ -251,7 +253,8 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); } - uct_rocm_ipc_export_retired_count = 0; + __atomic_store_n(&uct_rocm_ipc_export_retired_count, 0, + __ATOMIC_RELAXED); ucs_spin_unlock(&uct_rocm_ipc_export_lock); ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { @@ -291,7 +294,7 @@ uct_rocm_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, static void uct_rocm_ipc_cache_region_unref(uct_rocm_ipc_cache_region_t *region) { - if (ucs_atomic_fadd32(®ion->refcount, (uint32_t)-1) == 1) { + if (ucs_atomic_fsub32(®ion->refcount, 1) == 1) { ucs_free(region); } } @@ -467,6 +470,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, ucs_pgt_region_t *pgt_region; uct_rocm_ipc_cache_region_t *region; hsa_signal_t local_signal = {.handle = 0}; + hsa_signal_value_t signal_value; hsa_status_t hsa_status; int ret; @@ -480,19 +484,27 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); if (memcmp(&key->ipc, ®ion->key.ipc, sizeof(key->ipc)) == 0) { - ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, - cache->name, (void*)key->address, key->length, - UCS_PGT_REGION_ARG(®ion->super)); + signal_value = hsa_signal_load_scacquire(region->signal); + if (signal_value > 0) { + ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" + UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); + + *mapped_addr = region->mapped_addr; + return UCS_OK; + } - *mapped_addr = region->mapped_addr; - return UCS_OK; + ucs_trace("%s: rocm_ipc cache remove invalidated region:" + UCS_PGT_REGION_FMT, + cache->name, UCS_PGT_REGION_ARG(®ion->super)); + } else { + ucs_trace("%s: rocm_ipc cache remove stale region:" + UCS_PGT_REGION_FMT " new_addr:%p new_size:%lu", + cache->name, UCS_PGT_REGION_ARG(®ion->super), + (void*)key->address, key->length); } - ucs_trace("%s: rocm_ipc cache remove stale region:" UCS_PGT_REGION_FMT - " new_addr:%p new_size:%lu", - cache->name, UCS_PGT_REGION_ARG(®ion->super), - (void*)key->address, key->length); if (!uct_rocm_ipc_cache_region_drop_locked(region, deferred)) { status = UCS_ERR_IO_ERROR; goto err; @@ -525,8 +537,8 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, mapped_addr); if (ucs_unlikely(hsa_status != HSA_STATUS_SUCCESS)) { - ucs_error("%s: failed to open ipc mem handle. addr:%p len:%lu", - cache->name, (void*)key->address, key->length); + ucs_error("%s: failed to open ipc mem handle. addr:%p len:%lu hsa_status:%d", + cache->name, (void*)key->address, key->length, hsa_status); status = UCS_ERR_INVALID_ADDR; goto err_destroy_signal; } From c411d73d48267954fd808b4dc4e1461687479f4d Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Mon, 26 Oct 2026 21:25:18 +0000 Subject: [PATCH 03/16] UCT/ROCM: fix for review feedback --- src/uct/rocm/ipc/rocm_ipc_cache.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 2df5f429eaf..affa2693225 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -246,6 +246,7 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_head_init(&destroy_list); ucs_spin_lock(&uct_rocm_ipc_export_lock); ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { + hsa_signal_store_screlease(exp->signal, 0); ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); } @@ -342,6 +343,20 @@ uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, return 1; } +/* Caller holds the remote-cache write lock. A different or overlapping IPC + * handle proves that this mapping is stale, so wake its handler to release + * the reference it holds on the region and its signal. */ +static int +uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, + ucs_list_link_t *deferred) +{ + if (ucs_atomic_cswap32(®ion->signal_owner, 2, 2) == 2) { + hsa_signal_store_screlease(region->signal, 0); + } + + return uct_rocm_ipc_cache_region_drop_locked(region, deferred); +} + static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) { uct_rocm_ipc_cache_region_t *region, *tmp; @@ -451,7 +466,7 @@ uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, void *from, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { ucs_list_del(®ion->list); - uct_rocm_ipc_cache_region_drop_locked(region, deferred); + uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred); } ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, to); @@ -505,7 +520,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, (void*)key->address, key->length); } - if (!uct_rocm_ipc_cache_region_drop_locked(region, deferred)) { + if (!uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred)) { status = UCS_ERR_IO_ERROR; goto err; } From 763f5d3009e0679e8aa52f3fea9983d0979f8557 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Mon, 26 Oct 2026 21:41:51 +0000 Subject: [PATCH 04/16] UCT/ROCM: added environment variable to control cache invalidate --- src/uct/rocm/ipc/rocm_ipc_cache.c | 119 +++++++++++++++++++++++------- src/uct/rocm/ipc/rocm_ipc_md.h | 3 +- 2 files changed, 93 insertions(+), 29 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index affa2693225..714a1b33885 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -64,6 +65,49 @@ static uct_rocm_ipc_remote_cache_t uct_rocm_ipc_remote_cache; /* Runs before the registration cache (priority 1000). */ #define UCT_ROCM_IPC_MEM_EVENT_PRIO 100 +typedef struct { + int signal_invalidate; +} uct_rocm_ipc_cache_config_t; + +static ucs_config_field_t uct_rocm_ipc_cache_config_table[] = { + {"SIGNAL_INVALIDATE", "y", + "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" + "The IPC signal remains part of the packed rkey when this is n.", + ucs_offsetof(uct_rocm_ipc_cache_config_t, signal_invalidate), + UCS_CONFIG_TYPE_BOOL}, + + {NULL} +}; + +UCS_CONFIG_REGISTER_TABLE(uct_rocm_ipc_cache_config_table, "ROCm IPC", + "ROCM_IPC_", uct_rocm_ipc_cache_config_t, + &ucs_config_global_list); + +static int uct_rocm_ipc_signal_invalidate_enabled(void) +{ + static ucs_init_once_t once = UCS_INIT_ONCE_INITIALIZER; + static int enabled = 1; + + UCS_INIT_ONCE(&once) { + uct_rocm_ipc_cache_config_t config; + ucs_status_t status; + + status = ucs_config_parser_fill_opts( + &config, UCS_CONFIG_GET_TABLE(uct_rocm_ipc_cache_config_table), + UCS_DEFAULT_ENV_PREFIX, 0); + if (status == UCS_OK) { + enabled = config.signal_invalidate; + ucs_config_parser_release_opts(&config, + uct_rocm_ipc_cache_config_table); + } + if (!enabled) { + ucs_diag("rocm ipc signal invalidation is disabled"); + } + } + + return enabled; +} + typedef struct uct_rocm_ipc_export { ucs_list_link_t list; @@ -138,8 +182,7 @@ static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, uintptr_t start, end; if ((event_type != UCM_EVENT_MEM_TYPE_FREE) || - ((event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM) && - (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM_MANAGED))) { + (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM)) { return; } @@ -186,11 +229,13 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_signal_t signal; hsa_status_t hsa_status; - UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { - uct_rocm_ipc_export_install_handler(); - } + if (uct_rocm_ipc_signal_invalidate_enabled()) { + UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { + uct_rocm_ipc_export_install_handler(); + } - uct_rocm_ipc_export_reap(); + uct_rocm_ipc_export_reap(); + } hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, &signal); if (hsa_status != HSA_STATUS_SUCCESS) { @@ -246,7 +291,9 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_head_init(&destroy_list); ucs_spin_lock(&uct_rocm_ipc_export_lock); ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { - hsa_signal_store_screlease(exp->signal, 0); + if (uct_rocm_ipc_signal_invalidate_enabled()) { + hsa_signal_store_screlease(exp->signal, 0); + } ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); } @@ -499,6 +546,16 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); if (memcmp(&key->ipc, ®ion->key.ipc, sizeof(key->ipc)) == 0) { + if (region->signal.handle == 0) { + ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" + UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); + + *mapped_addr = region->mapped_addr; + return UCS_OK; + } + signal_value = hsa_signal_load_scacquire(region->signal); if (signal_value > 0) { ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" @@ -526,27 +583,30 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, } } - if (uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { - ucs_error("%s: rocm ipc key %p has no ipc signal", cache->name, - (void*)key->address); - status = UCS_ERR_INVALID_PARAM; - goto err; - } + if (uct_rocm_ipc_signal_invalidate_enabled()) { + if (uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { + ucs_error("%s: rocm ipc key %p has no ipc signal", cache->name, + (void*)key->address); + status = UCS_ERR_INVALID_PARAM; + goto err; + } - /* Acquire the signal reference before mapping memory. Once the mapping - * exists, the exporter may be blocked in hipFree and may destroy its - * signal reference immediately after dispatching the free event. */ - hsa_status = hsa_amd_ipc_signal_attach(&key->ipc_signal, &local_signal); - if (hsa_status != HSA_STATUS_SUCCESS) { - ucs_error("%s: failed to attach rocm ipc signal for %p: %d", - cache->name, (void*)key->address, hsa_status); - status = UCS_ERR_IO_ERROR; - goto err; - } + /* Acquire the signal reference before mapping memory. Once the + * mapping exists, the exporter may be blocked in hipFree and may + * destroy its signal reference after dispatching the free event. */ + hsa_status = hsa_amd_ipc_signal_attach(&key->ipc_signal, + &local_signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("%s: failed to attach rocm ipc signal for %p: %d", + cache->name, (void*)key->address, hsa_status); + status = UCS_ERR_IO_ERROR; + goto err; + } - if (hsa_signal_load_scacquire(local_signal) <= 0) { - status = UCS_ERR_INVALID_ADDR; - goto err_destroy_signal; + if (hsa_signal_load_scacquire(local_signal) <= 0) { + status = UCS_ERR_INVALID_ADDR; + goto err_destroy_signal; + } } hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, @@ -609,13 +669,16 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, } region->in_pgtable = 1; - *watch_region_p = region; + if (uct_rocm_ipc_signal_invalidate_enabled()) { + *watch_region_p = region; + } ucs_trace("%s: rocm_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); return UCS_OK; err_destroy_signal: - if (hsa_signal_destroy(local_signal) != HSA_STATUS_SUCCESS) { + if ((local_signal.handle != 0) && + (hsa_signal_destroy(local_signal) != HSA_STATUS_SUCCESS)) { ucs_warn("%s: failed to destroy rocm ipc signal for %p", cache->name, (void*)key->address); } diff --git a/src/uct/rocm/ipc/rocm_ipc_md.h b/src/uct/rocm/ipc/rocm_ipc_md.h index 7c2fc58293c..40724cf13c9 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.h +++ b/src/uct/rocm/ipc/rocm_ipc_md.h @@ -30,7 +30,8 @@ typedef struct uct_rocm_ipc_md_config { typedef struct uct_rocm_ipc_key { hsa_amd_ipc_memory_t ipc; - /* Owner sets this signal to 0 when the allocation is released */ + /* Owner sets this signal to 0 when the allocation is released. + * Unused when UCX_ROCM_IPC_SIGNAL_INVALIDATE is n. */ hsa_amd_ipc_signal_t ipc_signal; uintptr_t address; size_t length; From 366083532f872b2e45878c646c6afff225f99d25 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 14:01:50 +0000 Subject: [PATCH 05/16] UCT/ROCM: Implemented export reuse per allocation --- src/uct/rocm/ipc/rocm_ipc_cache.c | 62 ++++++++++++++++++++++++------- src/uct/rocm/ipc/rocm_ipc_cache.h | 3 +- src/uct/rocm/ipc/rocm_ipc_md.c | 2 +- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 714a1b33885..7dcc51a6c7d 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -114,6 +114,7 @@ typedef struct uct_rocm_ipc_export { uintptr_t address; size_t length; hsa_signal_t signal; + hsa_amd_ipc_signal_t ipc_signal; } uct_rocm_ipc_export_t; @@ -151,6 +152,21 @@ static void uct_rocm_ipc_export_destroy(uct_rocm_ipc_export_t *exp) ucs_free(exp); } +/* Caller holds the export lock. */ +static uct_rocm_ipc_export_t * +uct_rocm_ipc_export_find_active(uintptr_t address, size_t length) +{ + uct_rocm_ipc_export_t *exp; + + ucs_list_for_each(exp, &uct_rocm_ipc_export_active, list) { + if ((exp->address == address) && (exp->length == length)) { + return exp; + } + } + + return NULL; +} + /* An importer attaches the IPC signal before it maps memory. Therefore, after * the free event has been dispatched, every mapping that can delay hipFree * already holds its own signal reference and any thread may reap the export. */ @@ -225,8 +241,7 @@ static void uct_rocm_ipc_export_install_handler(void) ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_amd_ipc_signal_t *ipc_signal_p) { - uct_rocm_ipc_export_t *exp; - hsa_signal_t signal; + uct_rocm_ipc_export_t *exp, *existing; hsa_status_t hsa_status; if (uct_rocm_ipc_signal_invalidate_enabled()) { @@ -237,34 +252,55 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, uct_rocm_ipc_export_reap(); } - hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, &signal); + ucs_spin_lock(&uct_rocm_ipc_export_lock); + existing = uct_rocm_ipc_export_find_active(address, length); + if (existing != NULL) { + *ipc_signal_p = existing->ipc_signal; + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + return UCS_OK; + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + exp = ucs_malloc(sizeof(*exp), "uct_rocm_ipc_export_t"); + if (exp == NULL) { + return UCS_ERR_NO_MEMORY; + } + + hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, + &exp->signal); if (hsa_status != HSA_STATUS_SUCCESS) { ucs_error("failed to create rocm ipc signal for %p length %zu: %d", (void*)address, length, hsa_status); + ucs_free(exp); return UCS_ERR_IO_ERROR; } - hsa_status = hsa_amd_ipc_signal_create(signal, ipc_signal_p); + hsa_status = hsa_amd_ipc_signal_create(exp->signal, &exp->ipc_signal); if (hsa_status != HSA_STATUS_SUCCESS) { ucs_error("failed to export rocm ipc signal for %p length %zu: %d", (void*)address, length, hsa_status); - hsa_signal_destroy(signal); + hsa_signal_destroy(exp->signal); + ucs_free(exp); return UCS_ERR_IO_ERROR; } - exp = ucs_malloc(sizeof(*exp), "uct_rocm_ipc_export_t"); - if (exp == NULL) { - hsa_signal_destroy(signal); - return UCS_ERR_NO_MEMORY; - } - exp->address = address; exp->length = length; - exp->signal = signal; ucs_spin_lock(&uct_rocm_ipc_export_lock); - ucs_list_add_tail(&uct_rocm_ipc_export_active, &exp->list); + existing = uct_rocm_ipc_export_find_active(address, length); + if (existing == NULL) { + ucs_list_add_tail(&uct_rocm_ipc_export_active, &exp->list); + *ipc_signal_p = exp->ipc_signal; + } else { + *ipc_signal_p = existing->ipc_signal; + } ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + if (existing != NULL) { + uct_rocm_ipc_export_destroy(exp); + } + return UCS_OK; } diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 7968f74f35d..964ca427ec1 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -42,7 +42,8 @@ ucs_status_t uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); /** - * Create an IPC signal for an exported buffer and remember it until free. + * Create or reuse an IPC signal for an exported buffer and remember it until + * free. * The packed @a ipc_signal is the same for every peer that imports this key. */ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index fbebe305182..212946ba031 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -87,7 +87,7 @@ static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, key->pid = getpid(); key->pid_ns = ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID); - /* One IPC signal per export. Every importer of this key watches it. */ + /* One IPC signal per allocation. Every importer of this key watches it. */ if (uct_rocm_ipc_export_add(key->address, key->length, &key->ipc_signal) != UCS_OK) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; From 77755855093d45bca457469d361e68949729b745 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 14:28:42 +0000 Subject: [PATCH 06/16] UCT/ROCM: Cleanup of minor review items --- src/uct/rocm/ipc/rocm_ipc_cache.c | 30 ++++++------------------------ src/uct/rocm/ipc/rocm_ipc_cache.h | 16 +++++----------- src/uct/rocm/ipc/rocm_ipc_iface.c | 3 --- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 7dcc51a6c7d..ab174b337b9 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -121,7 +121,6 @@ typedef struct uct_rocm_ipc_export { static ucs_list_link_t uct_rocm_ipc_export_active; static ucs_list_link_t uct_rocm_ipc_export_retired; static ucs_spinlock_t uct_rocm_ipc_export_lock; -static uint32_t uct_rocm_ipc_export_retired_count; static ucs_init_once_t uct_rocm_ipc_export_handler_once = UCS_INIT_ONCE_INITIALIZER; @@ -181,8 +180,6 @@ static void uct_rocm_ipc_export_reap(void) ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); - __atomic_fetch_sub(&uct_rocm_ipc_export_retired_count, 1, - __ATOMIC_RELAXED); } ucs_spin_unlock(&uct_rocm_ipc_export_lock); @@ -217,8 +214,6 @@ static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, hsa_signal_store_screlease(exp->signal, 0); ucs_list_del(&exp->list); ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); - __atomic_fetch_add(&uct_rocm_ipc_export_retired_count, 1, - __ATOMIC_RELAXED); ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, exp->length); } @@ -304,16 +299,6 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, return UCS_OK; } -void uct_rocm_ipc_export_progress(void) -{ - if (ucs_likely(__atomic_load_n(&uct_rocm_ipc_export_retired_count, - __ATOMIC_RELAXED) == 0)) { - return; - } - - uct_rocm_ipc_export_reap(); -} - static void uct_rocm_ipc_export_cleanup(void) { uct_rocm_ipc_export_t *exp, *tmp; @@ -337,8 +322,6 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); } - __atomic_store_n(&uct_rocm_ipc_export_retired_count, 0, - __ATOMIC_RELAXED); ucs_spin_unlock(&uct_rocm_ipc_export_lock); ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { @@ -429,7 +412,7 @@ uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, /* Caller holds the remote-cache write lock. A different or overlapping IPC * handle proves that this mapping is stale, so wake its handler to release * the reference it holds on the region and its signal. */ -static int +static void uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, ucs_list_link_t *deferred) { @@ -437,7 +420,7 @@ uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, hsa_signal_store_screlease(region->signal, 0); } - return uct_rocm_ipc_cache_region_drop_locked(region, deferred); + uct_rocm_ipc_cache_region_drop_locked(region, deferred); } static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) @@ -564,10 +547,10 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, uct_rocm_ipc_cache_region_t **watch_region_p, ucs_list_link_t *deferred) { + hsa_signal_t local_signal = {.handle = 0}; ucs_status_t status; ucs_pgt_region_t *pgt_region; uct_rocm_ipc_cache_region_t *region; - hsa_signal_t local_signal = {.handle = 0}; hsa_signal_value_t signal_value; hsa_status_t hsa_status; int ret; @@ -613,10 +596,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, (void*)key->address, key->length); } - if (!uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred)) { - status = UCS_ERR_IO_ERROR; - goto err; - } + uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred); } if (uct_rocm_ipc_signal_invalidate_enabled()) { @@ -640,6 +620,8 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, } if (hsa_signal_load_scacquire(local_signal) <= 0) { + ucs_debug("%s: rocm ipc buffer %p was already released by the owner", + cache->name, (void*)key->address); status = UCS_ERR_INVALID_ADDR; goto err_destroy_signal; } diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 964ca427ec1..43d6eb643a6 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -13,6 +13,11 @@ #include "rocm_ipc_md.h" +typedef struct uct_rocm_ipc_cache { + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + char *name; /**< Name */ +} uct_rocm_ipc_cache_t; + typedef struct { ucs_pgt_region_t super; /**< Base class - page table region */ ucs_list_link_t list; /**< List element */ @@ -28,11 +33,6 @@ typedef struct { int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; -typedef struct uct_rocm_ipc_cache { - ucs_pgtable_t pgtable; /**< Page table to hold the regions */ - char *name; /**< Name */ -} uct_rocm_ipc_cache_t; - ucs_status_t uct_rocm_ipc_create_cache(uct_rocm_ipc_cache_t **cache, const char *name); @@ -49,10 +49,4 @@ uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_amd_ipc_signal_t *ipc_signal_p); -/** - * Destroy retired export signals. Importers attach the signal before mapping - * memory, so any thread may safely reap them. - */ -void uct_rocm_ipc_export_progress(void); - #endif diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.c b/src/uct/rocm/ipc/rocm_ipc_iface.c index 40e28b66e7a..d2a00092142 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.c +++ b/src/uct/rocm/ipc/rocm_ipc_iface.c @@ -11,7 +11,6 @@ #include "rocm_ipc_iface.h" #include "rocm_ipc_md.h" #include "rocm_ipc_ep.h" -#include "rocm_ipc_cache.h" #include #include @@ -172,8 +171,6 @@ static unsigned uct_rocm_ipc_iface_progress(uct_iface_h tl_iface) { uct_rocm_ipc_iface_t *iface = ucs_derived_of(tl_iface, uct_rocm_ipc_iface_t); - /* Destroy export signals whose hipFree has already returned. */ - uct_rocm_ipc_export_progress(); return uct_rocm_base_progress(&iface->signal_queue); } From ed31c5b92fd5161cadedc5b0553ba0572bb178bd Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 15:09:18 +0000 Subject: [PATCH 07/16] UCT/ROCM: Fix for teardown lifetime issue and implemented export reuse per allocation. --- src/uct/rocm/ipc/rocm_ipc_cache.c | 166 +++++++++++++++++++++--------- src/uct/rocm/ipc/rocm_ipc_cache.h | 14 ++- src/uct/rocm/ipc/rocm_ipc_md.c | 98 +++++++++++------- 3 files changed, 189 insertions(+), 89 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index ab174b337b9..770ea5b4d94 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -366,9 +367,9 @@ uct_rocm_ipc_cache_region_unref(uct_rocm_ipc_cache_region_t *region) } } -/* Caller holds the remote-cache write lock. The claim winner detaches the - * mapping immediately so the exporter's hipFree can finish. It destroys the - * IPC signal only when no handler has taken ownership of it. */ +/* Caller holds the remote-cache write lock. The claim winner detaches an + * unpinned mapping immediately so the exporter's hipFree can finish. It + * destroys the IPC signal only when no handler has taken ownership of it. */ static int uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, ucs_list_link_t *deferred) @@ -380,8 +381,13 @@ uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, return 0; } - mapped_addr = region->mapped_addr; - region->mapped_addr = NULL; + /* A pinned mapping is detached by the last unpin. */ + mapped_addr = NULL; + if (region->pin_count == 0) { + mapped_addr = region->mapped_addr; + region->mapped_addr = NULL; + } + /* A registered handler owns the signal and destroys it when it runs. */ if (ucs_atomic_cswap32(®ion->signal_owner, 0, 1) == 0) { region->retired_signal = region->signal; @@ -416,28 +422,12 @@ static void uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, ucs_list_link_t *deferred) { - if (ucs_atomic_cswap32(®ion->signal_owner, 2, 2) == 2) { - hsa_signal_store_screlease(region->signal, 0); - } + int handler_owned; + handler_owned = (ucs_atomic_cswap32(®ion->signal_owner, 2, 2) == 2); uct_rocm_ipc_cache_region_drop_locked(region, deferred); -} - -static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) -{ - uct_rocm_ipc_cache_region_t *region, *tmp; - - ucs_list_for_each_safe(region, tmp, deferred, list) { - ucs_list_del(®ion->list); - if (region->retired_signal.handle != 0) { - if (hsa_signal_destroy(region->retired_signal) != - HSA_STATUS_SUCCESS) { - ucs_warn("failed to destroy rocm ipc cache signal for %p", - (void*)region->key.address); - } - region->retired_signal.handle = 0; - } - uct_rocm_ipc_cache_region_unref(region); + if (handler_owned) { + hsa_signal_store_screlease(region->signal, 0); } } @@ -455,6 +445,19 @@ uct_rocm_ipc_cache_region_signal_destroy(uct_rocm_ipc_cache_region_t *region, } } +static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) +{ + uct_rocm_ipc_cache_region_t *region, *tmp; + + ucs_list_for_each_safe(region, tmp, deferred, list) { + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_signal_destroy(region, + region->retired_signal); + region->retired_signal.handle = 0; + uct_rocm_ipc_cache_region_unref(region); + } +} + static bool uct_rocm_ipc_cache_signal_cb(hsa_signal_value_t value, void *arg) { uct_rocm_ipc_cache_region_t *region = arg; @@ -462,19 +465,25 @@ static bool uct_rocm_ipc_cache_signal_cb(hsa_signal_value_t value, void *arg) hsa_signal_t signal = {.handle = 0}; int claimed; - (void)value; if (ucs_atomic_cswap32(®ion->handler_done, 0, 1) != 0) { return false; } ucs_list_head_init(&deferred); - ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); - claimed = uct_rocm_ipc_cache_region_drop_locked(region, &deferred); - if (region->signal.handle != 0) { + if (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) { + /* A stale or teardown drop already unlinked the region. */ signal = region->signal; region->signal.handle = 0; + claimed = 0; + } else { + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + claimed = uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + if (region->signal.handle != 0) { + signal = region->signal; + region->signal.handle = 0; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); } - ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); uct_rocm_ipc_cache_region_signal_destroy(region, signal); if (claimed) { @@ -510,7 +519,7 @@ static void uct_rocm_ipc_cache_purge(uct_rocm_ipc_cache_t *cache) ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { ucs_list_del(®ion->list); - uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + uct_rocm_ipc_cache_region_stale_drop_locked(region, &deferred); } ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); @@ -538,12 +547,14 @@ uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, void *from, to); } -/* Caller holds the remote-cache write lock. A new region is returned through +/* Caller holds the remote-cache write lock. The mapped region is returned + * through @a region_p. A new region is also returned through * @a watch_region_p so the caller can arm its IPC signal after releasing the * lock. @a deferred receives regions whose signals must be destroyed. */ static ucs_status_t uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, - uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_key_t *key, + uct_rocm_ipc_cache_region_t **region_p, uct_rocm_ipc_cache_region_t **watch_region_p, ucs_list_link_t *deferred) { @@ -553,6 +564,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, uct_rocm_ipc_cache_region_t *region; hsa_signal_value_t signal_value; hsa_status_t hsa_status; + void *mapped_addr; int ret; *watch_region_p = NULL; @@ -571,7 +583,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, cache->name, (void*)key->address, key->length, UCS_PGT_REGION_ARG(®ion->super)); - *mapped_addr = region->mapped_addr; + *region_p = region; return UCS_OK; } @@ -582,7 +594,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, cache->name, (void*)key->address, key->length, UCS_PGT_REGION_ARG(®ion->super)); - *mapped_addr = region->mapped_addr; + *region_p = region; return UCS_OK; } @@ -620,7 +632,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, } if (hsa_signal_load_scacquire(local_signal) <= 0) { - ucs_debug("%s: rocm ipc buffer %p was already released by the owner", + ucs_debug("%s: rocm ipc buffer %p already released by the owner", cache->name, (void*)key->address); status = UCS_ERR_INVALID_ADDR; goto err_destroy_signal; @@ -628,7 +640,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, } hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, - mapped_addr); + &mapped_addr); if (ucs_unlikely(hsa_status != HSA_STATUS_SUCCESS)) { ucs_error("%s: failed to open ipc mem handle. addr:%p len:%lu hsa_status:%d", cache->name, (void*)key->address, key->length, hsa_status); @@ -642,8 +654,8 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, "uct_rocm_ipc_cache_region"); if (ret != 0) { ucs_warn("failed to allocate uct_rocm_ipc_cache region"); - if (hsa_amd_ipc_memory_detach(*mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_fatal("failed to unmap addr:%p", *mapped_addr); + if (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", mapped_addr); } status = UCS_ERR_NO_MEMORY; goto err_destroy_signal; @@ -654,7 +666,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, region->super.end = ucs_align_up_pow2(key->address + key->length, UCS_PGT_ADDR_ALIGN); region->key = *key; - region->mapped_addr = *mapped_addr; + region->mapped_addr = mapped_addr; region->cache = cache; region->signal = local_signal; region->retired_signal.handle = 0; @@ -662,6 +674,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, region->deactivate = 0; region->handler_done = 0; region->signal_owner = 0; + region->pin_count = 0; region->in_pgtable = 0; status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, @@ -679,14 +692,15 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, " size:%lu :%s", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, ucs_status_string(status)); - if (hsa_amd_ipc_memory_detach(*mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_fatal("failed to unmap addr:%p", *mapped_addr); + if (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", mapped_addr); } ucs_free(region); goto err_destroy_signal; } region->in_pgtable = 1; + *region_p = region; if (uct_rocm_ipc_signal_invalidate_enabled()) { *watch_region_p = region; } @@ -847,12 +861,16 @@ uct_rocm_ipc_cache_region_watch(uct_rocm_ipc_cache_region_t *region) return status; } -ucs_status_t -uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) +/* Pins the mapping when @a pinned_p is not NULL. */ +static ucs_status_t +uct_rocm_ipc_cache_map(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **pinned_p) { uct_rocm_ipc_cache_hash_key_t hash_key = {key->pid, key->pid_ns, key->dev_num}; uct_rocm_ipc_cache_region_t *watch_region = NULL; + uct_rocm_ipc_cache_region_t *pinned = NULL; + uct_rocm_ipc_cache_region_t *region; uct_rocm_ipc_cache_t *cache; ucs_list_link_t deferred; ucs_status_t status; @@ -862,12 +880,20 @@ uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); status = uct_rocm_ipc_remote_cache_get(hash_key, &cache); if (status == UCS_OK) { - status = uct_rocm_ipc_cache_map_region(cache, key, &mapped, + status = uct_rocm_ipc_cache_map_region(cache, key, ®ion, &watch_region, &deferred); } - if ((status == UCS_OK) && (watch_region != NULL)) { - /* Keep the region alive until its IPC signal is armed. */ - ucs_atomic_add32(&watch_region->refcount, 1); + if (status == UCS_OK) { + mapped = region->mapped_addr; + if (watch_region != NULL) { + /* Keep the region alive until its IPC signal is armed. */ + ucs_atomic_add32(&watch_region->refcount, 1); + } + if (pinned_p != NULL) { + ++region->pin_count; + ucs_atomic_add32(®ion->refcount, 1); + pinned = region; + } } ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); @@ -875,10 +901,48 @@ uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) if ((status == UCS_OK) && (watch_region != NULL)) { status = uct_rocm_ipc_cache_region_watch(watch_region); } - if (status == UCS_OK) { - *mapped_addr = mapped; + if (status != UCS_OK) { + if (pinned != NULL) { + uct_rocm_ipc_cache_unpin(pinned); + } + return status; } - return status; + + *mapped_addr = mapped; + if (pinned_p != NULL) { + *pinned_p = pinned; + } + return UCS_OK; +} + +ucs_status_t +uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) +{ + return uct_rocm_ipc_cache_map(key, mapped_addr, NULL); +} + +ucs_status_t +uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **region_p) +{ + return uct_rocm_ipc_cache_map(key, mapped_addr, region_p); +} + +void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region) +{ + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + ucs_assert(region->pin_count > 0); + if ((--region->pin_count == 0) && + (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0)) { + if (hsa_amd_ipc_memory_detach(region->mapped_addr) != + HSA_STATUS_SUCCESS) { + ucs_warn("failed to unmap addr:%p", region->mapped_addr); + } + region->mapped_addr = NULL; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + + uct_rocm_ipc_cache_region_unref(region); } void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 43d6eb643a6..83980a23681 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -26,10 +26,11 @@ typedef struct { uct_rocm_ipc_cache_t *cache; /**< Cache that owns the region */ hsa_signal_t signal; /**< Attached IPC signal */ hsa_signal_t retired_signal; /**< Signal owned by claim winner */ - uint32_t refcount; /**< Pgtable, handler, and watcher */ + uint32_t refcount; /**< Pgtable, handler, watcher, pin */ uint32_t deactivate; /**< 1 after unmap is claimed */ uint32_t handler_done; /**< Async handler entered once */ uint32_t signal_owner; /**< 0 open, 1 claim, 2 handler */ + unsigned pin_count; /**< Consumers holding mapped_addr */ int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; @@ -41,6 +42,17 @@ void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache); ucs_status_t uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); +/** + * Map @a key and keep @a mapped_addr attached until + * @ref uct_rocm_ipc_cache_unpin, even if the cache invalidates the region. + * The owner's free of the buffer waits for that release. + */ +ucs_status_t +uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **region_p); + +void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region); + /** * Create or reuse an IPC signal for an exported buffer and remember it until * free. diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index 212946ba031..e3adb001423 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -18,6 +18,12 @@ #include +typedef struct { + uct_rocm_ipc_key_t super; + uct_rocm_ipc_cache_region_t *region; /**< Pinned by rkey_ptr */ + void *mapped_addr; /**< Mapping of @a region */ +} uct_rocm_ipc_unpacked_rkey_t; + static ucs_config_field_t uct_rocm_ipc_md_config_table[] = { {"", "", NULL, ucs_offsetof(uct_rocm_ipc_md_config_t, super), @@ -57,28 +63,28 @@ uct_rocm_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, return UCS_OK; } -static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, +static ucs_status_t uct_rocm_ipc_pack_key(void *address, size_t length, uct_rocm_ipc_key_t *key) { void *base_ptr = NULL; size_t size = 0; - hsa_status_t status; + hsa_status_t hsa_status; hsa_agent_t agent; hsa_amd_pointer_type_t mem_type; - status = uct_rocm_base_get_ptr_info(address, length, &base_ptr, &size, - &mem_type, &agent, NULL); - if ((status != HSA_STATUS_SUCCESS) || + hsa_status = uct_rocm_base_get_ptr_info(address, length, &base_ptr, &size, + &mem_type, &agent, NULL); + if ((hsa_status != HSA_STATUS_SUCCESS) || (mem_type == HSA_EXT_POINTER_TYPE_UNKNOWN)) { ucs_error("failed to get base ptr for %p/%lx, ROCm returned %p/%lx", address, length, base_ptr, size); - return status; + return UCS_ERR_INVALID_ADDR; } - status = hsa_amd_ipc_memory_create(base_ptr, size, &key->ipc); - if (status != HSA_STATUS_SUCCESS) { + hsa_status = hsa_amd_ipc_memory_create(base_ptr, size, &key->ipc); + if (hsa_status != HSA_STATUS_SUCCESS) { ucs_error("Failed to create ipc for %p/%lx", address, length); - return status; + return UCS_ERR_INVALID_ADDR; } key->address = (uintptr_t)base_ptr; @@ -88,12 +94,8 @@ static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, key->pid_ns = ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID); /* One IPC signal per allocation. Every importer of this key watches it. */ - if (uct_rocm_ipc_export_add(key->address, key->length, &key->ipc_signal) != - UCS_OK) { - return HSA_STATUS_ERROR_OUT_OF_RESOURCES; - } - - return HSA_STATUS_SUCCESS; + return uct_rocm_ipc_export_add(key->address, key->length, + &key->ipc_signal); } static ucs_status_t @@ -101,7 +103,7 @@ uct_rocm_ipc_mem_reg(uct_md_h md, void *address, size_t length, const uct_md_mem_reg_params_t *params, uct_mem_h *memh_p) { uct_rocm_ipc_key_t *key; - hsa_status_t status; + ucs_status_t status; key = ucs_malloc(sizeof(*key), "uct_rocm_ipc_key_t"); if (NULL == key) { @@ -110,9 +112,9 @@ uct_rocm_ipc_mem_reg(uct_md_h md, void *address, size_t length, } status = uct_rocm_ipc_pack_key(address, length, key); - if (status != HSA_STATUS_SUCCESS) { + if (status != UCS_OK) { ucs_free(key); - return UCS_ERR_INVALID_ADDR; + return status; } *memh_p = key; @@ -142,10 +144,12 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; uct_rocm_ipc_device_mem_element_t *rocm_ipc_mem_element = (uct_rocm_ipc_device_mem_element_t*)mem_elem_p; + uct_rocm_ipc_cache_region_t *region; void *mapped_addr; ucs_status_t status; - status = uct_rocm_ipc_cache_map_memhandle(key, &mapped_addr); + /* GPU kernels use the mapping directly, so it must outlive the element */ + status = uct_rocm_ipc_cache_pin_memhandle(key, &mapped_addr, ®ion); if (status != UCS_OK) { ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); return status; @@ -159,9 +163,15 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, (void*)key->address, mapped_addr, rocm_ipc_mem_element->mapped_offset); + *release_handle_p = region; return UCS_OK; } +static void uct_rocm_ipc_md_mem_elem_release(uct_md_h md, void *release_handle) +{ + uct_rocm_ipc_cache_unpin(release_handle); +} + static ucs_status_t uct_rocm_ipc_md_open(uct_component_h component, const char *md_name, const uct_md_config_t *uct_md_config, uct_md_h *md_p) @@ -185,7 +195,7 @@ uct_rocm_ipc_md_open(uct_component_h component, const char *md_name, ucs_empty_function_return_unsupported, .detect_memory_type = (uct_md_detect_memory_type_func_t) ucs_empty_function_return_unsupported, - .mem_elem_release = (uct_md_mem_elem_release_func_t)ucs_empty_function + .mem_elem_release = uct_rocm_ipc_md_mem_elem_release }; static uct_md_t md = { .ops = &md_ops, @@ -202,17 +212,19 @@ uct_rocm_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, uct_rkey_t *rkey_p, void **handle_p) { uct_rocm_ipc_key_t *packed = (uct_rocm_ipc_key_t *)rkey_buffer; - uct_rocm_ipc_key_t *key; + uct_rocm_ipc_unpacked_rkey_t *unpacked; - key = ucs_malloc(sizeof(uct_rocm_ipc_key_t), "uct_rocm_ipc_key_t"); - if (NULL == key) { - ucs_error("Failed to allocate memory for uct_rocm_ipc_key_t"); + unpacked = ucs_malloc(sizeof(*unpacked), "uct_rocm_ipc_unpacked_rkey_t"); + if (NULL == unpacked) { + ucs_error("Failed to allocate memory for uct_rocm_ipc_unpacked_rkey_t"); return UCS_ERR_NO_MEMORY; } - *key = *packed; - *handle_p = NULL; - *rkey_p = (uintptr_t)key; + unpacked->super = *packed; + unpacked->region = NULL; + unpacked->mapped_addr = NULL; + *handle_p = NULL; + *rkey_p = (uintptr_t)unpacked; return UCS_OK; } @@ -220,31 +232,43 @@ uct_rocm_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, static ucs_status_t uct_rocm_ipc_rkey_release(uct_component_t *component, uct_rkey_t rkey, void *handle) { + uct_rocm_ipc_unpacked_rkey_t *unpacked = + (uct_rocm_ipc_unpacked_rkey_t*)rkey; + ucs_assert(NULL == handle); - ucs_free((void *)rkey); + if (unpacked->region != NULL) { + uct_rocm_ipc_cache_unpin(unpacked->region); + } + ucs_free(unpacked); return UCS_OK; } ucs_status_t uct_rocm_ipc_rkey_ptr(uct_component_t *component, uct_rkey_t rkey, void *handle, uint64_t raddr, void **laddr_p) { - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; - void *mapped_addr; + uct_rocm_ipc_unpacked_rkey_t *unpacked = + (uct_rocm_ipc_unpacked_rkey_t*)rkey; ptrdiff_t offset; ucs_status_t status; - status = uct_rocm_ipc_cache_map_memhandle(key, &mapped_addr); - if (status != UCS_OK) { - ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); - return status; + /* The returned address is valid until the rkey is released */ + if (unpacked->region == NULL) { + status = uct_rocm_ipc_cache_pin_memhandle(&unpacked->super, + &unpacked->mapped_addr, + &unpacked->region); + if (status != UCS_OK) { + ucs_error("Failed to map IPC handle: %s", + ucs_status_string(status)); + return status; + } } /* Calculate offset from base address */ - offset = UCS_PTR_BYTE_DIFF(key->address, raddr); - *laddr_p = UCS_PTR_BYTE_OFFSET(mapped_addr, offset); + offset = UCS_PTR_BYTE_DIFF(unpacked->super.address, raddr); + *laddr_p = UCS_PTR_BYTE_OFFSET(unpacked->mapped_addr, offset); ucs_trace("rocm_ipc rkey_ptr: raddr=%p mapped=%p offset=%ld laddr=%p", - (void*)raddr, mapped_addr, offset, *laddr_p); + (void*)raddr, unpacked->mapped_addr, offset, *laddr_p); return UCS_OK; } From 86f081837cdb36922360f0856439b7204e3f4004 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 15:32:35 +0000 Subject: [PATCH 08/16] UCT/ROCM: Added hash based lookup --- src/uct/rocm/ipc/rocm_ipc_cache.c | 87 +++++++++++++++---------------- src/uct/rocm/ipc/rocm_ipc_cache.h | 2 +- 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 770ea5b4d94..daf25f3f1d7 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -119,7 +119,9 @@ typedef struct uct_rocm_ipc_export { } uct_rocm_ipc_export_t; -static ucs_list_link_t uct_rocm_ipc_export_active; +KHASH_MAP_INIT_INT64(rocm_ipc_exports, uct_rocm_ipc_export_t*); + +static khash_t(rocm_ipc_exports) uct_rocm_ipc_exports; static ucs_list_link_t uct_rocm_ipc_export_retired; static ucs_spinlock_t uct_rocm_ipc_export_lock; static ucs_init_once_t uct_rocm_ipc_export_handler_once = @@ -152,21 +154,6 @@ static void uct_rocm_ipc_export_destroy(uct_rocm_ipc_export_t *exp) ucs_free(exp); } -/* Caller holds the export lock. */ -static uct_rocm_ipc_export_t * -uct_rocm_ipc_export_find_active(uintptr_t address, size_t length) -{ - uct_rocm_ipc_export_t *exp; - - ucs_list_for_each(exp, &uct_rocm_ipc_export_active, list) { - if ((exp->address == address) && (exp->length == length)) { - return exp; - } - } - - return NULL; -} - /* An importer attaches the IPC signal before it maps memory. Therefore, after * the free event has been dispatched, every mapping that can delay hipFree * already holds its own signal reference and any thread may reap the export. */ @@ -192,28 +179,25 @@ static void uct_rocm_ipc_export_reap(void) static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, ucm_event_t *event, void *arg) { - uct_rocm_ipc_export_t *exp, *tmp; - uintptr_t start, end; + uct_rocm_ipc_export_t *exp; + khiter_t iter; + uintptr_t address; if ((event_type != UCM_EVENT_MEM_TYPE_FREE) || (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM)) { return; } - start = (uintptr_t)event->mem_type.address; - end = start + event->mem_type.size; + address = (uintptr_t)event->mem_type.address; /* Importers acquire a signal reference before mapping memory, so the * exported signal can be destroyed after this callback on any thread. */ ucs_spin_lock(&uct_rocm_ipc_export_lock); - ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { - if ((exp->address >= end) || - ((exp->address + exp->length) <= start)) { - continue; - } - + iter = kh_get(rocm_ipc_exports, &uct_rocm_ipc_exports, address); + if (iter != kh_end(&uct_rocm_ipc_exports)) { + exp = kh_val(&uct_rocm_ipc_exports, iter); + kh_del(rocm_ipc_exports, &uct_rocm_ipc_exports, iter); hsa_signal_store_screlease(exp->signal, 0); - ucs_list_del(&exp->list); ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, exp->length); @@ -239,6 +223,8 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, { uct_rocm_ipc_export_t *exp, *existing; hsa_status_t hsa_status; + khiter_t iter; + int ret; if (uct_rocm_ipc_signal_invalidate_enabled()) { UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { @@ -249,9 +235,9 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, } ucs_spin_lock(&uct_rocm_ipc_export_lock); - existing = uct_rocm_ipc_export_find_active(address, length); - if (existing != NULL) { - *ipc_signal_p = existing->ipc_signal; + iter = kh_get(rocm_ipc_exports, &uct_rocm_ipc_exports, address); + if (iter != kh_end(&uct_rocm_ipc_exports)) { + *ipc_signal_p = kh_val(&uct_rocm_ipc_exports, iter)->ipc_signal; ucs_spin_unlock(&uct_rocm_ipc_export_lock); return UCS_OK; } @@ -284,16 +270,23 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, exp->length = length; ucs_spin_lock(&uct_rocm_ipc_export_lock); - existing = uct_rocm_ipc_export_find_active(address, length); - if (existing == NULL) { - ucs_list_add_tail(&uct_rocm_ipc_export_active, &exp->list); + iter = kh_put(rocm_ipc_exports, &uct_rocm_ipc_exports, address, &ret); + if (ret == UCS_KH_PUT_FAILED) { + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + uct_rocm_ipc_export_destroy(exp); + return UCS_ERR_NO_MEMORY; + } + + if (ret != UCS_KH_PUT_KEY_PRESENT) { + kh_val(&uct_rocm_ipc_exports, iter) = exp; *ipc_signal_p = exp->ipc_signal; } else { + existing = kh_val(&uct_rocm_ipc_exports, iter); *ipc_signal_p = existing->ipc_signal; } ucs_spin_unlock(&uct_rocm_ipc_export_lock); - if (existing != NULL) { + if (ret == UCS_KH_PUT_KEY_PRESENT) { uct_rocm_ipc_export_destroy(exp); } @@ -312,13 +305,13 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_head_init(&destroy_list); ucs_spin_lock(&uct_rocm_ipc_export_lock); - ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_active, list) { + kh_foreach_value(&uct_rocm_ipc_exports, exp, { if (uct_rocm_ipc_signal_invalidate_enabled()) { hsa_signal_store_screlease(exp->signal, 0); } - ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); - } + }) + kh_clear(rocm_ipc_exports, &uct_rocm_ipc_exports); ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { ucs_list_del(&exp->list); ucs_list_add_tail(&destroy_list, &exp->list); @@ -389,7 +382,7 @@ uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, } /* A registered handler owns the signal and destroys it when it runs. */ - if (ucs_atomic_cswap32(®ion->signal_owner, 0, 1) == 0) { + if (!region->handler_armed) { region->retired_signal = region->signal; region->signal.handle = 0; } @@ -422,13 +415,13 @@ static void uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, ucs_list_link_t *deferred) { - int handler_owned; - - handler_owned = (ucs_atomic_cswap32(®ion->signal_owner, 2, 2) == 2); - uct_rocm_ipc_cache_region_drop_locked(region, deferred); - if (handler_owned) { + if (region->handler_armed) { + /* Wake the handler before the region is deactivated, so that it takes + * the locked path and does not race with this store. */ hsa_signal_store_screlease(region->signal, 0); } + + uct_rocm_ipc_cache_region_drop_locked(region, deferred); } static void @@ -673,7 +666,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, region->refcount = 1; region->deactivate = 0; region->handler_done = 0; - region->signal_owner = 0; + region->handler_armed = 0; region->pin_count = 0; region->in_pgtable = 0; @@ -806,12 +799,13 @@ uct_rocm_ipc_cache_region_watch(uct_rocm_ipc_cache_region_t *region) ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); if ((ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) || (region->signal.handle == 0) || - (ucs_atomic_cswap32(®ion->signal_owner, 0, 2) != 0)) { + region->handler_armed) { /* The mapping was claimed before the handler existed. */ ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); status = UCS_ERR_INVALID_ADDR; goto out_unref; } + region->handler_armed = 1; ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); ucs_atomic_add32(®ion->refcount, 1); @@ -964,7 +958,7 @@ UCS_STATIC_INIT if (status != UCS_OK) { ucs_fatal("failed to initialize rocm ipc export lock"); } - ucs_list_head_init(&uct_rocm_ipc_export_active); + kh_init_inplace(rocm_ipc_exports, &uct_rocm_ipc_exports); ucs_list_head_init(&uct_rocm_ipc_export_retired); } @@ -976,6 +970,7 @@ UCS_STATIC_CLEANUP kh_foreach_value(&uct_rocm_ipc_remote_cache.hash, cache, { uct_rocm_ipc_destroy_cache(cache); }) kh_destroy_inplace(rocm_ipc_rem_cache, &uct_rocm_ipc_remote_cache.hash); + kh_destroy_inplace(rocm_ipc_exports, &uct_rocm_ipc_exports); ucs_rw_spinlock_cleanup(&uct_rocm_ipc_remote_cache.lock); ucs_spinlock_destroy(&uct_rocm_ipc_export_lock); } diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 83980a23681..598a8175753 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -29,7 +29,7 @@ typedef struct { uint32_t refcount; /**< Pgtable, handler, watcher, pin */ uint32_t deactivate; /**< 1 after unmap is claimed */ uint32_t handler_done; /**< Async handler entered once */ - uint32_t signal_owner; /**< 0 open, 1 claim, 2 handler */ + int handler_armed; /**< Async handler owns signal */ unsigned pin_count; /**< Consumers holding mapped_addr */ int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; From 7dc616d824b1184256748037d74796203c52fedd Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 15:54:12 +0000 Subject: [PATCH 09/16] UCT/ROCM: Fixes based on review --- src/uct/rocm/ipc/rocm_ipc_cache.h | 8 +++++++- src/uct/rocm/ipc/rocm_ipc_ep.c | 12 ++++++++---- src/uct/rocm/ipc/rocm_ipc_md.c | 25 ++++++++++++------------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 598a8175753..201181f62ec 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -34,6 +34,12 @@ typedef struct { int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; +typedef struct { + uct_rocm_ipc_key_t super; + uct_rocm_ipc_cache_region_t *region; /**< Pinned by rkey_ptr */ + void *mapped_addr; /**< Mapping of @a region */ +} uct_rocm_ipc_unpacked_rkey_t; + ucs_status_t uct_rocm_ipc_create_cache(uct_rocm_ipc_cache_t **cache, const char *name); @@ -45,7 +51,7 @@ uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); /** * Map @a key and keep @a mapped_addr attached until * @ref uct_rocm_ipc_cache_unpin, even if the cache invalidates the region. - * The owner's free of the buffer waits for that release. + * The caller must keep the owner allocation alive until it unpins the mapping. */ ucs_status_t uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, diff --git a/src/uct/rocm/ipc/rocm_ipc_ep.c b/src/uct/rocm/ipc/rocm_ipc_ep.c index 7898c028649..de1e83d2feb 100644 --- a/src/uct/rocm/ipc/rocm_ipc_ep.c +++ b/src/uct/rocm/ipc/rocm_ipc_ep.c @@ -180,11 +180,13 @@ ucs_status_t uct_rocm_ipc_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); ucs_status_t ret; - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t *)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, - iov, key, comp, 1); + iov, &unpacked->super, comp, 1); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, ZCOPY, uct_iov_total_length(iov, iovcnt)); @@ -198,11 +200,13 @@ ucs_status_t uct_rocm_ipc_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); ucs_status_t ret; - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t *)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, - iov, key, comp, 0); + iov, &unpacked->super, comp, 0); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, ZCOPY, uct_iov_total_length(iov, iovcnt)); diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index e3adb001423..e282a9db30d 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -18,12 +18,6 @@ #include -typedef struct { - uct_rocm_ipc_key_t super; - uct_rocm_ipc_cache_region_t *region; /**< Pinned by rkey_ptr */ - void *mapped_addr; /**< Mapping of @a region */ -} uct_rocm_ipc_unpacked_rkey_t; - static ucs_config_field_t uct_rocm_ipc_md_config_table[] = { {"", "", NULL, ucs_offsetof(uct_rocm_ipc_md_config_t, super), @@ -141,7 +135,9 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, uct_device_mem_elem_t *mem_elem_p, void **release_handle_p) { - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); uct_rocm_ipc_device_mem_element_t *rocm_ipc_mem_element = (uct_rocm_ipc_device_mem_element_t*)mem_elem_p; uct_rocm_ipc_cache_region_t *region; @@ -149,18 +145,19 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, ucs_status_t status; /* GPU kernels use the mapping directly, so it must outlive the element */ - status = uct_rocm_ipc_cache_pin_memhandle(key, &mapped_addr, ®ion); + status = uct_rocm_ipc_cache_pin_memhandle(&unpacked->super, &mapped_addr, + ®ion); if (status != UCS_OK) { ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); return status; } /* Calculate offset: base_address - mapped_address */ - rocm_ipc_mem_element->mapped_offset = UCS_PTR_BYTE_DIFF(key->address, - mapped_addr); + rocm_ipc_mem_element->mapped_offset = + UCS_PTR_BYTE_DIFF(unpacked->super.address, mapped_addr); ucs_trace("rocm_ipc mem_elem_pack: key_addr=%p mapped=%p offset=%ld", - (void*)key->address, mapped_addr, + (void*)unpacked->super.address, mapped_addr, rocm_ipc_mem_element->mapped_offset); *release_handle_p = region; @@ -233,7 +230,8 @@ static ucs_status_t uct_rocm_ipc_rkey_release(uct_component_t *component, uct_rkey_t rkey, void *handle) { uct_rocm_ipc_unpacked_rkey_t *unpacked = - (uct_rocm_ipc_unpacked_rkey_t*)rkey; + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); ucs_assert(NULL == handle); if (unpacked->region != NULL) { @@ -247,7 +245,8 @@ ucs_status_t uct_rocm_ipc_rkey_ptr(uct_component_t *component, uct_rkey_t rkey, void *handle, uint64_t raddr, void **laddr_p) { uct_rocm_ipc_unpacked_rkey_t *unpacked = - (uct_rocm_ipc_unpacked_rkey_t*)rkey; + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); ptrdiff_t offset; ucs_status_t status; From 74dfd9d543395ced0130dbddf90cf1ea2474f2ea Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 16:18:55 +0000 Subject: [PATCH 10/16] UCT/ROCM: Fix for review feedback --- src/uct/rocm/ipc/rocm_ipc_cache.c | 28 ++++++++++++++-------------- src/uct/rocm/ipc/rocm_ipc_cache.h | 2 +- src/uct/rocm/ipc/rocm_ipc_md.c | 12 ++++++++++-- src/uct/rocm/ipc/rocm_ipc_md.h | 2 +- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index daf25f3f1d7..5044ae135b3 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -251,7 +251,7 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, &exp->signal); if (hsa_status != HSA_STATUS_SUCCESS) { - ucs_error("failed to create rocm ipc signal for %p length %zu: %d", + ucs_debug("failed to create rocm ipc signal for %p length %zu: %d", (void*)address, length, hsa_status); ucs_free(exp); return UCS_ERR_IO_ERROR; @@ -259,7 +259,7 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_status = hsa_amd_ipc_signal_create(exp->signal, &exp->ipc_signal); if (hsa_status != HSA_STATUS_SUCCESS) { - ucs_error("failed to export rocm ipc signal for %p length %zu: %d", + ucs_debug("failed to export rocm ipc signal for %p length %zu: %d", (void*)address, length, hsa_status); hsa_signal_destroy(exp->signal); ucs_free(exp); @@ -604,23 +604,23 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred); } - if (uct_rocm_ipc_signal_invalidate_enabled()) { - if (uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { - ucs_error("%s: rocm ipc key %p has no ipc signal", cache->name, - (void*)key->address); - status = UCS_ERR_INVALID_PARAM; - goto err; - } - + if (uct_rocm_ipc_signal_invalidate_enabled() && + !uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { /* Acquire the signal reference before mapping memory. Once the * mapping exists, the exporter may be blocked in hipFree and may * destroy its signal reference after dispatching the free event. */ hsa_status = hsa_amd_ipc_signal_attach(&key->ipc_signal, &local_signal); if (hsa_status != HSA_STATUS_SUCCESS) { - ucs_error("%s: failed to attach rocm ipc signal for %p: %d", - cache->name, (void*)key->address, hsa_status); - status = UCS_ERR_IO_ERROR; + if (hsa_status == HSA_STATUS_ERROR_INVALID_ARGUMENT) { + ucs_debug("%s: rocm ipc buffer %p already released by the owner", + cache->name, (void*)key->address); + status = UCS_ERR_INVALID_ADDR; + } else { + ucs_error("%s: failed to attach rocm ipc signal for %p: %d", + cache->name, (void*)key->address, hsa_status); + status = UCS_ERR_IO_ERROR; + } goto err; } @@ -694,7 +694,7 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, region->in_pgtable = 1; *region_p = region; - if (uct_rocm_ipc_signal_invalidate_enabled()) { + if (local_signal.handle != 0) { *watch_region_p = region; } ucs_trace("%s: rocm_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 201181f62ec..bc627b24c2a 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -51,7 +51,7 @@ uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); /** * Map @a key and keep @a mapped_addr attached until * @ref uct_rocm_ipc_cache_unpin, even if the cache invalidates the region. - * The caller must keep the owner allocation alive until it unpins the mapping. + * Known limitation: a pin does not keep the owner allocation alive remotely. */ ucs_status_t uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index e282a9db30d..9b06ade8ab9 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -63,6 +63,7 @@ static ucs_status_t uct_rocm_ipc_pack_key(void *address, size_t length, void *base_ptr = NULL; size_t size = 0; hsa_status_t hsa_status; + ucs_status_t status; hsa_agent_t agent; hsa_amd_pointer_type_t mem_type; @@ -88,8 +89,15 @@ static ucs_status_t uct_rocm_ipc_pack_key(void *address, size_t length, key->pid_ns = ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID); /* One IPC signal per allocation. Every importer of this key watches it. */ - return uct_rocm_ipc_export_add(key->address, key->length, - &key->ipc_signal); + memset(&key->ipc_signal, 0, sizeof(key->ipc_signal)); + status = uct_rocm_ipc_export_add(key->address, key->length, + &key->ipc_signal); + if (status != UCS_OK) { + ucs_warn("failed to export rocm ipc invalidation signal for %p: %s", + address, ucs_status_string(status)); + } + + return UCS_OK; } static ucs_status_t diff --git a/src/uct/rocm/ipc/rocm_ipc_md.h b/src/uct/rocm/ipc/rocm_ipc_md.h index 40724cf13c9..0f0da1cedb4 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.h +++ b/src/uct/rocm/ipc/rocm_ipc_md.h @@ -31,7 +31,7 @@ typedef struct uct_rocm_ipc_md_config { typedef struct uct_rocm_ipc_key { hsa_amd_ipc_memory_t ipc; /* Owner sets this signal to 0 when the allocation is released. - * Unused when UCX_ROCM_IPC_SIGNAL_INVALIDATE is n. */ + * Unused when invalidation is disabled or unavailable. */ hsa_amd_ipc_signal_t ipc_signal; uintptr_t address; size_t length; From 58238849890dab6dbf9ee10ea210507101b82cd0 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 16:39:14 +0000 Subject: [PATCH 11/16] UCT/ROCM: Merged two cache-hit branches --- src/uct/rocm/ipc/rocm_ipc_cache.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 5044ae135b3..f0476b033c3 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -555,7 +555,6 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, ucs_status_t status; ucs_pgt_region_t *pgt_region; uct_rocm_ipc_cache_region_t *region; - hsa_signal_value_t signal_value; hsa_status_t hsa_status; void *mapped_addr; int ret; @@ -570,18 +569,8 @@ uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); if (memcmp(&key->ipc, ®ion->key.ipc, sizeof(key->ipc)) == 0) { - if (region->signal.handle == 0) { - ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, - cache->name, (void*)key->address, key->length, - UCS_PGT_REGION_ARG(®ion->super)); - - *region_p = region; - return UCS_OK; - } - - signal_value = hsa_signal_load_scacquire(region->signal); - if (signal_value > 0) { + if ((region->signal.handle == 0) || + (hsa_signal_load_scacquire(region->signal) > 0)) { ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" UCS_PGT_REGION_FMT, cache->name, (void*)key->address, key->length, From cad4dc1807fdbac7d77ab540dc7c59f2c943531a Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 17:13:10 +0000 Subject: [PATCH 12/16] UCT/ROCM: Added early return --- src/uct/rocm/ipc/rocm_ipc_cache.c | 13 ++++++++----- src/uct/rocm/ipc/rocm_ipc_cache.h | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index f0476b033c3..c08d3c26b94 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -226,14 +226,17 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, khiter_t iter; int ret; - if (uct_rocm_ipc_signal_invalidate_enabled()) { - UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { - uct_rocm_ipc_export_install_handler(); - } + if (!uct_rocm_ipc_signal_invalidate_enabled()) { + memset(ipc_signal_p, 0, sizeof(*ipc_signal_p)); + return UCS_OK; + } - uct_rocm_ipc_export_reap(); + UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { + uct_rocm_ipc_export_install_handler(); } + uct_rocm_ipc_export_reap(); + ucs_spin_lock(&uct_rocm_ipc_export_lock); iter = kh_get(rocm_ipc_exports, &uct_rocm_ipc_exports, address); if (iter != kh_end(&uct_rocm_ipc_exports)) { diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index bc627b24c2a..0212574e25b 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -61,7 +61,7 @@ void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region); /** * Create or reuse an IPC signal for an exported buffer and remember it until - * free. + * free. Returns an empty signal when invalidation is disabled. * The packed @a ipc_signal is the same for every peer that imports this key. */ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, From a7f3987eb962596f2b4d3cb24dd04a593ec7f03e Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 17:41:20 +0000 Subject: [PATCH 13/16] UCT/ROCM: Fixes based on review feedback --- src/uct/rocm/ipc/rocm_ipc_cache.c | 19 ++++++++++++++----- src/uct/rocm/ipc/rocm_ipc_cache.h | 2 ++ src/uct/rocm/ipc/rocm_ipc_ep.c | 12 ++++-------- src/uct/rocm/ipc/rocm_ipc_md.c | 1 + 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index c08d3c26b94..b6320dbae6d 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -73,7 +73,9 @@ typedef struct { static ucs_config_field_t uct_rocm_ipc_cache_config_table[] = { {"SIGNAL_INVALIDATE", "y", "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" - "The IPC signal remains part of the packed rkey when this is n.", + "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" + "When set to n, a mapping is replaced only when the peer's IPC handle\n" + "changes. The packed rkey layout is the same in both cases.", ucs_offsetof(uct_rocm_ipc_cache_config_t, signal_invalidate), UCS_CONFIG_TYPE_BOOL}, @@ -218,6 +220,17 @@ static void uct_rocm_ipc_export_install_handler(void) } } +void uct_rocm_ipc_export_init(void) +{ + if (!uct_rocm_ipc_signal_invalidate_enabled()) { + return; + } + + UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { + uct_rocm_ipc_export_install_handler(); + } +} + ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, hsa_amd_ipc_signal_t *ipc_signal_p) { @@ -231,10 +244,6 @@ ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, return UCS_OK; } - UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { - uct_rocm_ipc_export_install_handler(); - } - uct_rocm_ipc_export_reap(); ucs_spin_lock(&uct_rocm_ipc_export_lock); diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 0212574e25b..67dbf7217fc 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -59,6 +59,8 @@ uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region); +void uct_rocm_ipc_export_init(void); + /** * Create or reuse an IPC signal for an exported buffer and remember it until * free. Returns an empty signal when invalidation is disabled. diff --git a/src/uct/rocm/ipc/rocm_ipc_ep.c b/src/uct/rocm/ipc/rocm_ipc_ep.c index de1e83d2feb..9b5e15f5f73 100644 --- a/src/uct/rocm/ipc/rocm_ipc_ep.c +++ b/src/uct/rocm/ipc/rocm_ipc_ep.c @@ -180,13 +180,11 @@ ucs_status_t uct_rocm_ipc_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { - uct_rocm_ipc_unpacked_rkey_t *unpacked = - ucs_derived_of((uct_rocm_ipc_key_t*)rkey, - uct_rocm_ipc_unpacked_rkey_t); ucs_status_t ret; + uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, - iov, &unpacked->super, comp, 1); + iov, key, comp, 1); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), PUT, ZCOPY, uct_iov_total_length(iov, iovcnt)); @@ -200,13 +198,11 @@ ucs_status_t uct_rocm_ipc_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { - uct_rocm_ipc_unpacked_rkey_t *unpacked = - ucs_derived_of((uct_rocm_ipc_key_t*)rkey, - uct_rocm_ipc_unpacked_rkey_t); ucs_status_t ret; + uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, - iov, &unpacked->super, comp, 0); + iov, key, comp, 0); UCT_TL_EP_STAT_OP(ucs_derived_of(tl_ep, uct_base_ep_t), GET, ZCOPY, uct_iov_total_length(iov, iovcnt)); diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index 9b06ade8ab9..e2ecc0e0433 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -207,6 +207,7 @@ uct_rocm_ipc_md_open(uct_component_h component, const char *md_name, .component = &uct_rocm_ipc_component.super, }; + uct_rocm_ipc_export_init(); *md_p = &md; return UCS_OK; } From 807db63962c0633499b15fe7afe42de41f9f68f6 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Tue, 27 Oct 2026 18:03:13 +0000 Subject: [PATCH 14/16] UCT/ROCM: Fixes to address reviewer requests --- src/uct/rocm/ipc/rocm_ipc_cache.c | 33 +++++++++++-------------------- src/uct/rocm/ipc/rocm_ipc_cache.h | 7 ++----- src/uct/rocm/ipc/rocm_ipc_ep.c | 2 +- src/uct/rocm/ipc/rocm_ipc_md.c | 6 +++--- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index b6320dbae6d..394703c6909 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -72,7 +72,6 @@ typedef struct { static ucs_config_field_t uct_rocm_ipc_cache_config_table[] = { {"SIGNAL_INVALIDATE", "y", - "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" "When set to n, a mapping is replaced only when the peer's IPC handle\n" "changes. The packed rkey layout is the same in both cases.", @@ -184,6 +183,7 @@ static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, uct_rocm_ipc_export_t *exp; khiter_t iter; uintptr_t address; + int reaped = 0; if ((event_type != UCM_EVENT_MEM_TYPE_FREE) || (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM)) { @@ -201,10 +201,15 @@ static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, kh_del(rocm_ipc_exports, &uct_rocm_ipc_exports, iter); hsa_signal_store_screlease(exp->signal, 0); ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); + reaped = 1; ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, exp->length); } ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + if (reaped) { + uct_rocm_ipc_export_reap(); + } } static void uct_rocm_ipc_export_install_handler(void) @@ -856,10 +861,9 @@ uct_rocm_ipc_cache_region_watch(uct_rocm_ipc_cache_region_t *region) return status; } -/* Pins the mapping when @a pinned_p is not NULL. */ -static ucs_status_t -uct_rocm_ipc_cache_map(uct_rocm_ipc_key_t *key, void **mapped_addr, - uct_rocm_ipc_cache_region_t **pinned_p) +ucs_status_t +uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **region_p) { uct_rocm_ipc_cache_hash_key_t hash_key = {key->pid, key->pid_ns, key->dev_num}; @@ -884,7 +888,7 @@ uct_rocm_ipc_cache_map(uct_rocm_ipc_key_t *key, void **mapped_addr, /* Keep the region alive until its IPC signal is armed. */ ucs_atomic_add32(&watch_region->refcount, 1); } - if (pinned_p != NULL) { + if (region_p != NULL) { ++region->pin_count; ucs_atomic_add32(®ion->refcount, 1); pinned = region; @@ -904,25 +908,12 @@ uct_rocm_ipc_cache_map(uct_rocm_ipc_key_t *key, void **mapped_addr, } *mapped_addr = mapped; - if (pinned_p != NULL) { - *pinned_p = pinned; + if (region_p != NULL) { + *region_p = pinned; } return UCS_OK; } -ucs_status_t -uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) -{ - return uct_rocm_ipc_cache_map(key, mapped_addr, NULL); -} - -ucs_status_t -uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, - uct_rocm_ipc_cache_region_t **region_p) -{ - return uct_rocm_ipc_cache_map(key, mapped_addr, region_p); -} - void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region) { ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index 67dbf7217fc..eb04356f1dd 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -45,16 +45,13 @@ ucs_status_t uct_rocm_ipc_create_cache(uct_rocm_ipc_cache_t **cache, void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache); -ucs_status_t -uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); - /** - * Map @a key and keep @a mapped_addr attached until + * Map @a key. A non-NULL @a region_p pins the mapping until * @ref uct_rocm_ipc_cache_unpin, even if the cache invalidates the region. * Known limitation: a pin does not keep the owner allocation alive remotely. */ ucs_status_t -uct_rocm_ipc_cache_pin_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, +uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, uct_rocm_ipc_cache_region_t **region_p); void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region); diff --git a/src/uct/rocm/ipc/rocm_ipc_ep.c b/src/uct/rocm/ipc/rocm_ipc_ep.c index 9b5e15f5f73..ef796ccadca 100644 --- a/src/uct/rocm/ipc/rocm_ipc_ep.c +++ b/src/uct/rocm/ipc/rocm_ipc_ep.c @@ -86,7 +86,7 @@ ucs_status_t uct_rocm_ipc_ep_zcopy(uct_ep_h tl_ep, } if (iface->config.enable_ipc_handle_cache) { - ret = uct_rocm_ipc_cache_map_memhandle(key, &remote_base_addr); + ret = uct_rocm_ipc_cache_map_memhandle(key, &remote_base_addr, NULL); if (ucs_unlikely(ret != UCS_OK)) { ucs_error("fail to attach ipc mem %p %d\n", (void*)key->address, ret); diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index e2ecc0e0433..6774bac5f32 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -93,7 +93,7 @@ static ucs_status_t uct_rocm_ipc_pack_key(void *address, size_t length, status = uct_rocm_ipc_export_add(key->address, key->length, &key->ipc_signal); if (status != UCS_OK) { - ucs_warn("failed to export rocm ipc invalidation signal for %p: %s", + ucs_diag("failed to export rocm ipc invalidation signal for %p: %s", address, ucs_status_string(status)); } @@ -153,7 +153,7 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, ucs_status_t status; /* GPU kernels use the mapping directly, so it must outlive the element */ - status = uct_rocm_ipc_cache_pin_memhandle(&unpacked->super, &mapped_addr, + status = uct_rocm_ipc_cache_map_memhandle(&unpacked->super, &mapped_addr, ®ion); if (status != UCS_OK) { ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); @@ -261,7 +261,7 @@ ucs_status_t uct_rocm_ipc_rkey_ptr(uct_component_t *component, uct_rkey_t rkey, /* The returned address is valid until the rkey is released */ if (unpacked->region == NULL) { - status = uct_rocm_ipc_cache_pin_memhandle(&unpacked->super, + status = uct_rocm_ipc_cache_map_memhandle(&unpacked->super, &unpacked->mapped_addr, &unpacked->region); if (status != UCS_OK) { From d37338eb0afcbed1224f699c748526d51406d041 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Wed, 28 Oct 2026 14:21:34 +0000 Subject: [PATCH 15/16] UCT/ROCM: Moved config parameter --- src/uct/rocm/ipc/rocm_ipc_cache.c | 47 +------------------------------ src/uct/rocm/ipc/rocm_ipc_iface.c | 34 ++++++++++++++++++++++ src/uct/rocm/ipc/rocm_ipc_iface.h | 3 ++ 3 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 394703c6909..0a3b40884c9 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -10,10 +10,10 @@ #endif #include "rocm_ipc_cache.h" +#include "rocm_ipc_iface.h" #include #include -#include #include #include #include @@ -66,51 +66,6 @@ static uct_rocm_ipc_remote_cache_t uct_rocm_ipc_remote_cache; /* Runs before the registration cache (priority 1000). */ #define UCT_ROCM_IPC_MEM_EVENT_PRIO 100 -typedef struct { - int signal_invalidate; -} uct_rocm_ipc_cache_config_t; - -static ucs_config_field_t uct_rocm_ipc_cache_config_table[] = { - {"SIGNAL_INVALIDATE", "y", - "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" - "When set to n, a mapping is replaced only when the peer's IPC handle\n" - "changes. The packed rkey layout is the same in both cases.", - ucs_offsetof(uct_rocm_ipc_cache_config_t, signal_invalidate), - UCS_CONFIG_TYPE_BOOL}, - - {NULL} -}; - -UCS_CONFIG_REGISTER_TABLE(uct_rocm_ipc_cache_config_table, "ROCm IPC", - "ROCM_IPC_", uct_rocm_ipc_cache_config_t, - &ucs_config_global_list); - -static int uct_rocm_ipc_signal_invalidate_enabled(void) -{ - static ucs_init_once_t once = UCS_INIT_ONCE_INITIALIZER; - static int enabled = 1; - - UCS_INIT_ONCE(&once) { - uct_rocm_ipc_cache_config_t config; - ucs_status_t status; - - status = ucs_config_parser_fill_opts( - &config, UCS_CONFIG_GET_TABLE(uct_rocm_ipc_cache_config_table), - UCS_DEFAULT_ENV_PREFIX, 0); - if (status == UCS_OK) { - enabled = config.signal_invalidate; - ucs_config_parser_release_opts(&config, - uct_rocm_ipc_cache_config_table); - } - if (!enabled) { - ucs_diag("rocm ipc signal invalidation is disabled"); - } - } - - return enabled; -} - - typedef struct uct_rocm_ipc_export { ucs_list_link_t list; uintptr_t address; diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.c b/src/uct/rocm/ipc/rocm_ipc_iface.c index d2a00092142..3d3f0920874 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.c +++ b/src/uct/rocm/ipc/rocm_ipc_iface.c @@ -15,6 +15,9 @@ #include #include +#include +#include + static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = { {"", "", NULL, ucs_offsetof(uct_rocm_ipc_iface_config_t, super), @@ -38,6 +41,13 @@ static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = { ucs_offsetof(uct_rocm_ipc_iface_config_t, params.sigpool_max_elems), UCS_CONFIG_TYPE_UINT}, + {"SIGNAL_INVALIDATE", "y", + "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" + "When set to n, a mapping is replaced only when the peer's IPC handle\n" + "changes. The packed rkey layout is the same in both cases.", + ucs_offsetof(uct_rocm_ipc_iface_config_t, params.signal_invalidate), + UCS_CONFIG_TYPE_BOOL}, + {NULL} }; @@ -263,3 +273,27 @@ static UCS_CLASS_DEFINE_DELETE_FUNC(uct_rocm_ipc_iface_t, uct_iface_t); UCT_TL_DEFINE(&uct_rocm_ipc_component.super, rocm_ipc, uct_rocm_base_query_devices, uct_rocm_ipc_iface_t, "ROCM_IPC_", uct_rocm_ipc_iface_config_table, uct_rocm_ipc_iface_config_t); + +int uct_rocm_ipc_signal_invalidate_enabled(void) +{ + static ucs_init_once_t once = UCS_INIT_ONCE_INITIALIZER; + static int enabled = 1; + + UCS_INIT_ONCE(&once) { + uct_rocm_ipc_iface_config_t config; + ucs_status_t status; + + status = ucs_config_parser_fill_opts(&config, &uct_rocm_ipc_tl.config, + UCS_DEFAULT_ENV_PREFIX, 0); + if (status == UCS_OK) { + enabled = config.params.signal_invalidate; + ucs_config_parser_release_opts(&config, + uct_rocm_ipc_iface_config_table); + } + if (!enabled) { + ucs_diag("rocm ipc signal invalidation is disabled"); + } + } + + return enabled; +} diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.h b/src/uct/rocm/ipc/rocm_ipc_iface.h index e9cd2425739..43d361037c0 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.h +++ b/src/uct/rocm/ipc/rocm_ipc_iface.h @@ -18,6 +18,7 @@ typedef struct uct_rocm_ipc_iface_config_params { double latency; int enable_ipc_handle_cache; unsigned sigpool_max_elems; + int signal_invalidate; } uct_rocm_ipc_iface_config_params_t; typedef struct uct_rocm_ipc_iface { @@ -32,4 +33,6 @@ typedef struct uct_rocm_ipc_iface_config { uct_rocm_ipc_iface_config_params_t params; } uct_rocm_ipc_iface_config_t; +int uct_rocm_ipc_signal_invalidate_enabled(void); + #endif From 63a1ccd947e6790338954d15587523c2ade5a277 Mon Sep 17 00:00:00 2001 From: David DeBonis Date: Wed, 28 Oct 2026 14:53:54 +0000 Subject: [PATCH 16/16] UCT/ROCM: Additional minor changes from review --- src/uct/rocm/ipc/rocm_ipc_cache.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index 0a3b40884c9..849cb0d0b92 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -278,9 +278,7 @@ static void uct_rocm_ipc_export_cleanup(void) ucs_list_head_init(&destroy_list); ucs_spin_lock(&uct_rocm_ipc_export_lock); kh_foreach_value(&uct_rocm_ipc_exports, exp, { - if (uct_rocm_ipc_signal_invalidate_enabled()) { - hsa_signal_store_screlease(exp->signal, 0); - } + hsa_signal_store_screlease(exp->signal, 0); ucs_list_add_tail(&destroy_list, &exp->list); }) kh_clear(rocm_ipc_exports, &uct_rocm_ipc_exports);