diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.c b/src/uct/rocm/ipc/rocm_ipc_cache.c index a3bbdd76fc0..849cb0d0b92 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.c +++ b/src/uct/rocm/ipc/rocm_ipc_cache.c @@ -10,15 +10,23 @@ #endif #include "rocm_ipc_cache.h" +#include "rocm_ipc_iface.h" +#include +#include #include +#include #include #include #include #include #include #include +#include #include +#include + +#include typedef struct uct_rocm_ipc_cache_hash_key { @@ -55,6 +63,236 @@ typedef struct uct_rocm_ipc_remote_cache { static uct_rocm_ipc_remote_cache_t uct_rocm_ipc_remote_cache; +/* Runs before the registration cache (priority 1000). */ +#define UCT_ROCM_IPC_MEM_EVENT_PRIO 100 + +typedef struct uct_rocm_ipc_export { + ucs_list_link_t list; + uintptr_t address; + size_t length; + hsa_signal_t signal; + hsa_amd_ipc_signal_t ipc_signal; +} uct_rocm_ipc_export_t; + + +KHASH_MAP_INIT_INT64(rocm_ipc_exports, uct_rocm_ipc_export_t*); + +static khash_t(rocm_ipc_exports) uct_rocm_ipc_exports; +static ucs_list_link_t uct_rocm_ipc_export_retired; +static ucs_spinlock_t uct_rocm_ipc_export_lock; +static ucs_init_once_t uct_rocm_ipc_export_handler_once = + UCS_INIT_ONCE_INITIALIZER; + + +static int +uct_rocm_ipc_signal_is_empty(const hsa_amd_ipc_signal_t *ipc_signal) +{ + size_t i; + + for (i = 0; i < ucs_static_array_size(ipc_signal->handle); ++i) { + if (ipc_signal->handle[i] != 0) { + return 0; + } + } + + return 1; +} + +static void uct_rocm_ipc_export_destroy(uct_rocm_ipc_export_t *exp) +{ + if (exp->signal.handle != 0) { + if (hsa_signal_destroy(exp->signal) != HSA_STATUS_SUCCESS) { + ucs_warn("failed to destroy rocm ipc export signal for %p", + (void*)exp->address); + } + } + + ucs_free(exp); +} + +/* An importer attaches the IPC signal before it maps memory. Therefore, after + * the free event has been dispatched, every mapping that can delay hipFree + * already holds its own signal reference and any thread may reap the export. */ +static void uct_rocm_ipc_export_reap(void) +{ + uct_rocm_ipc_export_t *exp, *tmp; + ucs_list_link_t destroy_list; + + ucs_list_head_init(&destroy_list); + + ucs_spin_lock(&uct_rocm_ipc_export_lock); + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { + ucs_list_del(&exp->list); + ucs_list_add_tail(&destroy_list, &exp->list); + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { + uct_rocm_ipc_export_destroy(exp); + } +} + +static void uct_rocm_ipc_export_mem_free_cb(ucm_event_type_t event_type, + ucm_event_t *event, void *arg) +{ + uct_rocm_ipc_export_t *exp; + khiter_t iter; + uintptr_t address; + int reaped = 0; + + if ((event_type != UCM_EVENT_MEM_TYPE_FREE) || + (event->mem_type.mem_type != UCS_MEMORY_TYPE_ROCM)) { + return; + } + + address = (uintptr_t)event->mem_type.address; + + /* Importers acquire a signal reference before mapping memory, so the + * exported signal can be destroyed after this callback on any thread. */ + ucs_spin_lock(&uct_rocm_ipc_export_lock); + iter = kh_get(rocm_ipc_exports, &uct_rocm_ipc_exports, address); + if (iter != kh_end(&uct_rocm_ipc_exports)) { + exp = kh_val(&uct_rocm_ipc_exports, iter); + kh_del(rocm_ipc_exports, &uct_rocm_ipc_exports, iter); + hsa_signal_store_screlease(exp->signal, 0); + ucs_list_add_tail(&uct_rocm_ipc_export_retired, &exp->list); + reaped = 1; + ucs_debug("rocm ipc signal export %p length %zu", (void*)exp->address, + exp->length); + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + if (reaped) { + uct_rocm_ipc_export_reap(); + } +} + +static void uct_rocm_ipc_export_install_handler(void) +{ + ucs_status_t status; + + status = ucm_set_event_handler(UCM_EVENT_MEM_TYPE_FREE, + UCT_ROCM_IPC_MEM_EVENT_PRIO, + uct_rocm_ipc_export_mem_free_cb, NULL); + if (status != UCS_OK) { + ucs_diag("rocm ipc free handler not installed: %s", + ucs_status_string(status)); + } +} + +void uct_rocm_ipc_export_init(void) +{ + if (!uct_rocm_ipc_signal_invalidate_enabled()) { + return; + } + + UCS_INIT_ONCE(&uct_rocm_ipc_export_handler_once) { + uct_rocm_ipc_export_install_handler(); + } +} + +ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, + hsa_amd_ipc_signal_t *ipc_signal_p) +{ + uct_rocm_ipc_export_t *exp, *existing; + hsa_status_t hsa_status; + khiter_t iter; + int ret; + + if (!uct_rocm_ipc_signal_invalidate_enabled()) { + memset(ipc_signal_p, 0, sizeof(*ipc_signal_p)); + return UCS_OK; + } + + uct_rocm_ipc_export_reap(); + + ucs_spin_lock(&uct_rocm_ipc_export_lock); + iter = kh_get(rocm_ipc_exports, &uct_rocm_ipc_exports, address); + if (iter != kh_end(&uct_rocm_ipc_exports)) { + *ipc_signal_p = kh_val(&uct_rocm_ipc_exports, iter)->ipc_signal; + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + return UCS_OK; + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + exp = ucs_malloc(sizeof(*exp), "uct_rocm_ipc_export_t"); + if (exp == NULL) { + return UCS_ERR_NO_MEMORY; + } + + hsa_status = hsa_amd_signal_create(1, 0, NULL, HSA_AMD_SIGNAL_IPC, + &exp->signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_debug("failed to create rocm ipc signal for %p length %zu: %d", + (void*)address, length, hsa_status); + ucs_free(exp); + return UCS_ERR_IO_ERROR; + } + + hsa_status = hsa_amd_ipc_signal_create(exp->signal, &exp->ipc_signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_debug("failed to export rocm ipc signal for %p length %zu: %d", + (void*)address, length, hsa_status); + hsa_signal_destroy(exp->signal); + ucs_free(exp); + return UCS_ERR_IO_ERROR; + } + + exp->address = address; + exp->length = length; + + ucs_spin_lock(&uct_rocm_ipc_export_lock); + iter = kh_put(rocm_ipc_exports, &uct_rocm_ipc_exports, address, &ret); + if (ret == UCS_KH_PUT_FAILED) { + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + uct_rocm_ipc_export_destroy(exp); + return UCS_ERR_NO_MEMORY; + } + + if (ret != UCS_KH_PUT_KEY_PRESENT) { + kh_val(&uct_rocm_ipc_exports, iter) = exp; + *ipc_signal_p = exp->ipc_signal; + } else { + existing = kh_val(&uct_rocm_ipc_exports, iter); + *ipc_signal_p = existing->ipc_signal; + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + if (ret == UCS_KH_PUT_KEY_PRESENT) { + uct_rocm_ipc_export_destroy(exp); + } + + return UCS_OK; +} + +static void uct_rocm_ipc_export_cleanup(void) +{ + uct_rocm_ipc_export_t *exp, *tmp; + ucs_list_link_t destroy_list; + + UCS_CLEANUP_ONCE(&uct_rocm_ipc_export_handler_once) { + ucm_unset_event_handler(UCM_EVENT_MEM_TYPE_FREE, + uct_rocm_ipc_export_mem_free_cb, NULL); + } + + ucs_list_head_init(&destroy_list); + ucs_spin_lock(&uct_rocm_ipc_export_lock); + kh_foreach_value(&uct_rocm_ipc_exports, exp, { + hsa_signal_store_screlease(exp->signal, 0); + ucs_list_add_tail(&destroy_list, &exp->list); + }) + kh_clear(rocm_ipc_exports, &uct_rocm_ipc_exports); + ucs_list_for_each_safe(exp, tmp, &uct_rocm_ipc_export_retired, list) { + ucs_list_del(&exp->list); + ucs_list_add_tail(&destroy_list, &exp->list); + } + ucs_spin_unlock(&uct_rocm_ipc_export_lock); + + ucs_list_for_each_safe(exp, tmp, &destroy_list, list) { + uct_rocm_ipc_export_destroy(exp); + } +} + static ucs_pgt_dir_t *uct_rocm_ipc_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) { void *ptr; @@ -84,31 +322,179 @@ uct_rocm_ipc_cache_region_collect_callback(const ucs_pgtable_t *pgtable, ucs_list_add_tail(list, ®ion->list); } +static void +uct_rocm_ipc_cache_region_unref(uct_rocm_ipc_cache_region_t *region) +{ + if (ucs_atomic_fsub32(®ion->refcount, 1) == 1) { + ucs_free(region); + } +} + +/* Caller holds the remote-cache write lock. The claim winner detaches an + * unpinned mapping immediately so the exporter's hipFree can finish. It + * destroys the IPC signal only when no handler has taken ownership of it. */ +static int +uct_rocm_ipc_cache_region_drop_locked(uct_rocm_ipc_cache_region_t *region, + ucs_list_link_t *deferred) +{ + void *mapped_addr; + ucs_status_t status; + + if (ucs_atomic_cswap32(®ion->deactivate, 0, 1) != 0) { + return 0; + } + + /* A pinned mapping is detached by the last unpin. */ + mapped_addr = NULL; + if (region->pin_count == 0) { + mapped_addr = region->mapped_addr; + region->mapped_addr = NULL; + } + + /* A registered handler owns the signal and destroys it when it runs. */ + if (!region->handler_armed) { + region->retired_signal = region->signal; + region->signal.handle = 0; + } + + if (region->in_pgtable) { + status = ucs_pgtable_remove(®ion->cache->pgtable, ®ion->super); + if (status != UCS_OK) { + ucs_fatal("%s: failed to remove address:%p from cache (%s)", + region->cache->name, (void*)region->key.address, + ucs_status_string(status)); + } + region->in_pgtable = 0; + } + + if ((mapped_addr != NULL) && + (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS)) { + ucs_warn("failed to unmap addr:%p", mapped_addr); + } + + ucs_trace("rocm ipc cache drop addr:%p mapped:%p", + (void*)region->key.address, mapped_addr); + ucs_list_add_tail(deferred, ®ion->list); + return 1; +} + +/* Caller holds the remote-cache write lock. A different or overlapping IPC + * handle proves that this mapping is stale, so wake its handler to release + * the reference it holds on the region and its signal. */ +static void +uct_rocm_ipc_cache_region_stale_drop_locked(uct_rocm_ipc_cache_region_t *region, + ucs_list_link_t *deferred) +{ + if (region->handler_armed) { + /* Wake the handler before the region is deactivated, so that it takes + * the locked path and does not race with this store. */ + hsa_signal_store_screlease(region->signal, 0); + } + + uct_rocm_ipc_cache_region_drop_locked(region, deferred); +} + +static void +uct_rocm_ipc_cache_region_signal_destroy(uct_rocm_ipc_cache_region_t *region, + hsa_signal_t signal) +{ + if (signal.handle == 0) { + return; + } + + if (hsa_signal_destroy(signal) != HSA_STATUS_SUCCESS) { + ucs_warn("failed to destroy rocm ipc cache signal for %p", + (void*)region->key.address); + } +} + +static void uct_rocm_ipc_cache_region_finalize(ucs_list_link_t *deferred) +{ + uct_rocm_ipc_cache_region_t *region, *tmp; + + ucs_list_for_each_safe(region, tmp, deferred, list) { + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_signal_destroy(region, + region->retired_signal); + region->retired_signal.handle = 0; + uct_rocm_ipc_cache_region_unref(region); + } +} + +static bool uct_rocm_ipc_cache_signal_cb(hsa_signal_value_t value, void *arg) +{ + uct_rocm_ipc_cache_region_t *region = arg; + ucs_list_link_t deferred; + hsa_signal_t signal = {.handle = 0}; + int claimed; + + if (ucs_atomic_cswap32(®ion->handler_done, 0, 1) != 0) { + return false; + } + + ucs_list_head_init(&deferred); + if (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) { + /* A stale or teardown drop already unlinked the region. */ + signal = region->signal; + region->signal.handle = 0; + claimed = 0; + } else { + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + claimed = uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + if (region->signal.handle != 0) { + signal = region->signal; + region->signal.handle = 0; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + } + + uct_rocm_ipc_cache_region_signal_destroy(region, signal); + if (claimed) { + uct_rocm_ipc_cache_region_finalize(&deferred); + } + uct_rocm_ipc_cache_region_unref(region); + return false; +} + +static void +uct_rocm_ipc_cache_purge_collect_cb(const ucs_pgtable_t *pgtable, + ucs_pgt_region_t *pgt_region, void *arg) +{ + ucs_list_link_t *list = arg; + uct_rocm_ipc_cache_region_t *region; + + /* pgtable_purge has already unlinked the region. */ + region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); + region->in_pgtable = 0; + ucs_list_add_tail(list, ®ion->list); +} + static void uct_rocm_ipc_cache_purge(uct_rocm_ipc_cache_t *cache) { uct_rocm_ipc_cache_region_t *region, *tmp; - ucs_list_link_t region_list; + ucs_list_link_t region_list, deferred; ucs_list_head_init(®ion_list); - ucs_pgtable_purge(&cache->pgtable, uct_rocm_ipc_cache_region_collect_callback, - ®ion_list); + ucs_list_head_init(&deferred); + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + ucs_pgtable_purge(&cache->pgtable, uct_rocm_ipc_cache_purge_collect_cb, + ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_warn("failed to unmap addr:%p", region->mapped_addr); - } - - ucs_free(region); + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_stale_drop_locked(region, &deferred); } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_finalize(&deferred); ucs_trace("%s: rocm ipc cache purged", cache->name); } -static void uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, - void *from, void *to) +static void +uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, void *from, + void *to, ucs_list_link_t *deferred) { ucs_list_link_t region_list; - ucs_status_t status; uct_rocm_ipc_cache_region_t *region, *tmp; ucs_list_head_init(®ion_list); @@ -117,114 +503,169 @@ static void uct_rocm_ipc_cache_invalidate_regions(uct_rocm_ipc_cache_t *cache, uct_rocm_ipc_cache_region_collect_callback, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { - status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); - if (status != UCS_OK) { - ucs_error("failed to remove address:%p from cache (%s)", - (void *)region->key.address, ucs_status_string(status)); - } - - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_warn("failed to unmap addr:%p", region->mapped_addr); - } - ucs_free(region); + ucs_list_del(®ion->list); + uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred); } - ucs_trace("%s: closed memhandles in the range [%p..%p]", - cache->name, from, to); + ucs_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, + to); } -static ucs_status_t uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, - uct_rocm_ipc_key_t *key, - void **mapped_addr) +/* Caller holds the remote-cache write lock. The mapped region is returned + * through @a region_p. A new region is also returned through + * @a watch_region_p so the caller can arm its IPC signal after releasing the + * lock. @a deferred receives regions whose signals must be destroyed. */ +static ucs_status_t +uct_rocm_ipc_cache_map_region(uct_rocm_ipc_cache_t *cache, + uct_rocm_ipc_key_t *key, + uct_rocm_ipc_cache_region_t **region_p, + uct_rocm_ipc_cache_region_t **watch_region_p, + ucs_list_link_t *deferred) { + hsa_signal_t local_signal = {.handle = 0}; ucs_status_t status; ucs_pgt_region_t *pgt_region; uct_rocm_ipc_cache_region_t *region; hsa_status_t hsa_status; + void *mapped_addr; int ret; + *watch_region_p = NULL; + /* A per-cache lock would be needed only if the caller holds no lock or * just a read lock on the global remote-cache lock, allowing concurrent * access to this page table. */ - pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, - &cache->pgtable, key->address); + pgt_region = UCS_PROFILE_CALL(ucs_pgtable_lookup, &cache->pgtable, + key->address); if (ucs_likely(pgt_region != NULL)) { region = ucs_derived_of(pgt_region, uct_rocm_ipc_cache_region_t); if (memcmp(&key->ipc, ®ion->key.ipc, sizeof(key->ipc)) == 0) { - /*cache hit */ - ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" - UCS_PGT_REGION_FMT, cache->name, (void *)key->address, - key->length, UCS_PGT_REGION_ARG(®ion->super)); + if ((region->signal.handle == 0) || + (hsa_signal_load_scacquire(region->signal) > 0)) { + ucs_trace("%s: rocm_ipc cache hit addr:%p size:%lu region:" + UCS_PGT_REGION_FMT, + cache->name, (void*)key->address, key->length, + UCS_PGT_REGION_ARG(®ion->super)); + + *region_p = region; + return UCS_OK; + } - *mapped_addr = region->mapped_addr; - return UCS_OK; + ucs_trace("%s: rocm_ipc cache remove invalidated region:" + UCS_PGT_REGION_FMT, + cache->name, UCS_PGT_REGION_ARG(®ion->super)); } else { ucs_trace("%s: rocm_ipc cache remove stale region:" UCS_PGT_REGION_FMT " new_addr:%p new_size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), - (void *)key->address, key->length); + (void*)key->address, key->length); + } - status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); - if (status != UCS_OK) { - ucs_error("%s: failed to remove address:%p from cache", - cache->name, (void *)key->address); - goto err; - } + uct_rocm_ipc_cache_region_stale_drop_locked(region, deferred); + } - if (hsa_amd_ipc_memory_detach(region->mapped_addr) != HSA_STATUS_SUCCESS) { - ucs_warn("failed to unmap addr:%p", region->mapped_addr); + if (uct_rocm_ipc_signal_invalidate_enabled() && + !uct_rocm_ipc_signal_is_empty(&key->ipc_signal)) { + /* Acquire the signal reference before mapping memory. Once the + * mapping exists, the exporter may be blocked in hipFree and may + * destroy its signal reference after dispatching the free event. */ + hsa_status = hsa_amd_ipc_signal_attach(&key->ipc_signal, + &local_signal); + if (hsa_status != HSA_STATUS_SUCCESS) { + if (hsa_status == HSA_STATUS_ERROR_INVALID_ARGUMENT) { + ucs_debug("%s: rocm ipc buffer %p already released by the owner", + cache->name, (void*)key->address); + status = UCS_ERR_INVALID_ADDR; + } else { + ucs_error("%s: failed to attach rocm ipc signal for %p: %d", + cache->name, (void*)key->address, hsa_status); + status = UCS_ERR_IO_ERROR; } + goto err; + } - ucs_free(region); + if (hsa_signal_load_scacquire(local_signal) <= 0) { + ucs_debug("%s: rocm ipc buffer %p already released by the owner", + cache->name, (void*)key->address); + status = UCS_ERR_INVALID_ADDR; + goto err_destroy_signal; } } - hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, mapped_addr); + hsa_status = hsa_amd_ipc_memory_attach(&key->ipc, key->length, 0, NULL, + &mapped_addr); if (ucs_unlikely(hsa_status != HSA_STATUS_SUCCESS)) { ucs_error("%s: failed to open ipc mem handle. addr:%p len:%lu hsa_status:%d", - cache->name, (void *)key->address, key->length, hsa_status); + cache->name, (void*)key->address, key->length, hsa_status); status = UCS_ERR_INVALID_ADDR; - goto err; + goto err_destroy_signal; } - /*create new cache entry */ - ret = ucs_posix_memalign((void **)®ion, - ucs_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), + ret = ucs_posix_memalign((void**)®ion, + ucs_max(sizeof(void*), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(uct_rocm_ipc_cache_region_t), "uct_rocm_ipc_cache_region"); if (ret != 0) { ucs_warn("failed to allocate uct_rocm_ipc_cache region"); + if (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", mapped_addr); + } status = UCS_ERR_NO_MEMORY; - goto err; + goto err_destroy_signal; } - region->super.start = ucs_align_down_pow2(key->address, UCS_PGT_ADDR_ALIGN); - region->super.end = ucs_align_up_pow2(key->address + key->length, UCS_PGT_ADDR_ALIGN); - region->key = *key; - region->mapped_addr = *mapped_addr; - - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + region->super.start = + ucs_align_down_pow2(key->address, UCS_PGT_ADDR_ALIGN); + region->super.end = + ucs_align_up_pow2(key->address + key->length, UCS_PGT_ADDR_ALIGN); + region->key = *key; + region->mapped_addr = mapped_addr; + region->cache = cache; + region->signal = local_signal; + region->retired_signal.handle = 0; + region->refcount = 1; + region->deactivate = 0; + region->handler_done = 0; + region->handler_armed = 0; + region->pin_count = 0; + region->in_pgtable = 0; + + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); if (status == UCS_ERR_ALREADY_EXISTS) { - /* overlapped region means memory freed at source. remove and try insert */ - uct_rocm_ipc_cache_invalidate_regions(cache, - (void *)region->super.start, - (void *)region->super.end); - status = UCS_PROFILE_CALL(ucs_pgtable_insert, - &cache->pgtable, ®ion->super); + /* Overlap means the source freed the memory. Drop it and retry. */ + uct_rocm_ipc_cache_invalidate_regions(cache, (void*)region->super.start, + (void*)region->super.end, + deferred); + status = UCS_PROFILE_CALL(ucs_pgtable_insert, &cache->pgtable, + ®ion->super); } if (status != UCS_OK) { - - ucs_error("%s: failed to insert region:"UCS_PGT_REGION_FMT" size:%lu :%s", + ucs_error("%s: failed to insert region:" UCS_PGT_REGION_FMT + " size:%lu :%s", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length, ucs_status_string(status)); + if (hsa_amd_ipc_memory_detach(mapped_addr) != HSA_STATUS_SUCCESS) { + ucs_fatal("failed to unmap addr:%p", mapped_addr); + } ucs_free(region); - goto err; + goto err_destroy_signal; } - ucs_trace("%s: rocm_ipc cache new region:"UCS_PGT_REGION_FMT" size:%lu", + region->in_pgtable = 1; + *region_p = region; + if (local_signal.handle != 0) { + *watch_region_p = region; + } + ucs_trace("%s: rocm_ipc cache new region:" UCS_PGT_REGION_FMT " size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), key->length); - return UCS_OK; + +err_destroy_signal: + if ((local_signal.handle != 0) && + (hsa_signal_destroy(local_signal) != HSA_STATUS_SUCCESS)) { + ucs_warn("%s: failed to destroy rocm ipc signal for %p", cache->name, + (void*)key->address); + } err: return status; } @@ -304,22 +745,143 @@ uct_rocm_ipc_remote_cache_get(uct_rocm_ipc_cache_hash_key_t key, return UCS_OK; } +static ucs_status_t +uct_rocm_ipc_cache_region_watch(uct_rocm_ipc_cache_region_t *region) +{ + hsa_signal_t signal = {.handle = 0}; + hsa_signal_value_t value; + hsa_status_t hsa_status; + ucs_list_link_t deferred; + ucs_status_t status; + + ucs_list_head_init(&deferred); + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + if ((ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) || + (region->signal.handle == 0) || + region->handler_armed) { + /* The mapping was claimed before the handler existed. */ + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + status = UCS_ERR_INVALID_ADDR; + goto out_unref; + } + region->handler_armed = 1; + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + + ucs_atomic_add32(®ion->refcount, 1); + hsa_status = hsa_amd_signal_async_handler(region->signal, + HSA_SIGNAL_CONDITION_EQ, 0, + uct_rocm_ipc_cache_signal_cb, + region); + if (hsa_status != HSA_STATUS_SUCCESS) { + ucs_error("%s: failed to arm rocm ipc signal for %p: %d", + region->cache->name, (void*)region->key.address, hsa_status); + ucs_atomic_fadd32(®ion->refcount, (uint32_t)-1); + status = UCS_ERR_IO_ERROR; + goto drop_mapping; + } + + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + if (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0) { + /* The handler owns both its region reference and the signal. */ + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + status = UCS_ERR_INVALID_ADDR; + goto out_unref; + } + + value = hsa_signal_load_scacquire(region->signal); + if (value <= 0) { + uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + status = UCS_ERR_INVALID_ADDR; + } else { + status = UCS_OK; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_finalize(&deferred); + goto out_unref; + +drop_mapping: + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + signal = region->signal; + region->signal.handle = 0; + uct_rocm_ipc_cache_region_drop_locked(region, &deferred); + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + uct_rocm_ipc_cache_region_signal_destroy(region, signal); + uct_rocm_ipc_cache_region_finalize(&deferred); +out_unref: + /* Drops the reference taken before the watcher was called. The handler + * reference, once arming succeeds, belongs to the signal callback. */ + uct_rocm_ipc_cache_region_unref(region); + return status; +} + ucs_status_t -uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr) +uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **region_p) { uct_rocm_ipc_cache_hash_key_t hash_key = {key->pid, key->pid_ns, key->dev_num}; + uct_rocm_ipc_cache_region_t *watch_region = NULL; + uct_rocm_ipc_cache_region_t *pinned = NULL; + uct_rocm_ipc_cache_region_t *region; uct_rocm_ipc_cache_t *cache; + ucs_list_link_t deferred; ucs_status_t status; + void *mapped; + ucs_list_head_init(&deferred); ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); status = uct_rocm_ipc_remote_cache_get(hash_key, &cache); if (status == UCS_OK) { - status = uct_rocm_ipc_cache_map_region(cache, key, mapped_addr); + status = uct_rocm_ipc_cache_map_region(cache, key, ®ion, + &watch_region, &deferred); + } + if (status == UCS_OK) { + mapped = region->mapped_addr; + if (watch_region != NULL) { + /* Keep the region alive until its IPC signal is armed. */ + ucs_atomic_add32(&watch_region->refcount, 1); + } + if (region_p != NULL) { + ++region->pin_count; + ucs_atomic_add32(®ion->refcount, 1); + pinned = region; + } } ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); - return status; + uct_rocm_ipc_cache_region_finalize(&deferred); + if ((status == UCS_OK) && (watch_region != NULL)) { + status = uct_rocm_ipc_cache_region_watch(watch_region); + } + if (status != UCS_OK) { + if (pinned != NULL) { + uct_rocm_ipc_cache_unpin(pinned); + } + return status; + } + + *mapped_addr = mapped; + if (region_p != NULL) { + *region_p = pinned; + } + return UCS_OK; +} + +void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region) +{ + ucs_rw_spinlock_write_lock(&uct_rocm_ipc_remote_cache.lock); + ucs_assert(region->pin_count > 0); + if ((--region->pin_count == 0) && + (ucs_atomic_cswap32(®ion->deactivate, 0, 0) != 0)) { + if (hsa_amd_ipc_memory_detach(region->mapped_addr) != + HSA_STATUS_SUCCESS) { + ucs_warn("failed to unmap addr:%p", region->mapped_addr); + } + region->mapped_addr = NULL; + } + ucs_rw_spinlock_write_unlock(&uct_rocm_ipc_remote_cache.lock); + + uct_rocm_ipc_cache_region_unref(region); } void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache) @@ -332,16 +894,28 @@ void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache) UCS_STATIC_INIT { + ucs_status_t status; + ucs_rw_spinlock_init(&uct_rocm_ipc_remote_cache.lock); kh_init_inplace(rocm_ipc_rem_cache, &uct_rocm_ipc_remote_cache.hash); + + status = ucs_spinlock_init(&uct_rocm_ipc_export_lock, 0); + if (status != UCS_OK) { + ucs_fatal("failed to initialize rocm ipc export lock"); + } + kh_init_inplace(rocm_ipc_exports, &uct_rocm_ipc_exports); + ucs_list_head_init(&uct_rocm_ipc_export_retired); } UCS_STATIC_CLEANUP { uct_rocm_ipc_cache_t *cache; + uct_rocm_ipc_export_cleanup(); kh_foreach_value(&uct_rocm_ipc_remote_cache.hash, cache, { uct_rocm_ipc_destroy_cache(cache); }) kh_destroy_inplace(rocm_ipc_rem_cache, &uct_rocm_ipc_remote_cache.hash); + kh_destroy_inplace(rocm_ipc_exports, &uct_rocm_ipc_exports); ucs_rw_spinlock_cleanup(&uct_rocm_ipc_remote_cache.lock); + ucs_spinlock_destroy(&uct_rocm_ipc_export_lock); } diff --git a/src/uct/rocm/ipc/rocm_ipc_cache.h b/src/uct/rocm/ipc/rocm_ipc_cache.h index c8348267649..eb04356f1dd 100644 --- a/src/uct/rocm/ipc/rocm_ipc_cache.h +++ b/src/uct/rocm/ipc/rocm_ipc_cache.h @@ -13,23 +13,57 @@ #include "rocm_ipc_md.h" +typedef struct uct_rocm_ipc_cache { + ucs_pgtable_t pgtable; /**< Page table to hold the regions */ + char *name; /**< Name */ +} uct_rocm_ipc_cache_t; + typedef struct { ucs_pgt_region_t super; /**< Base class - page table region */ ucs_list_link_t list; /**< List element */ uct_rocm_ipc_key_t key; /**< Remote memory key */ void *mapped_addr; /**< Local mapped address */ + uct_rocm_ipc_cache_t *cache; /**< Cache that owns the region */ + hsa_signal_t signal; /**< Attached IPC signal */ + hsa_signal_t retired_signal; /**< Signal owned by claim winner */ + uint32_t refcount; /**< Pgtable, handler, watcher, pin */ + uint32_t deactivate; /**< 1 after unmap is claimed */ + uint32_t handler_done; /**< Async handler entered once */ + int handler_armed; /**< Async handler owns signal */ + unsigned pin_count; /**< Consumers holding mapped_addr */ + int in_pgtable; /**< Region is linked in @a cache */ } uct_rocm_ipc_cache_region_t; -typedef struct uct_rocm_ipc_cache { - ucs_pgtable_t pgtable; /**< Page table to hold the regions */ - char *name; /**< Name */ -} uct_rocm_ipc_cache_t; +typedef struct { + uct_rocm_ipc_key_t super; + uct_rocm_ipc_cache_region_t *region; /**< Pinned by rkey_ptr */ + void *mapped_addr; /**< Mapping of @a region */ +} uct_rocm_ipc_unpacked_rkey_t; ucs_status_t uct_rocm_ipc_create_cache(uct_rocm_ipc_cache_t **cache, const char *name); void uct_rocm_ipc_destroy_cache(uct_rocm_ipc_cache_t *cache); +/** + * Map @a key. A non-NULL @a region_p pins the mapping until + * @ref uct_rocm_ipc_cache_unpin, even if the cache invalidates the region. + * Known limitation: a pin does not keep the owner allocation alive remotely. + */ ucs_status_t -uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr); +uct_rocm_ipc_cache_map_memhandle(uct_rocm_ipc_key_t *key, void **mapped_addr, + uct_rocm_ipc_cache_region_t **region_p); + +void uct_rocm_ipc_cache_unpin(uct_rocm_ipc_cache_region_t *region); + +void uct_rocm_ipc_export_init(void); + +/** + * Create or reuse an IPC signal for an exported buffer and remember it until + * free. Returns an empty signal when invalidation is disabled. + * The packed @a ipc_signal is the same for every peer that imports this key. + */ +ucs_status_t uct_rocm_ipc_export_add(uintptr_t address, size_t length, + hsa_amd_ipc_signal_t *ipc_signal_p); + #endif diff --git a/src/uct/rocm/ipc/rocm_ipc_ep.c b/src/uct/rocm/ipc/rocm_ipc_ep.c index 7898c028649..ef796ccadca 100644 --- a/src/uct/rocm/ipc/rocm_ipc_ep.c +++ b/src/uct/rocm/ipc/rocm_ipc_ep.c @@ -86,7 +86,7 @@ ucs_status_t uct_rocm_ipc_ep_zcopy(uct_ep_h tl_ep, } if (iface->config.enable_ipc_handle_cache) { - ret = uct_rocm_ipc_cache_map_memhandle(key, &remote_base_addr); + ret = uct_rocm_ipc_cache_map_memhandle(key, &remote_base_addr, NULL); if (ucs_unlikely(ret != UCS_OK)) { ucs_error("fail to attach ipc mem %p %d\n", (void*)key->address, ret); @@ -181,7 +181,7 @@ ucs_status_t uct_rocm_ipc_ep_put_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uct_completion_t *comp) { ucs_status_t ret; - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t *)rkey; + uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, iov, key, comp, 1); @@ -199,7 +199,7 @@ ucs_status_t uct_rocm_ipc_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, siz uct_completion_t *comp) { ucs_status_t ret; - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t *)rkey; + uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; ret = UCS_PROFILE_CALL_ALWAYS(uct_rocm_ipc_ep_zcopy, tl_ep, remote_addr, iov, key, comp, 0); diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.c b/src/uct/rocm/ipc/rocm_ipc_iface.c index d2a00092142..3d3f0920874 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.c +++ b/src/uct/rocm/ipc/rocm_ipc_iface.c @@ -15,6 +15,9 @@ #include #include +#include +#include + static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = { {"", "", NULL, ucs_offsetof(uct_rocm_ipc_iface_config_t, super), @@ -38,6 +41,13 @@ static ucs_config_field_t uct_rocm_ipc_iface_config_table[] = { ucs_offsetof(uct_rocm_ipc_iface_config_t, params.sigpool_max_elems), UCS_CONFIG_TYPE_UINT}, + {"SIGNAL_INVALIDATE", "y", + "Detach cached ROCm IPC mappings when the exported buffer is freed.\n" + "When set to n, a mapping is replaced only when the peer's IPC handle\n" + "changes. The packed rkey layout is the same in both cases.", + ucs_offsetof(uct_rocm_ipc_iface_config_t, params.signal_invalidate), + UCS_CONFIG_TYPE_BOOL}, + {NULL} }; @@ -263,3 +273,27 @@ static UCS_CLASS_DEFINE_DELETE_FUNC(uct_rocm_ipc_iface_t, uct_iface_t); UCT_TL_DEFINE(&uct_rocm_ipc_component.super, rocm_ipc, uct_rocm_base_query_devices, uct_rocm_ipc_iface_t, "ROCM_IPC_", uct_rocm_ipc_iface_config_table, uct_rocm_ipc_iface_config_t); + +int uct_rocm_ipc_signal_invalidate_enabled(void) +{ + static ucs_init_once_t once = UCS_INIT_ONCE_INITIALIZER; + static int enabled = 1; + + UCS_INIT_ONCE(&once) { + uct_rocm_ipc_iface_config_t config; + ucs_status_t status; + + status = ucs_config_parser_fill_opts(&config, &uct_rocm_ipc_tl.config, + UCS_DEFAULT_ENV_PREFIX, 0); + if (status == UCS_OK) { + enabled = config.params.signal_invalidate; + ucs_config_parser_release_opts(&config, + uct_rocm_ipc_iface_config_table); + } + if (!enabled) { + ucs_diag("rocm ipc signal invalidation is disabled"); + } + } + + return enabled; +} diff --git a/src/uct/rocm/ipc/rocm_ipc_iface.h b/src/uct/rocm/ipc/rocm_ipc_iface.h index e9cd2425739..43d361037c0 100644 --- a/src/uct/rocm/ipc/rocm_ipc_iface.h +++ b/src/uct/rocm/ipc/rocm_ipc_iface.h @@ -18,6 +18,7 @@ typedef struct uct_rocm_ipc_iface_config_params { double latency; int enable_ipc_handle_cache; unsigned sigpool_max_elems; + int signal_invalidate; } uct_rocm_ipc_iface_config_params_t; typedef struct uct_rocm_ipc_iface { @@ -32,4 +33,6 @@ typedef struct uct_rocm_ipc_iface_config { uct_rocm_ipc_iface_config_params_t params; } uct_rocm_ipc_iface_config_t; +int uct_rocm_ipc_signal_invalidate_enabled(void); + #endif diff --git a/src/uct/rocm/ipc/rocm_ipc_md.c b/src/uct/rocm/ipc/rocm_ipc_md.c index 2092a9a5520..6774bac5f32 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.c +++ b/src/uct/rocm/ipc/rocm_ipc_md.c @@ -57,28 +57,29 @@ uct_rocm_ipc_mkey_pack(uct_md_h uct_md, uct_mem_h memh, void *address, return UCS_OK; } -static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, +static ucs_status_t uct_rocm_ipc_pack_key(void *address, size_t length, uct_rocm_ipc_key_t *key) { void *base_ptr = NULL; size_t size = 0; - hsa_status_t status; + hsa_status_t hsa_status; + ucs_status_t status; hsa_agent_t agent; hsa_amd_pointer_type_t mem_type; - status = uct_rocm_base_get_ptr_info(address, length, &base_ptr, &size, - &mem_type, &agent, NULL); - if ((status != HSA_STATUS_SUCCESS) || + hsa_status = uct_rocm_base_get_ptr_info(address, length, &base_ptr, &size, + &mem_type, &agent, NULL); + if ((hsa_status != HSA_STATUS_SUCCESS) || (mem_type == HSA_EXT_POINTER_TYPE_UNKNOWN)) { ucs_error("failed to get base ptr for %p/%lx, ROCm returned %p/%lx", address, length, base_ptr, size); - return status; + return UCS_ERR_INVALID_ADDR; } - status = hsa_amd_ipc_memory_create(base_ptr, size, &key->ipc); - if (status != HSA_STATUS_SUCCESS) { + hsa_status = hsa_amd_ipc_memory_create(base_ptr, size, &key->ipc); + if (hsa_status != HSA_STATUS_SUCCESS) { ucs_error("Failed to create ipc for %p/%lx", address, length); - return status; + return UCS_ERR_INVALID_ADDR; } key->address = (uintptr_t)base_ptr; @@ -87,7 +88,16 @@ static hsa_status_t uct_rocm_ipc_pack_key(void *address, size_t length, key->pid = getpid(); key->pid_ns = ucs_sys_get_ns(UCS_SYS_NS_TYPE_PID); - return HSA_STATUS_SUCCESS; + /* One IPC signal per allocation. Every importer of this key watches it. */ + memset(&key->ipc_signal, 0, sizeof(key->ipc_signal)); + status = uct_rocm_ipc_export_add(key->address, key->length, + &key->ipc_signal); + if (status != UCS_OK) { + ucs_diag("failed to export rocm ipc invalidation signal for %p: %s", + address, ucs_status_string(status)); + } + + return UCS_OK; } static ucs_status_t @@ -95,7 +105,7 @@ uct_rocm_ipc_mem_reg(uct_md_h md, void *address, size_t length, const uct_md_mem_reg_params_t *params, uct_mem_h *memh_p) { uct_rocm_ipc_key_t *key; - hsa_status_t status; + ucs_status_t status; key = ucs_malloc(sizeof(*key), "uct_rocm_ipc_key_t"); if (NULL == key) { @@ -104,9 +114,9 @@ uct_rocm_ipc_mem_reg(uct_md_h md, void *address, size_t length, } status = uct_rocm_ipc_pack_key(address, length, key); - if (status != HSA_STATUS_SUCCESS) { + if (status != UCS_OK) { ucs_free(key); - return UCS_ERR_INVALID_ADDR; + return status; } *memh_p = key; @@ -122,6 +132,7 @@ uct_rocm_ipc_mem_dereg(uct_md_h md, UCT_MD_MEM_DEREG_CHECK_PARAMS(params, 0); + /* The IPC signal is owned by the export table and stays until free. */ mem_hndl = params->memh; ucs_free(mem_hndl); return UCS_OK; @@ -132,29 +143,40 @@ uct_rocm_ipc_md_mem_elem_pack(uct_md_h md_h, uct_mem_h memh, uct_rkey_t rkey, uct_device_mem_elem_t *mem_elem_p, void **release_handle_p) { - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); uct_rocm_ipc_device_mem_element_t *rocm_ipc_mem_element = (uct_rocm_ipc_device_mem_element_t*)mem_elem_p; + uct_rocm_ipc_cache_region_t *region; void *mapped_addr; ucs_status_t status; - status = uct_rocm_ipc_cache_map_memhandle(key, &mapped_addr); + /* GPU kernels use the mapping directly, so it must outlive the element */ + status = uct_rocm_ipc_cache_map_memhandle(&unpacked->super, &mapped_addr, + ®ion); if (status != UCS_OK) { ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); return status; } /* Calculate offset: base_address - mapped_address */ - rocm_ipc_mem_element->mapped_offset = UCS_PTR_BYTE_DIFF(key->address, - mapped_addr); + rocm_ipc_mem_element->mapped_offset = + UCS_PTR_BYTE_DIFF(unpacked->super.address, mapped_addr); ucs_trace("rocm_ipc mem_elem_pack: key_addr=%p mapped=%p offset=%ld", - (void*)key->address, mapped_addr, + (void*)unpacked->super.address, mapped_addr, rocm_ipc_mem_element->mapped_offset); + *release_handle_p = region; return UCS_OK; } +static void uct_rocm_ipc_md_mem_elem_release(uct_md_h md, void *release_handle) +{ + uct_rocm_ipc_cache_unpin(release_handle); +} + static ucs_status_t uct_rocm_ipc_md_open(uct_component_h component, const char *md_name, const uct_md_config_t *uct_md_config, uct_md_h *md_p) @@ -178,13 +200,14 @@ uct_rocm_ipc_md_open(uct_component_h component, const char *md_name, ucs_empty_function_return_unsupported, .detect_memory_type = (uct_md_detect_memory_type_func_t) ucs_empty_function_return_unsupported, - .mem_elem_release = (uct_md_mem_elem_release_func_t)ucs_empty_function + .mem_elem_release = uct_rocm_ipc_md_mem_elem_release }; static uct_md_t md = { .ops = &md_ops, .component = &uct_rocm_ipc_component.super, }; + uct_rocm_ipc_export_init(); *md_p = &md; return UCS_OK; } @@ -195,17 +218,19 @@ uct_rocm_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, uct_rkey_t *rkey_p, void **handle_p) { uct_rocm_ipc_key_t *packed = (uct_rocm_ipc_key_t *)rkey_buffer; - uct_rocm_ipc_key_t *key; + uct_rocm_ipc_unpacked_rkey_t *unpacked; - key = ucs_malloc(sizeof(uct_rocm_ipc_key_t), "uct_rocm_ipc_key_t"); - if (NULL == key) { - ucs_error("Failed to allocate memory for uct_rocm_ipc_key_t"); + unpacked = ucs_malloc(sizeof(*unpacked), "uct_rocm_ipc_unpacked_rkey_t"); + if (NULL == unpacked) { + ucs_error("Failed to allocate memory for uct_rocm_ipc_unpacked_rkey_t"); return UCS_ERR_NO_MEMORY; } - *key = *packed; - *handle_p = NULL; - *rkey_p = (uintptr_t)key; + unpacked->super = *packed; + unpacked->region = NULL; + unpacked->mapped_addr = NULL; + *handle_p = NULL; + *rkey_p = (uintptr_t)unpacked; return UCS_OK; } @@ -213,31 +238,45 @@ uct_rocm_ipc_rkey_unpack(uct_component_t *component, const void *rkey_buffer, static ucs_status_t uct_rocm_ipc_rkey_release(uct_component_t *component, uct_rkey_t rkey, void *handle) { + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); + ucs_assert(NULL == handle); - ucs_free((void *)rkey); + if (unpacked->region != NULL) { + uct_rocm_ipc_cache_unpin(unpacked->region); + } + ucs_free(unpacked); return UCS_OK; } ucs_status_t uct_rocm_ipc_rkey_ptr(uct_component_t *component, uct_rkey_t rkey, void *handle, uint64_t raddr, void **laddr_p) { - uct_rocm_ipc_key_t *key = (uct_rocm_ipc_key_t*)rkey; - void *mapped_addr; + uct_rocm_ipc_unpacked_rkey_t *unpacked = + ucs_derived_of((uct_rocm_ipc_key_t*)rkey, + uct_rocm_ipc_unpacked_rkey_t); ptrdiff_t offset; ucs_status_t status; - status = uct_rocm_ipc_cache_map_memhandle(key, &mapped_addr); - if (status != UCS_OK) { - ucs_error("Failed to map IPC handle: %s", ucs_status_string(status)); - return status; + /* The returned address is valid until the rkey is released */ + if (unpacked->region == NULL) { + status = uct_rocm_ipc_cache_map_memhandle(&unpacked->super, + &unpacked->mapped_addr, + &unpacked->region); + if (status != UCS_OK) { + ucs_error("Failed to map IPC handle: %s", + ucs_status_string(status)); + return status; + } } /* Calculate offset from base address */ - offset = UCS_PTR_BYTE_DIFF(key->address, raddr); - *laddr_p = UCS_PTR_BYTE_OFFSET(mapped_addr, offset); + offset = UCS_PTR_BYTE_DIFF(unpacked->super.address, raddr); + *laddr_p = UCS_PTR_BYTE_OFFSET(unpacked->mapped_addr, offset); ucs_trace("rocm_ipc rkey_ptr: raddr=%p mapped=%p offset=%ld laddr=%p", - (void*)raddr, mapped_addr, offset, *laddr_p); + (void*)raddr, unpacked->mapped_addr, offset, *laddr_p); return UCS_OK; } diff --git a/src/uct/rocm/ipc/rocm_ipc_md.h b/src/uct/rocm/ipc/rocm_ipc_md.h index 3f0f70b39db..0f0da1cedb4 100644 --- a/src/uct/rocm/ipc/rocm_ipc_md.h +++ b/src/uct/rocm/ipc/rocm_ipc_md.h @@ -30,6 +30,9 @@ typedef struct uct_rocm_ipc_md_config { typedef struct uct_rocm_ipc_key { hsa_amd_ipc_memory_t ipc; + /* Owner sets this signal to 0 when the allocation is released. + * Unused when invalidation is disabled or unavailable. */ + hsa_amd_ipc_signal_t ipc_signal; uintptr_t address; size_t length; int dev_num;