diff --git a/projects/rccl/src/device/op128.h b/projects/rccl/src/device/op128.h index 3f4b857c0f01..8bfcbf753604 100644 --- a/projects/rccl/src/device/op128.h +++ b/projects/rccl/src/device/op128.h @@ -12,13 +12,27 @@ #include "device/rccl_ptr.h" inline __device__ void load128(const uint64_t* ptr, uint64_t &v0, uint64_t &v1) { +#if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + union { v4u v; uint64_t u64[2]; } u; + u.v = __builtin_amdgcn_global_load_b128((v4u_gptr) ptr, RCCL_SYSTEM_SYNCSCOPE); + v0 = u.u64[0]; + v1 = u.u64[1]; +#else v0 = __builtin_nontemporal_load((u64_gptr) ptr); v1 = __builtin_nontemporal_load((u64_gptr) ptr+1); +#endif } inline __device__ void store128(uint64_t* ptr, uint64_t v0, uint64_t v1) { +#if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + union { v4u v; uint64_t u64[2]; } u; + u.u64[0] = v0; + u.u64[1] = v1; + __builtin_amdgcn_global_store_b128((v4u_gptr) ptr, u.v, RCCL_SYSTEM_SYNCSCOPE); +#else *((u64_gptr) ptr) = v0; *((u64_gptr) ptr + 1) = v1; +#endif } inline __device__ uint64_t* shmemCvtPtr(volatile uint64_t* shmemGenericPtr) { @@ -240,11 +254,33 @@ template<> __device__ __forceinline__ void st_global<0>(uintptr_t addr, BytePack // template<> __device__ __forceinline__ void st_relaxed_gpu_global<0>(uintptr_t addr, BytePack<0> value) {} // Used to define implementations for above prototypes. -#define DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ +#define DEFINE_ld_st__size_space_hip_atomic(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ + template<> \ + __device__ __forceinline__ BytePack ld_##space(addr_cxx_ty addr) { \ + data_cxx_ty tmp; \ + tmp = *((__attribute__((address_space(1))) data_cxx_ty *)addr); \ + BytePack ans; \ + ans.native = tmp; \ + return ans; \ + } \ + template<> \ + __device__ __forceinline__ BytePack ld_volatile_##space(addr_cxx_ty addr) { \ + data_cxx_ty tmp; \ + tmp = __hip_atomic_load((__attribute__((address_space(1))) data_cxx_ty *)addr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); \ + BytePack ans; \ + ans.native = tmp; \ + return ans; \ + } \ + template<> \ + __device__ __forceinline__ void st_##space(addr_cxx_ty addr, BytePack value) { \ + __hip_atomic_store((__attribute__((address_space(1))) data_cxx_ty *)addr, value.native, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); \ + } + +#define DEFINE_ld_st__size_space_fallback(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ template<> \ __device__ __forceinline__ BytePack ld_##space(addr_cxx_ty addr) { \ data_cxx_ty tmp; \ - tmp = *((data_cxx_ty *)addr); \ + tmp = *((__attribute__((address_space(1))) data_cxx_ty *)addr); \ BytePack ans; \ ans.native = tmp; \ return ans; \ @@ -252,16 +288,24 @@ template<> __device__ __forceinline__ void st_global<0>(uintptr_t addr, BytePack template<> \ __device__ __forceinline__ BytePack ld_volatile_##space(addr_cxx_ty addr) { \ data_cxx_ty tmp; \ - tmp = __builtin_nontemporal_load((data_cxx_ty *)addr); \ + tmp = __builtin_nontemporal_load((__attribute__((address_space(1))) data_cxx_ty *)addr); \ BytePack ans; \ ans.native = tmp; \ return ans; \ } \ template<> \ __device__ __forceinline__ void st_##space(addr_cxx_ty addr, BytePack value) { \ - __builtin_nontemporal_store(value.native, (data_cxx_ty *)addr); \ + __builtin_nontemporal_store(value.native, (__attribute__((address_space(1))) data_cxx_ty *)addr); \ } +#if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS +#define DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ + DEFINE_ld_st__size_space_hip_atomic(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) +#else +#define DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ + DEFINE_ld_st__size_space_fallback(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) +#endif + // #if __CUDA_ARCH__ >= 700 // #define PTX_relaxed_gpu "relaxed.gpu" // #else @@ -296,6 +340,8 @@ DEFINE_ld_st__size(4, uint32_t, b32, r) DEFINE_ld_st__size(8, uint64_t, b64, l) #undef DEFINE_ld_st__size_space +#undef DEFINE_ld_st__size_space_hip_atomic +#undef DEFINE_ld_st__size_space_fallback #undef DEFINE_ld_st__size @@ -362,12 +408,20 @@ DEFINE_ld_st_16__space(global, uintptr_t, l) __device__ __forceinline__ uint64_t ld_volatile_global(uint64_t *ptr) { uint64_t ans; + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + ans = __hip_atomic_load((__attribute__((address_space(1)))uint64_t *)ptr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + #else ans = __builtin_nontemporal_load(ptr); + #endif return ans; } __device__ __forceinline__ uint64_t ld_relaxed_sys_global(uint64_t *ptr) { uint64_t ans; + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + ans = __hip_atomic_load((__attribute__((address_space(1)))uint64_t *)ptr, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + #else ans = __builtin_nontemporal_load(ptr); + #endif return ans; } @@ -383,18 +437,34 @@ __device__ __forceinline__ uint64_t ld_relaxed_sys_global(uint64_t *ptr) { __device__ __forceinline__ uint64_t ld_acquire_sys_global(uint64_t *ptr) { uint64_t ans; + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + ans = __hip_atomic_load((__attribute__((address_space(1)))uint64_t *)ptr, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_SYSTEM); + #else ans = __atomic_load_n(ptr ,__ATOMIC_SEQ_CST); + #endif return ans; } __device__ __forceinline__ void st_volatile_global(uint64_t *ptr, uint64_t val) { + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + __hip_atomic_store((__attribute__((address_space(1)))uint64_t *)ptr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + #else __builtin_nontemporal_store(val, ptr); + #endif } __device__ __forceinline__ void st_relaxed_sys_global(uint64_t *ptr, uint64_t val) { + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + __hip_atomic_store((__attribute__((address_space(1)))uint64_t *)ptr, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM); + #else __builtin_nontemporal_store(val, ptr); + #endif } __device__ __forceinline__ void st_release_sys_global(uint64_t *ptr, uint64_t val) { + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + __hip_atomic_store((__attribute__((address_space(1)))uint64_t *)ptr, val, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM); + #else __atomic_store_n(ptr, val, __ATOMIC_SEQ_CST); + #endif } __device__ __forceinline__ void fence_acq_rel_sys() { diff --git a/projects/rccl/src/device/prims_ll.h b/projects/rccl/src/device/prims_ll.h index 07e060bc6973..76617872cae8 100644 --- a/projects/rccl/src/device/prims_ll.h +++ b/projects/rccl/src/device/prims_ll.h @@ -167,6 +167,8 @@ class Primitivesi4)); +#elif RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + i4.v4u = __builtin_amdgcn_global_load_b128((v4u_gptr)src->v, RCCL_SYSTEM_SYNCSCOPE); #else *((u64_gptr)i4.v) = __builtin_nontemporal_load((u64_gptr)src->v); *((u64_gptr)i4.v + 1) = __builtin_nontemporal_load((u64_gptr)src->v+1); @@ -210,6 +212,8 @@ class Primitivesi4)); +#elif RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + line[i].v4u = __builtin_amdgcn_global_load_b128((v4u_gptr)src->v, RCCL_SYSTEM_SYNCSCOPE); #else line[i].v[0] = __builtin_nontemporal_load(src->v); line[i].v[1] = __builtin_nontemporal_load(src->v+1); @@ -237,6 +241,8 @@ class Primitivesi4)); +#elif RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + line[i].v4u = __builtin_amdgcn_global_load_b128((v4u_gptr)src->v, RCCL_SYSTEM_SYNCSCOPE); #else line[i].v[0] = __builtin_nontemporal_load((u64_gptr)src->v); line[i].v[1] = __builtin_nontemporal_load((u64_gptr)src->v+1); @@ -268,8 +274,13 @@ class Primitives> 32); i4.flag2 = flag; + #if RCCL_HAVE_GLOBAL_DWORDX4_BUILTINS + // System scope store that bypasses the hardware caches, should generate global_store_dwordx4 instruction with sc0 and sc1 bits set to 1 on gfx942/gfx950. + __builtin_amdgcn_global_store_b128((v4u_gptr) dst->v, i4.v4u, RCCL_SYSTEM_SYNCSCOPE); + #else *((u64_gptr) dst->v) = *((u64_gptr) i4.v); - *((u64_gptr) dst->v+1) = *((u64_gptr) i4.v+1); + *((u64_gptr) dst->v+1) = *((u64_gptr) i4.v+1); + #endif #if defined(__gfx950__) && ROCM_VERSION < 70002 __builtin_amdgcn_fence(__ATOMIC_RELEASE, ""); // flush cache on gfx950 if ROCr fix for hipHostMallocUncached is not available (ROCm version < 7.0.2) #endif @@ -290,6 +301,16 @@ class Primitives