diff --git a/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.cc b/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.cc old mode 100755 new mode 100644 index bedbf1d26f5..36b01749ac3 --- a/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.cc +++ b/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.cc @@ -43,6 +43,9 @@ * */ +#include "suites/performance/memory_async_copy_numa.h" + +#if ENABLE_COPY_NUMA #include #include #include @@ -54,7 +57,6 @@ #include "suites/test_common/test_base.h" #include "hsa/hsa.h" #include "hsa/hsa_ext_amd.h" -#include "suites/performance/memory_async_copy_numa.h" #include "common/base_rocr_utils.h" #include "common/helper_funcs.h" #include "gtest/gtest.h" @@ -124,10 +126,17 @@ void MemoryAsyncCopyNUMA::Run(void) { hwloc_bitmap_free(cpu_bind_set_chk); // Bind Memory +#if ENABLE_COPY_NUMA_MEMBIND_NODESET ret = hwloc_set_membind_nodeset(topology_, cpu_hwl_numa_nodeset_, HWLOC_MEMBIND_BIND, 0); ASSERT_TRUE(ret == 0 && "hwloc: membind not supported or cannot be enforced. Check errno."); +#else + if (verbosity() >= VERBOSE_STANDARD) { + std::cout << "hwloc: membind nodeset not available in linked libhwloc; skipping." + << std::endl; + } +#endif } for (Transaction t : tran_) { RunBenchmarkWithVerification(&t); @@ -360,5 +369,5 @@ void MemoryAsyncCopyNUMA::RunBenchmarkWithVerification(Transaction *t) { t->benchmark_copy_time->push_back(GetMeanTime(&time)); } } - #undef RET_IF_HSA_ERR +#endif diff --git a/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.h b/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.h index bc598060740..a30074bcbbb 100755 --- a/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.h +++ b/projects/rocr-runtime/rocrtst/suites/performance/memory_async_copy_numa.h @@ -46,6 +46,7 @@ #ifndef ROCRTST_SUITES_PERFORMANCE_MEMORY_ASYNC_COPY_NUMA_H_ #define ROCRTST_SUITES_PERFORMANCE_MEMORY_ASYNC_COPY_NUMA_H_ +#if ENABLE_COPY_NUMA #include #include @@ -70,5 +71,6 @@ class MemoryAsyncCopyNUMA : public MemoryAsyncCopy { // @Brief: Run for Benchmark mode with verification virtual void RunBenchmarkWithVerification(Transaction *t); }; +#endif // ENABLE_COPY_NUMA #endif // ROCRTST_SUITES_PERFORMANCE_MEMORY_ASYNC_COPY_NUMA_H_ diff --git a/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt b/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt old mode 100755 new mode 100644 index 6ece62f87e2..f5b459a3260 --- a/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt +++ b/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt @@ -371,6 +371,14 @@ aux_source_directory(${ROCRTST_ROOT}/suites/negative negativeSources) aux_source_directory(${ROCRTST_ROOT}/suites/stress stressSources) aux_source_directory(${ROCRTST_ROOT}/suites/test_common testCommonSources) +# Optional: enable NUMA async copy test (memory_async_copy_numa.cc) +# Force disabled by default to avoid building the NUMA test source. +set(ENABLE_COPY_NUMA OFF CACHE BOOL "Build Memory_Async_Copy_NUMA test" FORCE) +list(FILTER performanceSources EXCLUDE REGEX ".*memory_async_copy_numa\\.cc$") + +# Optional: enable hwloc membind nodeset call in NUMA test +set(ENABLE_COPY_NUMA_MEMBIND_NODESET OFF CACHE BOOL "Enable hwloc membind nodeset in NUMA test") + # Header file include path include_directories(${ROCRTST_ROOT}) @@ -480,33 +488,88 @@ set(BITCODE_LIBS "${COMMON_BITCODE_LIBS}") set(CL_FILE_LIST "${KERNELS_DIR}/cu_mask_kernels.cl") build_sample_for_devices("cu_mask") -set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) +# Configure RPATH settings before creating executable target +set(CMAKE_BUILD_WITH_INSTALL_RPATH OFF) +set(CMAKE_SKIP_BUILD_RPATH FALSE) +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.14") + set(CMAKE_BUILD_RPATH_USE_ORIGIN TRUE) +endif() +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +# Force disable NUMA copy functionality +option(ENABLE_COPY_NUMA "Enable NUMA async copy tests" OFF) + +# Always exclude memory_async_copy_numa.cc since ENABLE_COPY_NUMA is forced OFF +message(STATUS "ENABLE_COPY_NUMA is OFF - excluding memory_async_copy_numa.cc from build") +list(FILTER performanceSources EXCLUDE REGEX ".*memory_async_copy_numa\\.cc$") -# Check for numa library and headers (same approach as libhsakmt) +# Check for numa library and header find_package(PkgConfig) -find_package(NUMA) -if(NUMA_FOUND) - set(NUMA_LIBS ${NUMA_LIBRARIES}) - message(STATUS "Found NUMA package: ${NUMA_LIBRARIES}") - if(NUMA_INCLUDE_DIRS) - include_directories(${NUMA_INCLUDE_DIRS}) - message(STATUS "NUMA include dirs: ${NUMA_INCLUDE_DIRS}") +set(HAVE_NUMA FALSE) +set(NUMA_TARGET "") + +if(PKG_CONFIG_FOUND) + pkg_check_modules(NUMA_PC numa) + if(NUMA_PC_FOUND) + # Try to find the actual library file for SHARED IMPORTED target + find_library(NUMA_LIBRARY_FILE + NAMES numa rocm_sysdeps_numa + HINTS ${NUMA_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + ) + + if(NUMA_LIBRARY_FILE) + # Create SHARED IMPORTED target for proper RPATH handling + add_library(numa_imported SHARED IMPORTED) + set_target_properties(numa_imported PROPERTIES + IMPORTED_LOCATION "${NUMA_LIBRARY_FILE}" + INTERFACE_INCLUDE_DIRECTORIES "${NUMA_PC_INCLUDE_DIRS}" + ) + message(STATUS "Found NUMA via pkg-config (SHARED): ${NUMA_LIBRARY_FILE}") + else() + # Fallback to INTERFACE target if we only have link flags + add_library(numa_imported INTERFACE IMPORTED) + set_target_properties(numa_imported PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${NUMA_PC_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${NUMA_PC_LIBRARIES}" + INTERFACE_LINK_DIRECTORIES "${NUMA_PC_LIBRARY_DIRS}" + ) + message(STATUS "Found NUMA via pkg-config (INTERFACE): ${NUMA_PC_LIBRARIES}") + endif() + + set(NUMA_TARGET numa_imported) + set(HAVE_NUMA TRUE) + message(STATUS " Include dirs: ${NUMA_PC_INCLUDE_DIRS}") + message(STATUS " Library dirs: ${NUMA_PC_LIBRARY_DIRS}") endif() - set(HAVE_NUMA TRUE) -else() - # Fallback to manual detection - find_library(NUMA_LIBRARY NAMES numa) - if(NUMA_LIBRARY) - set(NUMA_LIBS ${NUMA_LIBRARY}) - message(STATUS "Found numa library (fallback): ${NUMA_LIBRARY}") +endif() + +if(NOT HAVE_NUMA) + find_package(NUMA) + if(NUMA_FOUND) + add_library(numa_imported INTERFACE IMPORTED) + set_target_properties(numa_imported PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${NUMA_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${NUMA_LIBRARIES}" + ) + set(NUMA_TARGET numa_imported) set(HAVE_NUMA TRUE) + message(STATUS "Found NUMA package: ${NUMA_LIBRARIES}") else() - message(WARNING "NUMA library not found. Building without numa support.") - message(WARNING " Install libnuma-dev (Debian/Ubuntu) or numactl-devel (RHEL/CentOS)") - set(NUMA_LIBS "") - set(HAVE_NUMA FALSE) - # Exclude numa-dependent files from build - list(FILTER performanceSources EXCLUDE REGEX ".*memory_async_copy.*\\.cc$") + # Last resort: manual detection + find_library(NUMA_LIBRARY NAMES numa) + if(NUMA_LIBRARY) + add_library(numa_imported INTERFACE IMPORTED) + set_target_properties(numa_imported PROPERTIES + INTERFACE_LINK_LIBRARIES "${NUMA_LIBRARY}" + ) + set(NUMA_TARGET numa_imported) + set(HAVE_NUMA TRUE) + message(STATUS "Found numa library (manual): ${NUMA_LIBRARY}") + else() + message(WARNING "NUMA library not found. Building without numa support.") + message(WARNING " Install libnuma-dev (Debian/Ubuntu) or numactl-devel (RHEL/CentOS)") + endif() endif() endif() @@ -517,48 +580,89 @@ add_executable(${ROCRTST} ${performanceSources} ${functionalSources} ${negativeS # Add HAVE_NUMA compile definition so code can conditionally compile NUMA features target_compile_definitions(${ROCRTST} PRIVATE HAVE_NUMA=$) -# Find hwloc library (optional - only needed for certain performance tests) -# Try pkg-config first (for super-project builds) -if(PKG_CONFIG_FOUND) - pkg_check_modules(HWLOC QUIET hwloc) - if(HWLOC_FOUND) - set(HWLOC_LIBRARY ${HWLOC_LINK_LIBRARIES}) - message(STATUS "Found hwloc via pkg-config: ${HWLOC_LIBRARY}") - endif() +# Find hwloc library +set(HAVE_HWLOC FALSE) +set(HWLOC_TARGET "") + +# First priority: Try to find hwloc via CMake package (from TheRock sysdeps) +find_package(hwloc QUIET CONFIG) +if(hwloc_FOUND AND TARGET hwloc::hwloc) + set(HWLOC_TARGET hwloc::hwloc) + set(HAVE_HWLOC TRUE) + message(STATUS "Found hwloc via CMake package (TheRock sysdeps)") endif() -# If not found via pkg-config, check for bundled version (any version) -if(NOT HWLOC_FOUND) - # Use file(GLOB) to find any libhwloc.so* file regardless of version - file(GLOB HWLOC_LIBRARY "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/lib/libhwloc.so*") - if(HWLOC_LIBRARY) - list(GET HWLOC_LIBRARY 0 HWLOC_LIBRARY) # Take first match if multiple versions exist - set(HWLOC_FOUND TRUE) - message(STATUS "Found bundled hwloc library: ${HWLOC_LIBRARY}") - else() - # Fall back to system hwloc - find_library(HWLOC_LIBRARY NAMES hwloc) - if(HWLOC_LIBRARY) - set(HWLOC_FOUND TRUE) - message(WARNING "Using system hwloc library: ${HWLOC_LIBRARY}") - message(WARNING " Consider using bundled version to avoid compatibility issues") +# Second priority: Try pkg-config +if(NOT HAVE_HWLOC AND PKG_CONFIG_FOUND) + pkg_check_modules(HWLOC_PC hwloc) + if(HWLOC_PC_FOUND) + # Try to find the actual library file for SHARED IMPORTED target + find_library(HWLOC_LIBRARY_FILE + NAMES hwloc + HINTS ${HWLOC_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + ) + + if(HWLOC_LIBRARY_FILE) + # Create SHARED IMPORTED target for proper RPATH handling + add_library(hwloc_imported SHARED IMPORTED) + set_target_properties(hwloc_imported PROPERTIES + IMPORTED_LOCATION "${HWLOC_LIBRARY_FILE}" + INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_PC_INCLUDE_DIRS}" + ) + set(HWLOC_TARGET hwloc_imported) + set(HAVE_HWLOC TRUE) + message(STATUS "Found hwloc via pkg-config (SHARED): ${HWLOC_LIBRARY_FILE}") + message(STATUS " Include dirs: ${HWLOC_PC_INCLUDE_DIRS}") + message(STATUS " Library dirs: ${HWLOC_PC_LIBRARY_DIRS}") endif() endif() endif() -# Handle hwloc availability -if(HWLOC_FOUND) - set(HWLOC_LIBS ${HWLOC_LIBRARY}) - set(HAVE_HWLOC TRUE) - message(STATUS "hwloc support enabled") -else() - message(FATAL_ERROR "hwloc library not found. Required for rocrtst build. Install libhwloc-dev (Debian/Ubuntu) or hwloc-devel (RHEL/CentOS)") +if(NOT HAVE_HWLOC) + # Fallback: Search in thirdparty (may have symbol version incompatibilities with TheRock's numa) + find_library(HWLOC_LIBRARY + NAMES hwloc + HINTS ${ROCRTST_ROOT}/thirdparty/lib + PATH_SUFFIXES lib lib64 + NO_DEFAULT_PATH + ) + + # If not found, try versioned library directly (e.g., libhwloc.so.5) + if(NOT HWLOC_LIBRARY) + find_file(HWLOC_LIBRARY + NAMES libhwloc.so.5 libhwloc.so.15 + HINTS ${ROCRTST_ROOT}/thirdparty/lib + PATH_SUFFIXES lib lib64 + NO_DEFAULT_PATH + ) + endif() + + if(HWLOC_LIBRARY) + add_library(hwloc_imported SHARED IMPORTED) + set_target_properties(hwloc_imported PROPERTIES + IMPORTED_LOCATION "${HWLOC_LIBRARY}" + ) + set(HWLOC_TARGET hwloc_imported) + set(HAVE_HWLOC TRUE) + message(STATUS "Found hwloc library (fallback): ${HWLOC_LIBRARY}") + message(WARNING "Using pre-built hwloc from thirdparty/lib - may have symbol version incompatibilities with TheRock's numa") + else() + message(FATAL_ERROR "hwloc library not found. Install libhwloc-dev (Debian/Ubuntu) or hwloc-devel (RHEL/CentOS) or add hwloc as a TheRock dependency") + endif() endif() # Add HAVE_HWLOC compile definition target_compile_definitions(${ROCRTST} PRIVATE HAVE_HWLOC=$) -target_link_libraries(${ROCRTST} ${ROCRTST_LIBS} c stdc++ dl pthread rt ${NUMA_LIBS} ${HWLOC_LIBS}) +# Link libraries - CMake will automatically handle RPATH for imported targets +target_link_libraries(${ROCRTST} ${ROCRTST_LIBS} c stdc++ dl pthread rt) +if(HAVE_NUMA) + target_link_libraries(${ROCRTST} ${NUMA_TARGET}) +endif() +if(HAVE_HWLOC) + target_link_libraries(${ROCRTST} ${HWLOC_TARGET}) +endif() #Build kernels add_custom_target(rocrtst_kernels ALL DEPENDS ${HSACO_TARG_LIST}) @@ -566,9 +670,13 @@ add_custom_target(rocrtst_kernels ALL DEPENDS ${HSACO_TARG_LIST}) #Build symlinks add_custom_target(rocrtst_links ALL DEPENDS ${ROCRTST_LINKS_LIST} ) -## Set RUNPATH to pickup local copy of hwloc -set_property(TARGET ${ROCRTST} PROPERTY INSTALL_RPATH "$ORIGIN;$ORIGIN/thirdparty/lib;$ORIGIN/../lib/rocrtst/thirdparty/lib" ) -set_property(TARGET ${ROCRTST} PROPERTY LINK_FLAGS "-Wl,--enable-new-dtags") +# CMake automatically sets RPATH for linked libraries when CMAKE_INSTALL_RPATH_USE_LINK_PATH is TRUE +# This works for SHARED IMPORTED targets (numa, hwloc) found via pkg-config +set_target_properties(${ROCRTST} PROPERTIES + LINK_FLAGS "-Wl,--enable-new-dtags" + INSTALL_RPATH "$ORIGIN/../lib/rocrtst/lib;$ORIGIN/../lib/rocm_sysdeps/lib" + BUILD_RPATH "$ORIGIN/../../dist/lib/rocm_sysdeps/lib;$ORIGIN/../lib/rocrtst/lib" +) install(TARGETS ${ROCRTST} ARCHIVE DESTINATION lib @@ -577,4 +685,4 @@ install(TARGETS ${ROCRTST} install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/lib DESTINATION lib/rocrtst ) -include ( CPack ) +include ( CPack ) \ No newline at end of file diff --git a/projects/rocr-runtime/rocrtst/suites/test_common/main.cc b/projects/rocr-runtime/rocrtst/suites/test_common/main.cc index d29d2c7977a..09ebbee81fb 100644 --- a/projects/rocr-runtime/rocrtst/suites/test_common/main.cc +++ b/projects/rocr-runtime/rocrtst/suites/test_common/main.cc @@ -61,7 +61,9 @@ #include "suites/functional/svm_memory.h" #include "suites/performance/dispatch_time.h" #include "suites/performance/memory_async_copy.h" +#if ENABLE_COPY_NUMA #include "suites/performance/memory_async_copy_numa.h" +#endif #include "suites/performance/memory_async_copy_on_engine.h" #include "suites/performance/enqueueLatency.h" #include "suites/negative/memory_allocate_negative_tests.h" @@ -778,10 +780,12 @@ TEST(rocrtstPerf, ENQUEUE_LATENCY) { RunGenericTest(&multiPacketequeue); } +#if ENABLE_COPY_NUMA TEST(rocrtstPerf, DISABLED_Memory_Async_Copy_NUMA) { MemoryAsyncCopyNUMA numa; RunGenericTest(&numa); } +#endif TEST(rocrtstPerf, AQL_Dispatch_Time_Single_SpinWait) { DispatchTime dt(true, true);