diff --git a/projects/rccl/tools/topo_expl/.gitignore b/projects/rccl/tools/topo_expl/.gitignore index af79f3e4472a..3d5e122cd3d2 100644 --- a/projects/rccl/tools/topo_expl/.gitignore +++ b/projects/rccl/tools/topo_expl/.gitignore @@ -1,4 +1,5 @@ hipify_rccl topo_expl include/nccl.h +include/rccl/rccl.h third_party/ diff --git a/projects/rccl/tools/topo_expl/Makefile b/projects/rccl/tools/topo_expl/Makefile index e48f7f7e4171..43b3bea2a582 100644 --- a/projects/rccl/tools/topo_expl/Makefile +++ b/projects/rccl/tools/topo_expl/Makefile @@ -1,6 +1,29 @@ # Copyright (c) 2019-2026 Advanced Micro Devices, Inc. All rights reserved. ROCM_PATH ?= /opt/rocm HIPCC = $(ROCM_PATH)/bin/hipcc + +# Probe compiler. Not hipcc: its HIP driver mode pulls cuda_wrappers/bits/c++config.h, +# which clashes with recent libstdc++ and makes a valid probe fail. +# Locate it the same way toolchain-linux.cmake does (bin/ then llvm/bin/); a +# missing probe compiler must fail loudly, not silently degrade a probe to "no" +# and recompile the colliding UALoE compat types this build set out to avoid. +# Use ifeq rather than ?= so an explicitly empty override (PROBE_CXX= make) +# still triggers detection instead of being treated as already set. +ifeq ($(strip $(PROBE_CXX)),) +PROBE_CXX := $(shell \ + for c in $(ROCM_PATH)/bin/amdclang++ $(ROCM_PATH)/llvm/bin/amdclang++ $(ROCM_PATH)/llvm/bin/clang++; do \ + if [ -x "$$c" ]; then echo "$$c"; break; fi; \ + done) +endif +ifeq ($(strip $(PROBE_CXX)),) +$(error Cannot find amdclang++/clang++ under $(ROCM_PATH)/bin or $(ROCM_PATH)/llvm/bin. Set PROBE_CXX explicitly.) +endif +# Resolve PROBE_CXX (absolute path or bare name on PATH) and confirm the target +# is executable. Testing -x on the resolved path catches the exists-but-not- +# runnable case that a bare command -v would still report as found. +ifeq ($(shell p=$$(command -v $(PROBE_CXX) 2>/dev/null); [ -n "$$p" ] && [ -x "$$p" ] && echo ok),) +$(error PROBE_CXX ($(PROBE_CXX)) is not executable. Set PROBE_CXX to a valid C++ compiler.) +endif HIPIFY_PL_EXE ?= $(ROCM_PATH)/bin/hipify-perl HIPIFY_PL_FLAGS = -inplace -quiet-warnings @@ -29,11 +52,19 @@ NCCL_OS_DEFINE := -DNCCL_OS_WINDOWS endif # topo_expl bypasses RCCL's CMake configuration, so reproduce the two -# compile-time feature probes needed by headers it imports. +# compile-time feature probes needed by headers it imports. Probes compile a +# real temp file (like CMake's check_cxx_source_compiles) with the plain +# compiler PROBE_CXX rather than piping through hipcc's HIP driver. +PROBE_TMP := $(shell mktemp --suffix=.cpp) +ifeq ($(strip $(PROBE_TMP)),) +$(error Failed to create a temp file for the feature probes. Check that TMPDIR is writable.) +endif + HAVE_HIP_FABRIC_API := $(shell printf '%s\n' \ '#include ' \ - 'int main() { hipMemFabricHandle_t h{}; (void)h; auto fn = &hipMemImportFromShareableHandle; (void)fn; }' | \ - $(HIPCC) -x c++ -std=c++17 -fsyntax-only -D__HIP_PLATFORM_AMD__ -I$(ROCM_PATH)/include - \ + 'int main() { hipMemFabricHandle_t h{}; (void)h; auto fn = &hipMemImportFromShareableHandle; (void)fn; }' \ + > $(PROBE_TMP) && \ + $(PROBE_CXX) -x c++ -std=c++17 -fsyntax-only -D__HIP_PLATFORM_AMD__ -I$(ROCM_PATH)/include $(PROBE_TMP) \ >/dev/null 2>&1 && echo yes || echo no) ifeq ($(HAVE_HIP_FABRIC_API),yes) FEATURE_DEFINES += -DHIP_FABRIC_API @@ -41,8 +72,9 @@ endif HAVE_AMDSMI_FABRIC_API := $(shell printf '%s\n' \ '#include ' \ - 'int main() { amdsmi_fabric_info_t info{}; (void)info; }' | \ - $(HIPCC) -x c++ -std=c++17 -fsyntax-only -I$(ROCM_PATH)/include - \ + 'int main() { amdsmi_fabric_info_t info{}; (void)info; }' \ + > $(PROBE_TMP) && \ + $(PROBE_CXX) -x c++ -std=c++17 -fsyntax-only -I$(ROCM_PATH)/include $(PROBE_TMP) \ >/dev/null 2>&1 && echo yes || echo no) ifeq ($(HAVE_AMDSMI_FABRIC_API),yes) FEATURE_DEFINES += -DAMDSMI_FABRIC_DIRECT @@ -53,13 +85,16 @@ endif # compat typedefs that collide with the system header (extra topo_expl compile). HAVE_AMDSMI_FABRIC_TELEMETRY := $(shell printf '%s\n' \ '#include ' \ - 'int main() { amdsmi_fabric_telemetry_category_t cat = AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN; (void)cat; }' | \ - $(HIPCC) -x c++ -std=c++17 -fsyntax-only -I$(ROCM_PATH)/include - \ + 'int main() { amdsmi_fabric_telemetry_category_t cat = AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN; (void)cat; }' \ + > $(PROBE_TMP) && \ + $(PROBE_CXX) -x c++ -std=c++17 -fsyntax-only -I$(ROCM_PATH)/include $(PROBE_TMP) \ >/dev/null 2>&1 && echo yes || echo no) ifeq ($(HAVE_AMDSMI_FABRIC_TELEMETRY),yes) FEATURE_DEFINES += -DAMDSMI_HEADER_HAS_FABRIC_TELEMETRY -DAMDSMI_FABRIC_DIRECT endif +PROBE_CLEANUP := $(shell rm -f $(PROBE_TMP)) + CXXFLAGS = -g -ffunction-sections -fdata-sections -DROCM_VERSION=$(ROCM_VERSION) \ -std=c++17 \ -Wl,--gc-sections -fgpu-rdc \ @@ -67,6 +102,7 @@ CXXFLAGS = -g -ffunction-sections -fdata-sections -DROCM_VERSION=$(ROCM_VERSION) -Ihipify_rccl/include/nccl_device \ -Ihipify_rccl/src/device/include -Ihipify_rccl/graph -I$(ROCM_PATH)/include/ \ -DTOPO_EXPL -DENABLE_TRACE -DENABLE_LL128 -DNVTX_NO_IMPL -DRCCL_EXPOSE_STATIC \ + -DUSE_AMDSMI \ $(NCCL_OS_DEFINE) $(FEATURE_DEFINES) \ -lpthread diff --git a/projects/rccl/tools/topo_expl/src/topo_expl_impl.cpp b/projects/rccl/tools/topo_expl/src/topo_expl_impl.cpp index d823ae088025..6cbbaf162130 100644 --- a/projects/rccl/tools/topo_expl/src/topo_expl_impl.cpp +++ b/projects/rccl/tools/topo_expl/src/topo_expl_impl.cpp @@ -30,7 +30,7 @@ #include "coll_net.h" #include "model.h" #include "topo_expl_impl.h" -#include "rocm_smi/rocm_smi.h" +#include "amdsmi_wrap.h" const char* ncclFuncStr[NCCL_NUM_FUNCTIONS+4] = { "AllGather", "AllReduce", "AlltoAllPivot", "AlltoAllGda", "AlltoAllvGda", "Broadcast", "Reduce", "ReduceScatter", "SendRecv" }; const char* ncclAlgoStr[NCCL_NUM_ALGORITHMS] = { "Tree", "Ring", "CollNetDirect", "CollNetChain", "NVLS", "NVLSTree", "PAT" }; @@ -1327,23 +1327,34 @@ ncclResult_t initTransportsRank_3(struct ncclComm* comm, struct allGatherInfo *a goto exit; } -ncclResult_t rocm_smi_init() { +ncclResult_t amd_smi_init() { return ncclSuccess; } -ncclResult_t rocm_smi_getNumDevice(uint32_t* num_devs) { +ncclResult_t amd_smi_getNumDevice(uint32_t* num_devs) { + if (num_devs) *num_devs = 0; return ncclSuccess; } -ncclResult_t rocm_smi_getDevicePciBusIdString(uint32_t deviceIndex, char* busId, size_t len) { +ncclResult_t amd_smi_getDevicePciBusIdString(uint32_t deviceIndex, char* busId, size_t len) { return ncclSuccess; } -ncclResult_t rocm_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* deviceIndex) { +ncclResult_t amd_smi_getDeviceIndexByPciBusId(const char* pciBusId, uint32_t* deviceIndex) { return ncclSuccess; } -ncclResult_t rocm_smi_getLinkInfo(int srcIndex, int dstIndex, RSMI_IO_LINK_TYPE* rsmi_type, int *hops, int *count) { +ncclResult_t amd_smi_getLinkInfo(int srcDev, int dstDev, amdsmi_link_type_t* type, int* hops, int* count) { + return ncclSuccess; +} + +ncclResult_t amd_smi_getFabricDeviceInfo(uint32_t deviceIndex, struct amdsmiFabricDeviceInfo* info) { + if (info) *info = {}; + return ncclSuccess; +} + +ncclResult_t amd_smi_getFirmwareVersion(uint32_t deviceIndex, uint64_t* fwVersion) { + if (fwVersion) *fwVersion = 0; return ncclSuccess; }