From dd07e4f6708d7efe4035e338fce7ec19a5b92ae8 Mon Sep 17 00:00:00 2001 From: haosdent Date: Fri, 22 May 2026 00:16:17 +0800 Subject: [PATCH] [Bugfix][ROCm] Unify per-extension hipify into a single shared target The four `hipify${NAME}` cmake custom targets (`hipify_C`, `hipify_moe_C`, `hipify_rocm_C`, `hipify_C_stable_libtorch`) all ran in parallel under `ninja -j` and wrote into the same `${CMAKE_CURRENT_BINARY_DIR}/csrc` output directory. When two processes overlapped on `shutil.copytree` + `matched_files_iter` inside `torch.utils.hipify.hipify_python`, one could read a `.cu` file mid-copy and report `[skipped, no changes]`, leaving the `.hip` byproduct missing. The build then failed with: clang++: error: no such file or directory: '.../csrc/attention/paged_attention_v1.hip' Accumulate per-extension sources into global properties and create a single `hipify_all` target that runs `cmake/hipify.py` exactly once on their union. Every HIP extension now depends on `hipify_all` instead of its own private target. A `FATAL_ERROR` guard catches future regressions that add a HIP extension after the finalizer call. Signed-off-by: haosdent --- CMakeLists.txt | 6 ++++++ cmake/utils.cmake | 53 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d5039470d41b..be2870f30a6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1289,6 +1289,12 @@ if(VLLM_GPU_LANG STREQUAL "HIP") WITH_SOABI) endif() +# Must run after the last HIP `define_extension_target` so every extension +# has registered its sources. +if (VLLM_GPU_LANG STREQUAL "HIP") + vllm_finalize_hipify_target() +endif() + # For CUDA and HIP builds also build the triton_kernels external package. if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") include(cmake/external_projects/triton_kernels.cmake) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index e95333457b57..f81882ccbc25 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -47,12 +47,17 @@ macro (append_cmake_prefix_path PKG EXPR) list(APPEND CMAKE_PREFIX_PATH ${_PREFIX_PATH}) endmacro() -# -# Add a target named `hipify${NAME}` that runs the hipify preprocessor on a set -# of CUDA source files. The names of the corresponding "hipified" sources are -# stored in `OUT_SRCS`. -# +# Resolve hipified output paths for `NAME` into `OUT_SRCS` and register the +# `.cu` sources with the shared `hipify_all` target. Per-extension hipify +# targets are unsafe to run in parallel against a shared csrc/ output dir, so +# accumulation here is paired with a single finalize step. function (hipify_sources_target OUT_SRCS NAME ORIG_SRCS) + if (TARGET hipify_all) + message(FATAL_ERROR + "hipify_sources_target(${NAME}) called after vllm_finalize_hipify_target. " + "Add the new HIP extension before the finalizer call in CMakeLists.txt.") + endif() + # # Split into C++ and non-C++ (i.e. CUDA) sources. # @@ -73,19 +78,41 @@ function (hipify_sources_target OUT_SRCS NAME ORIG_SRCS) list(APPEND HIP_SRCS "${CMAKE_CURRENT_BINARY_DIR}/${SRC}") endforeach() - set(CSRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/csrc) - add_custom_target( - hipify${NAME} - COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/hipify.py -p ${CMAKE_SOURCE_DIR}/csrc -o ${CSRC_BUILD_DIR} ${SRCS} - DEPENDS ${CMAKE_SOURCE_DIR}/cmake/hipify.py ${SRCS} - BYPRODUCTS ${HIP_SRCS} - COMMENT "Running hipify on ${NAME} extension source files.") + set_property(GLOBAL APPEND PROPERTY VLLM_HIPIFY_ALL_SRCS ${SRCS}) + set_property(GLOBAL APPEND PROPERTY VLLM_HIPIFY_ALL_BYPRODUCTS ${HIP_SRCS}) # Swap out original extension sources with hipified sources. list(APPEND HIP_SRCS ${CXX_SRCS}) set(${OUT_SRCS} ${HIP_SRCS} PARENT_SCOPE) endfunction() +# Define the single shared `hipify_all` custom target that runs hipify once +# on the union of every HIP extension's sources. Call after the last HIP +# `define_extension_target`. +function (vllm_finalize_hipify_target) + if (TARGET hipify_all) + return() + endif() + + get_property(ALL_SRCS GLOBAL PROPERTY VLLM_HIPIFY_ALL_SRCS) + get_property(ALL_BYPRODUCTS GLOBAL PROPERTY VLLM_HIPIFY_ALL_BYPRODUCTS) + + if (NOT ALL_SRCS) + return() + endif() + + list(REMOVE_DUPLICATES ALL_SRCS) + list(REMOVE_DUPLICATES ALL_BYPRODUCTS) + + set(CSRC_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/csrc) + add_custom_target( + hipify_all + COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/hipify.py -p ${CMAKE_SOURCE_DIR}/csrc -o ${CSRC_BUILD_DIR} ${ALL_SRCS} + DEPENDS ${CMAKE_SOURCE_DIR}/cmake/hipify.py ${ALL_SRCS} + BYPRODUCTS ${ALL_BYPRODUCTS} + COMMENT "Running hipify on all extension source files.") +endfunction() + # # Get additional GPU compiler flags from torch. # @@ -551,7 +578,7 @@ function (define_extension_target MOD_NAME) if (ARG_LANGUAGE STREQUAL "HIP") # Make this target dependent on the hipify preprocessor step. - add_dependencies(${MOD_NAME} hipify${MOD_NAME}) + add_dependencies(${MOD_NAME} hipify_all) # Make sure we include the hipified versions of the headers, and avoid conflicts with the ones in the original source folder target_include_directories(${MOD_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/csrc ${ARG_INCLUDE_DIRECTORIES})