[#6102][fix] support non-system python installation

Signed-off-by: Yuan Tong <[email protected]>

Author: tongyuantongyu

cpp/tensorrt_llm/CMakeLists.txt

@@ -294,7 +294,10 @@ if(TARGET ${NIXL_WRAPPER_TARGET})
 endif()
 
 if(NOT WIN32)
-  set_target_properties(${SHARED_TARGET} PROPERTIES BUILD_RPATH "$ORIGIN")
+  # Load libraries at $PREFIX/lib from
+  # $PREFIX/lib/python3.12/site-packages/tensorrt_llm/libs
+  set_target_properties(${SHARED_TARGET}
+                        PROPERTIES BUILD_RPATH "$ORIGIN;$ORIGIN/../../../..")
 endif()
 
 if(BUILD_PYT)

cpp/tensorrt_llm/nanobind/CMakeLists.txt

@@ -41,8 +41,6 @@ endif()
 target_link_libraries(
   ${TRTLLM_NB_MODULE}
   PUBLIC ${SHARED_TARGET}
-         ${UNDEFINED_FLAG}
-         ${NO_AS_NEEDED_FLAG}
          ${Python3_LIBRARIES}
          ${TORCH_LIBRARIES}
          torch_python
@@ -54,10 +52,14 @@ target_compile_definitions(
                              PYBIND11_DETAILED_ERROR_MESSAGES=1)
 
 if(NOT WIN32)
-  set_target_properties(
-    ${TRTLLM_NB_MODULE}
-    PROPERTIES
-      LINK_FLAGS
-      "-Wl,-rpath,'$ORIGIN/libs' -Wl,-rpath,'$ORIGIN/../nvidia/nccl/lib' ${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}"
+  set(TRTLLM_NB_MODULE_RPATH_LIST
+      "$ORIGIN/libs" # TRTLLM libraries
+      "$ORIGIN/../../.." # Shared libraries under $PREFIX/lib
+      "$ORIGIN/../nvidia/nccl/lib" # NCCL libraries
   )
+  set_target_properties(${TRTLLM_NB_MODULE}
+                        PROPERTIES BUILD_RPATH "${TRTLLM_NB_MODULE_RPATH_LIST}")
+  set_target_properties(
+    ${TRTLLM_NB_MODULE} PROPERTIES LINK_FLAGS
+                                   "${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}")
 endif()

cpp/tensorrt_llm/pybind/CMakeLists.txt

@@ -43,8 +43,6 @@ endif()
 target_link_libraries(
   ${TRTLLM_PYBIND_MODULE}
   PUBLIC ${SHARED_TARGET}
-         ${UNDEFINED_FLAG}
-         ${NO_AS_NEEDED_FLAG}
          ${Python3_LIBRARIES}
          ${TORCH_LIBRARIES}
          torch_python
@@ -56,10 +54,15 @@ target_compile_definitions(
                                  PYBIND11_DETAILED_ERROR_MESSAGES=1)
 
 if(NOT WIN32)
-  set_target_properties(
-    ${TRTLLM_PYBIND_MODULE}
-    PROPERTIES
-      LINK_FLAGS
-      "-Wl,-rpath,'$ORIGIN/libs' -Wl,-rpath,'$ORIGIN/../nvidia/nccl/lib' ${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}"
+  set(TRTLLM_PYBIND_MODULE_RPATH_LIST
+      "$ORIGIN/libs" # TRTLLM libraries
+      "$ORIGIN/../../.." # Shared libraries under $PREFIX/lib
+      "$ORIGIN/../nvidia/nccl/lib" # NCCL libraries
   )
+  set_target_properties(
+    ${TRTLLM_PYBIND_MODULE} PROPERTIES BUILD_RPATH
+                                       "${TRTLLM_PYBIND_MODULE_RPATH_LIST}")
+  set_target_properties(
+    ${TRTLLM_PYBIND_MODULE} PROPERTIES LINK_FLAGS
+                                       "${AS_NEEDED_FLAG} ${UNDEFINED_FLAG}")
 endif()