[TRTLLM-14628][feat] Add --out-of-tree wheel builds that never write into the checkout

cpp/CMakeLists.txt

@@ -158,9 +158,29 @@ else()
   message(FATAL_ERROR "Failed to determine TensorRT LLM version")
 endif()
 
+# Where the configured executor/version.h is written. Out-of-tree builds point
+# this at the build tree so the source tree is never written (it may be mounted
+# read-only); the default keeps the in-source location.
+set(TRTLLM_VERSION_H_INCLUDE_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    CACHE PATH "Include root for the configured executor/version.h")
 configure_file(
   cmake/templates/version.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/include/tensorrt_llm/executor/version.h)
+  "${TRTLLM_VERSION_H_INCLUDE_DIR}/tensorrt_llm/executor/version.h")
+if(NOT TRTLLM_VERSION_H_INCLUDE_DIR STREQUAL
+   "${CMAKE_CURRENT_SOURCE_DIR}/include")
+  # BEFORE: the configured header must win over a stale in-source copy.
+  include_directories(BEFORE "${TRTLLM_VERSION_H_INCLUDE_DIR}")
+endif()
+
+# Generated FMHA sources (see contextFusedMultiHeadAttention/CMakeLists.txt for
+# the full definition and default). fmha_cubin.h is included transitively by
+# headers used across several targets, so when generation is redirected out of
+# the source tree the directory must be on every target's include path, not just
+# the FMHA target's.
+if(DEFINED TRTLLM_FMHA_GEN_DIR AND NOT "${TRTLLM_FMHA_GEN_DIR}" STREQUAL "")
+  include_directories("${TRTLLM_FMHA_GEN_DIR}")
+endif()
 
 setup_cuda_compiler()
 

cpp/kernels/fmha_v2/setup.py

@@ -2394,7 +2394,10 @@ def get_kernel_code(kspec, kname, lname):
     params_str = 'reinterpret_cast<bert::Fused_multihead_attention_params_v2 &>(params)' if generate_cu_trtllm else 'params'
     attn_mask_type_str = 'using Attention_mask_type = ContextAttentionMaskType;' if generate_cu_trtllm else 'using Attention_mask_type = fmha::Attention_mask_type;'
     bert_launch_params = '' if generate_cu_trtllm else 'using Launch_params = bert::Fused_multihead_attention_launch_params;'
-    include_str = '#include "../fused_multihead_attention_common.h"\n' if generate_cu_trtllm else ''
+    # No "../" prefix: the generated .cu may live outside the source tree
+    # (TRTLLM_FMHA_GEN_DIR); the consuming CMake target puts the
+    # contextFusedMultiHeadAttention directory on the include path.
+    include_str = '#include "fused_multihead_attention_common.h"\n' if generate_cu_trtllm else ''
     include_str += '#include "tensorrt_llm/common/config.h"' if generate_cu_trtllm else ''
     num_compute_groups_str = '' if generate_cu_trtllm else 'static constexpr int NUM_COMPUTE_GROUPS = 2;'
     fused_multihead_attention_params_v2_str = 'Fused_multihead_attention_params_v2' if generate_cu_trtllm else f'{params_type}'

cpp/tensorrt_llm/kernels/contextFusedMultiHeadAttention/CMakeLists.txt

@@ -15,9 +15,31 @@
 # the License.
 #
 
+# Directory holding the generated FMHA sources (fmha_v2_cu/*.cu and
+# cubin/fmha_cubin.{h,cpp}). scripts/build_wheel.py points this at an
+# out-of-tree location when --out-of-tree is set; by default the generated files
+# live in this directory as before.
+set(TRTLLM_FMHA_GEN_DIR
+    "${CMAKE_CURRENT_SOURCE_DIR}"
+    CACHE PATH "Directory containing generated FMHA sources")
+
 file(GLOB_RECURSE SRC_CPP *.cpp)
 file(GLOB_RECURSE SRC_CU *.cu)
 list(FILTER SRC_CU EXCLUDE REGEX "fmha_v2_cu/.*")
+# fmha_cubin.cpp is generated; always take it from TRTLLM_FMHA_GEN_DIR so an
+# out-of-tree copy is used when configured and a stale in-tree copy is never
+# compiled alongside it.
+list(FILTER SRC_CPP EXCLUDE REGEX "cubin/fmha_cubin\\.cpp$")
+if(EXISTS "${TRTLLM_FMHA_GEN_DIR}/cubin/fmha_cubin.cpp")
+  list(APPEND SRC_CPP "${TRTLLM_FMHA_GEN_DIR}/cubin/fmha_cubin.cpp")
+else()
+  message(
+    FATAL_ERROR
+      "Generated FMHA source ${TRTLLM_FMHA_GEN_DIR}/cubin/fmha_cubin.cpp is "
+      "missing. It is produced by the FMHA generation step "
+      "(scripts/build_wheel.py runs it before CMake); a missing file means "
+      "generation did not complete. Regenerate before configuring.")
+endif()
 # The skip_softmax sm_120/sm_121 warp-specialized FMHA is a hand-written TU that
 # only compiles for the sm_120 family; it is added explicitly to the sm_120 arch
 # target below, not to the all-arch source list.
@@ -31,6 +53,9 @@ filter_source_cuda_architectures(
   IMPLICIT_FAMILY)
 
 target_sources(context_attention_src PRIVATE ${SRC_CPP} ${SRC_CU})
+# For #include "cubin/fmha_cubin.h" when the generated header is out of tree.
+target_include_directories(context_attention_src
+                           PRIVATE "${TRTLLM_FMHA_GEN_DIR}")
 target_compile_definitions(context_attention_src PRIVATE USE_DEMO_BERT_PARAMS=1
                                                          GENERATE_CUBIN=1)
 set_target_properties(
@@ -68,7 +93,7 @@ foreach(arch IN ITEMS 80 86 89 90 100 120)
     continue()
   endif()
 
-  file(GLOB arch_files "fmha_v2_cu/*_sm${arch}.cu")
+  file(GLOB arch_files "${TRTLLM_FMHA_GEN_DIR}/fmha_v2_cu/*_sm${arch}.cu")
   # Compile the hand-written skip_softmax warp-specialized FMHA into the sm_120
   # family target. It uses sm_120-only TMA + sync-MMA, so it must never be
   # compiled for other architectures.
@@ -94,12 +119,28 @@ foreach(arch IN ITEMS 80 86 89 90 100 120)
     set_target_properties(
       ${TARGET_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON
                                 CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+    # CMAKE_CURRENT_SOURCE_DIR resolves the generated kernels' #include
+    # "fused_multihead_attention_common.h" when they live out of tree (and is
+    # where includer-relative lookup lands in-tree).
     target_include_directories(
       ${TARGET_NAME}
-      PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../../kernels/fmha_v2/src/
+      PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+              ${CMAKE_CURRENT_SOURCE_DIR}/../../../kernels/fmha_v2/src/
               ${CMAKE_CURRENT_SOURCE_DIR}/../../../kernels/fmha_v2/generated/)
     set_cuda_architectures(${TARGET_NAME} ${TARGET_ARCH})
     target_sources(context_attention_src
                    PUBLIC $<TARGET_OBJECTS:${TARGET_NAME}>)
+  else()
+    # arch is enabled (disabled archs already `continue`d above) but the
+    # generation step produced no *_sm${arch}.cu kernels for it. Fail rather
+    # than silently building without this architecture's FMHA kernels, which
+    # would surface only as a missing-kernel failure at runtime. (arch 120
+    # always has the hand-written skip_softmax TU appended above, so this branch
+    # covers the generated-only architectures.)
+    message(
+      FATAL_ERROR
+        "No generated FMHA kernels (*_sm${arch}.cu) found in "
+        "${TRTLLM_FMHA_GEN_DIR}/fmha_v2_cu for enabled architecture ${arch}. "
+        "The FMHA generation step did not produce sources for this arch.")
   endif()
 endforeach()

docs/source/installation/build-from-source.md

@@ -109,7 +109,7 @@ With `--build_root <dir>` set, the following default under `<dir>` instead of th
 
 Conan's `cpp/CMakeUserPresets.json` convenience file is also skipped in this mode, since it would reference the (possibly ephemeral) out-of-tree build directory.
 
-Only final artifacts are still written into the checkout: `tensorrt_llm/libs`, `tensorrt_llm/include`, Python bindings and stubs, generated FMHA kernel sources, and the `.whl` output directory (`--dist_dir`).
+Only final artifacts are still written into the checkout: `tensorrt_llm/libs`, `tensorrt_llm/include`, Python bindings and stubs, generated FMHA kernel sources, the configured `cpp/include/tensorrt_llm/executor/version.h`, and the `.whl` output directory (`--dist_dir`).
 
 Related knobs for shared-storage workflows:
 
@@ -119,6 +119,16 @@ Related knobs for shared-storage workflows:
 
 Plain local-disk builds are unaffected: without `--build_root`, all paths behave as before.
 
+#### Out-of-tree wheel builds (read-only checkout)
+
+For CI or ephemeral-node workflows that only need a wheel, add `--out-of-tree` to guarantee the checkout is never written — it can even be mounted read-only:
+
+```bash
+python3 scripts/build_wheel.py --build_root /tmp/trtllm-build --out-of-tree --use_ccache -a "90-real"
+```
+
+In this mode the generated FMHA kernel sources and the configured `version.h` go to the build tree (via the `TRTLLM_FMHA_GEN_DIR` and `TRTLLM_VERSION_H_INCLUDE_DIR` CMake variables), and the wheel is assembled from a staging copy of the Python package under `<build_root>/package`, landing in `<build_root>/dist` by default. Submodules and git-lfs content must be materialized before the build (the usual `git submodule update --init --recursive`), since the build will not modify the checkout. Editable-install workflows (`--skip_building_wheel`, `--linking_install_binary`, `--install`) are incompatible with `--out-of-tree`: they import compiled artifacts from the checkout by design. `--version-override` is also incompatible, since it edits `tensorrt_llm/version.py` in the checkout.
+
 ### Python-only build (no C++ compilation)
 
 If you only need to modify Python code, you can skip C++ compilation entirely by reusing precompiled binaries: