Skip to content

[CUDA] Fix Windows DeepGEMM build errors and warnings #32485

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Tianlei Wu (tianleiwu) merged 2 commits into from
Sep 9, 2026
Merged

[CUDA] Fix Windows DeepGEMM build errors and warnings #32485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fd2fe3c
Fix Windows CUDA build errors and warnings
tianleiwu Sep 8, 2026
9a97acc
Gate QMoE DeepGEMM path by availability
tianleiwu Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmake/onnxruntime_cuda_source_filters.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,9 @@ function(onnxruntime_extract_llm_sources CU_SRC_LIST)
set(_llm_sm90_srcs)
set(_llm_fp4_srcs)
set(_llm_excluded_srcs)
if(WIN32)
list(FILTER _list EXCLUDE REGEX "/moe_gemm/deep_gemm_sm90\\.cu$")
endif()
foreach(_src IN LISTS _list)
if(_src MATCHES "/contrib_ops/cuda/llm/.*\\.cu$")
if(onnxruntime_USE_FPA_INTB_GEMM AND NOT onnxruntime_USE_FPA_INTB_GEMM_FULL AND
Expand Down
3 changes: 2 additions & 1 deletion cmake/onnxruntime_providers_cuda.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,9 +393,10 @@

include(cutlass)
target_include_directories(${target} PRIVATE ${cutlass_SOURCE_DIR}/include ${cutlass_SOURCE_DIR}/examples ${cutlass_SOURCE_DIR}/tools/util/include)
if(ORT_HAS_SM90_OR_LATER AND NOT onnxruntime_CUDA_MINIMAL AND NOT onnxruntime_DISABLE_CONTRIB_OPS)
if(ORT_HAS_SM90_OR_LATER AND NOT WIN32 AND NOT onnxruntime_CUDA_MINIMAL AND NOT onnxruntime_DISABLE_CONTRIB_OPS)
Comment thread
tianleiwu marked this conversation as resolved.
Resolved
include(deep_gemm)
target_include_directories(${target} PRIVATE ${deep_gemm_SOURCE_DIR}/deep_gemm/include)
target_compile_definitions(${target} PRIVATE USE_DEEP_GEMM)
endif()
target_link_libraries(${target} PRIVATE Eigen3::Eigen)
target_include_directories(${target} PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
Expand Down
3 changes: 2 additions & 1 deletion cmake/onnxruntime_providers_cuda_plugin.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,10 @@ endif()

include(cudnn_frontend)
include(cutlass)
if(ORT_HAS_SM90_OR_LATER AND NOT onnxruntime_DISABLE_CONTRIB_OPS)
if(ORT_HAS_SM90_OR_LATER AND NOT WIN32 AND NOT onnxruntime_DISABLE_CONTRIB_OPS)
Comment thread
tianleiwu marked this conversation as resolved.
Resolved
include(deep_gemm)
target_include_directories(onnxruntime_providers_cuda_plugin PRIVATE ${deep_gemm_SOURCE_DIR}/deep_gemm/include)
target_compile_definitions(onnxruntime_providers_cuda_plugin PRIVATE USE_DEEP_GEMM)
endif()

# TMA compile definitions — mirror config_cuda_provider_shared_module in onnxruntime_providers_cuda.cmake
Expand Down
3 changes: 2 additions & 1 deletion model_package/src/manifest_parser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,9 @@ constexpr std::array<std::string_view, 6> kVariantKnownKeys = {
ModelPackageStatus* ReadFileToString(const fs::path& path, std::string* out) {
std::ifstream f(path, std::ios::binary);
if (!f) {
const std::error_code error_code(errno, std::generic_category());
return MakeStatus(MODEL_PACKAGE_ERR_IO,
"Cannot open file: '" + path.string() + "': " + std::strerror(errno));
"Cannot open file: '" + path.string() + "': " + error_code.message());
}
std::ostringstream buf;
buf << f.rdbuf();
Expand Down
6 changes: 3 additions & 3 deletions onnxruntime/contrib_ops/cuda/llm/moe_gemm/moe_kernels.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
#include "contrib_ops/cuda/llm/moe_gemm/moe_util_kernels.h"
#include "contrib_ops/cuda/llm/moe_gemm/moe_gemm_activation_kernels.cuh"
#include "contrib_ops/cuda/llm/moe_gemm/moe_gemm_utils.cuh"
#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
#include "contrib_ops/cuda/llm/moe_gemm/deep_gemm_sm90.h"
#endif

Expand Down Expand Up @@ -2213,7 +2213,7 @@ CutlassMoeFCRunner<T, WeightType, OutputType, InputType, ScaleBiasType, Enable>:
size_t smoothed_act_size = use_awq ? std::max(permuted_elems, interbuf_elems) * sizeof(T) * 2
: 0; // Extra workspace required by AWQ for smoothing activations
size_t fp4_deep_gemm_workspace_size = 0;
#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
if constexpr (std::is_same_v<T, __nv_bfloat16> && std::is_same_v<WeightType, __nv_bfloat16> &&
std::is_same_v<OutputType, __nv_bfloat16> && std::is_same_v<InputType, __nv_bfloat16>) {
if (use_fp4_deep_gemm_ && num_rows > 0 && num_rows <= deep_gemm_sm90::kMaxTokensPerExpert &&
Expand Down Expand Up @@ -2945,7 +2945,7 @@ void CutlassMoeFCRunner<T, WeightType, OutputType, InputType, ScaleBiasType, Ena

sync_check_cuda_error(stream);

#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
if constexpr (std::is_same_v<T, __nv_bfloat16> && std::is_same_v<WeightType, __nv_bfloat16> &&
std::is_same_v<OutputType, __nv_bfloat16> && std::is_same_v<InputType, __nv_bfloat16>) {
const bool use_fp4_deep_gemm =
Expand Down
10 changes: 5 additions & 5 deletions onnxruntime/contrib_ops/cuda/moe/moe_quantization.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#include "contrib_ops/cuda/llm/fpA_intB_gemm_preprocessors.h"
#include "contrib_ops/cuda/llm/moe_gemm/moe_gemv_fp4.h"
#include "contrib_ops/cuda/llm/moe_gemm/moe_util_kernels.h"
#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
#include "contrib_ops/cuda/llm/moe_gemm/deep_gemm_sm90.h"
#endif

Expand Down Expand Up @@ -172,7 +172,7 @@ bool StaticFp4CutlassShapeSupported(const OpKernelInfo& op_kernel_info, bool is_

// Returns the per-rank expert count DeepGEMM would run, or 0 if the static shapes rule it out.
int StaticFp4DeepGemmNumExperts(const OpKernelInfo& op_kernel_info) {
#if !defined(HAS_SM90_OR_LATER)
#if !defined(HAS_SM90_OR_LATER) || !defined(USE_DEEP_GEMM)
ORT_UNUSED_PARAMETER(op_kernel_info);
return 0;
#else
Expand Down Expand Up @@ -895,7 +895,7 @@ Status QMoE::ComputeInternal(OpKernelContext* context) const {
// through the fused GEMV or the dense A16 fallback instead. (MXFP4 keeps its existing routing.)
!(is_nvfp4 && fp4_prefill_min_tokens_ > 0 &&
static_cast<int64_t>(moe_params.num_rows) < fp4_prefill_min_tokens_);
#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
const bool use_fp4_deep_gemm =
enable_fp4_deep_gemm_ && moe_params.num_rows > 0 &&
moe_params.num_rows <= onnxruntime::llm::kernels::deep_gemm_sm90::kMaxTokensPerExpert &&
Expand Down Expand Up @@ -2042,7 +2042,7 @@ Status QMoE::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc,
#define DUMP_PACK_TENSOR(name, packed_scales, scales)
#endif

#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
if (enable_fp4_deep_gemm_ && (input_idx == 2 || input_idx == 5 || input_idx == 3 || input_idx == 6)) {
const bool fc1 = input_idx == 2 || input_idx == 3;
const bool weight = input_idx == 2 || input_idx == 5;
Expand Down Expand Up @@ -2736,7 +2736,7 @@ void QMoE::TryBuildGemvFp4Scales(int fc, cudaStream_t stream, AllocatorPtr alloc
}

void QMoE::TryBuildFp4DeepGemmWeights(int fc, cudaStream_t stream, AllocatorPtr alloc) {
#if defined(HAS_SM90_OR_LATER)
#if defined(HAS_SM90_OR_LATER) && defined(USE_DEEP_GEMM)
if (!enable_fp4_deep_gemm_) {
return;
}
Expand Down
28 changes: 14 additions & 14 deletions onnxruntime/core/mlas/lib/amd64/QgemmU8X8KernelAvx2.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1141,6 +1141,20 @@ ProcessCountM4:
ProcessCountM6:
ProcessCountM 6, ASigned, BSigned

ProcessCountM1:
cmp DWORD PTR GemmInt8KernelFrame.PreviousP1Home[rsp],-1
je ProcessCountM1AvxVnni
ProcessCountM 1, ASigned, BSigned

ProcessCountM1AvxVnni:
ProcessCount1AvxVnni 1, ASigned, BSigned

ProcessCountM3:
ProcessCountM 3, ASigned, BSigned

ProcessCountM5:
ProcessCountM 5, ASigned, BSigned

;
; Restore non-volatile registers and return.
;
Expand Down Expand Up @@ -1170,20 +1184,6 @@ ExitKernel:
pop rbp
ret

ProcessCountM1:
cmp DWORD PTR GemmInt8KernelFrame.PreviousP1Home[rsp],-1
je ProcessCountM1AvxVnni
ProcessCountM 1, ASigned, BSigned

ProcessCountM1AvxVnni:
ProcessCount1AvxVnni 1, ASigned, BSigned

ProcessCountM3:
ProcessCountM 3, ASigned, BSigned

ProcessCountM5:
ProcessCountM 5, ASigned, BSigned

ENDM

;
Expand Down
Loading