Skip to content

[CI] Force-reinstall nvidia-cutlass-dsl-libs-cu13 last to avoid wheel-mix TypeError #25958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Kangyan-Zhou merged 3 commits into from
May 21, 2026
Merged

[CI] Force-reinstall nvidia-cutlass-dsl-libs-cu13 last to avoid wheel-mix TypeError #25958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a0dbf2
[CI] Force-reinstall nvidia-cutlass-dsl-libs-cu13 last to avoid wheel…
Kangyan-Zhou May 21, 2026
c64835a
Address review: use REPO_ROOT path, bump cutlass-dsl to 4.5.1
Kangyan-Zhou May 21, 2026
13f8cf2
Merge branch 'main' into fix_cutlass_libs_install_order
hnyls2002 May 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies = [
"ninja",
"easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports
"numpy",
"nvidia-cutlass-dsl[cu13]==4.5.0",
"nvidia-cutlass-dsl[cu13]==4.5.1",
"nvidia-ml-py",
"openai-harmony==0.0.4",
"openai==2.6.1",
Expand Down
29 changes: 29 additions & 0 deletions scripts/ci/cuda/ci_install_dependency.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,34 @@ download_flashinfer_cache() {
mark_step_done "${FUNCNAME[0]}"
}

force_reinstall_cutlass_dsl_libs_cu13() {
# nvidia-cutlass-dsl[cu13] has additive PyPI extras: installing it pulls in
# both -libs-base and -libs-cu13. The two wheels ship intentionally-different
# content for the same paths (cutlass/_mlir/dialects/_gpu_ops_gen.py and
# cutlass/_mlir/_mlir_libs/_cutlass_ir.cpython-*.so) -- each Python wrapper
# is paired with a matching pybind11 .so. If install order leaves the .py
# from one wheel and the .so from the other, GPUModuleOp.__init__ raises
# TypeError: incompatible function arguments at kernel-compile time.
#
# Force-reinstall -libs-cu13 LAST so both files come from the same wheel
# (BOTH-cu13 state), eliminating the mismatch. The version is parsed from
# pyproject.toml so this stays in sync with whatever nvidia-cutlass-dsl
# version the project pins.
if [ "$CU_MAJOR" != "13" ]; then
return
fi

CUTLASS_DSL_VERSION=$(grep -Po -m1 'nvidia-cutlass-dsl(\[[^]]+\])?==\K[0-9A-Za-z\.\-]+' "${REPO_ROOT}/python/pyproject.toml" || echo "")
if [ -z "$CUTLASS_DSL_VERSION" ]; then
echo "WARNING: could not detect nvidia-cutlass-dsl version from pyproject.toml; skipping libs-cu13 force-reinstall"
return
fi

$PIP_CMD install --force-reinstall --no-deps "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}" $PIP_INSTALL_SUFFIX

mark_step_done "${FUNCNAME[0]}"
}

stabilize_flashinfer_jit_paths() {
# In venv mode, FlashInfer JIT writes build.ninja with hardcoded -isystem
# paths. Per-job venvs get unique paths, but the JIT cache is shared on the
Expand Down Expand Up @@ -488,6 +516,7 @@ main() {
install_sglang_kernel
install_sglang_router
download_flashinfer_cache
force_reinstall_cutlass_dsl_libs_cu13
stabilize_flashinfer_jit_paths
install_extra_deps
install_test_tools
Expand Down
Loading