From a4b51f7e06a6c02a0049e707c5bc73a2e8c38eac Mon Sep 17 00:00:00 2001 From: Yusheng Su Date: Wed, 8 Jul 2026 14:47:17 -0700 Subject: [PATCH] Add temporary CI-image Dockerfile variant with sglang PR 30499 Builds sglang from pull/30499/head (GDN in_proj_ba LoRA support) so the Qwen3.5 LoRA CI in radixark/miles#1597 can run before the sglang PR merges. Throwaway branch; delete after sglang-miles picks up the PR. Signed-off-by: Yusheng Su --- docker/Dockerfile.qwen35-gdn | 206 +++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docker/Dockerfile.qwen35-gdn diff --git a/docker/Dockerfile.qwen35-gdn b/docker/Dockerfile.qwen35-gdn new file mode 100644 index 00000000000..f714d0afa92 --- /dev/null +++ b/docker/Dockerfile.qwen35-gdn @@ -0,0 +1,206 @@ +# doc-dev: docs/ci/02-docker-build.md +# Build via docker/build.py (single source of truth for build-args, platforms, +# and tags) — see docs/ci/02-docker-build.md. Examples: +# python docker/build.py --variant cu13 --image-tag dev --push # radixark/miles:dev (amd64+arm64) +# python docker/build.py --variant cu13-x86 --image-tag dev --push # radixark/miles:dev (amd64) +# python docker/build.py --variant cu13-aarch64 --image-tag dev --push # radixark/miles:dev (arm64) +# python docker/build.py --variant cu12-x86 --image-tag dev --push # radixark/miles:dev-cu12 (CUDA 12.9) +# +# A multi-arch build runs the recipe once per platform; it installs the wheels +# release for the current arch picked from TARGETARCH — WHEELS_TAG_X86 on amd64, +# WHEELS_TAG_ARM64 on arm64 (cu12-x86 overrides WHEELS_TAG_X86). + +ARG SGLANG_IMAGE_TAG=v0.5.14 +FROM lmsysorg/sglang:${SGLANG_IMAGE_TAG} AS sglang + +# ======================================== Arguments ============================================= + +# Temporary CI-image variant: sglang from the unmerged in_proj_ba PR +# (sgl-project/sglang#30499). Delete this file once the PR lands in sglang-miles. +ARG SGLANG_BRANCH=pull/30499/head +ARG SGLANG_COMMIT="" + +ARG MEGATRON_REPO=radixark/Megatron-LM +ARG MEGATRON_BRANCH=miles-main + +ARG ENABLE_CUDA_13=1 + +ARG WHEELS_REPO=yueming-yuan/miles-wheels +# Two complete wheels release tags (the wheels repo's own names). The build picks +# one by TARGETARCH and installs it verbatim — no on-the-fly tag assembly here. +ARG TARGETARCH +ARG WHEELS_TAG_X86=cu130-x86_64-v0.5.12 +ARG WHEELS_TAG_ARM64=cu130-aarch64-v0.5.12 + +# ======================================== Setup ============================================= + +WORKDIR /root/ + +# ======================================== Apt dependencies ============================================= + +RUN apt update +# ethtool for network diagnostics +RUN apt install -y nvtop rsync dnsutils ethtool + +# nccl-tests for diagnostics +RUN git clone https://github.com/NVIDIA/nccl-tests.git /tmp/nccl-tests && \ + cd /tmp/nccl-tests && \ + git checkout ae98985f5599617be94042f4aa3637d10014ce89 && \ + make -j$(nproc) CUDA_HOME=/usr/local/cuda && \ + cp /tmp/nccl-tests/build/*_perf /usr/local/bin/ && \ + rm -rf /tmp/nccl-tests + +# ====================================== Collect pre-built wheels ============================================ +# Optional: drop pre-built wheels into /wheels/ to skip the release download below. +# `wheel[s]/` is a glob — silently no-ops when wheels/ is absent (CI's path). +COPY wheel[s]/ /tmp/wheels/ + +RUN case "${TARGETARCH}" in \ + amd64) WHEELS_TAG="${WHEELS_TAG_X86}" ;; \ + arm64) WHEELS_TAG="${WHEELS_TAG_ARM64}" ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + echo "Fetching wheels release ${WHEELS_TAG}" && \ + mkdir -p /tmp/wheels && \ + curl -sL "https://api.github.com/repos/${WHEELS_REPO}/releases/tags/${WHEELS_TAG}" \ + | python3 -c "import sys,json,subprocess,os; w='/tmp/wheels'; \ +[subprocess.run(['curl','-fSL','-o',os.path.join(w,a['name']),a['browser_download_url']],check=True) \ + for a in json.load(sys.stdin).get('assets',[]) \ + if a['name'].endswith(('.whl', '.tar.gz')) and not os.path.exists(os.path.join(w,a['name']))]" && \ + ls -lh /tmp/wheels/ + +# ====================================== Python dependencies ============================================ + +# flash-attn +RUN pip install /tmp/wheels/flash_attn-*.whl + +# flash-attn hopper (FA3): Hopper-only (sm_90a). Coexists with FA2. +RUN pip install /tmp/wheels/flash_attn_3-*.whl && \ + python_path=$(python -c "import site; print(site.getsitepackages()[0])") && \ + mkdir -p $python_path/flash_attn_3 && \ + curl -fSL https://raw.githubusercontent.com/Dao-AILab/flash-attention/fbf24f67cf7f6442c5cfb2c1057f4bfc57e72d89/hopper/flash_attn_interface.py \ + -o $python_path/flash_attn_3/flash_attn_interface.py + +RUN pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps + +RUN pip install flash-linear-attention==0.4.2 +# required for DeepSeek V4 +RUN pip install tilelang==0.1.8 -f https://tile-ai.github.io/whl/nightly/cu128/ +RUN pip install --no-deps tile_kernels==1.0.0 +# FlashQLA backend for Qwen GDN linear-attention layers (requires SM90+, CUDA 12.8+, PyTorch 2.8+; built on tilelang above) +RUN pip install -v --no-build-isolation "git+https://github.com/QwenLM/FlashQLA.git" +RUN pip install "git+https://github.com/Dao-AILab/fast-hadamard-transform.git@e7706faf8d1c3b9f241e36860640ad1dac644ede" --no-build-isolation + +# Mamba kernels for nemotron_h hybrid (mamba+attention) models. +RUN pip install causal-conv1d==1.6.1 mamba-ssm==2.3.1 --no-build-isolation + +# transformer_engine +RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \ + pip install --no-deps transformer_engine==2.12.0 && \ + pip install transformer_engine_cu13==2.12.0 && \ + if ls /tmp/wheels/transformer_engine_torch-*.whl 2>/dev/null | grep -q .; then \ + pip install /tmp/wheels/transformer_engine_torch-*.whl; \ + else \ + pip install nvidia-mathdx==25.6.0 && \ + pip -v install --no-build-isolation transformer_engine_torch==2.12.0; \ + fi; \ + else \ + pip -v install --no-build-isolation "transformer_engine[pytorch]==2.10.0"; \ + fi + +# TE patches (cu13): B300/GB300 sm103 FA2 whitelist fix +COPY docker/patch/ /tmp/patches/ +RUN if [ "${ENABLE_CUDA_13}" = "1" ] && [ -d /tmp/patches/cu13 ]; then \ + TE_DIR=$(python -c 'import transformer_engine; print(transformer_engine.__path__[0])') && \ + for p in /tmp/patches/cu13/*.patch; do \ + echo "Applying $(basename $p) to $TE_DIR" && \ + patch -d "$TE_DIR" -p1 < "$p"; \ + done; \ + fi && rm -rf /tmp/patches + +# apex +RUN pip install /tmp/wheels/apex-*.whl + +RUN git clone https://github.com/${MEGATRON_REPO}.git --recursive -b ${MEGATRON_BRANCH} Megatron-LM && \ + cd Megatron-LM && \ + pip install -e . + +RUN pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@d64a639 --no-cache-dir --force-reinstall +RUN pip install "nvidia-modelopt[torch]>=0.37.0" --no-build-isolation +RUN pip install git+https://github.com/radixark/Megatron-Bridge.git@bridge --no-deps --no-build-isolation +RUN pip install megatron-energon --no-deps +RUN pip install multi-storage-client --no-deps + +COPY requirements.txt /tmp/requirements.txt +RUN rm -rf /usr/lib/python3/dist-packages/jwt /usr/lib/python3/dist-packages/PyJWT* && pip install -r /tmp/requirements.txt + +# https://github.com/pytorch/pytorch/issues/168167 +RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \ + pip install nvidia-cudnn-cu13==9.16.0.29; \ + else \ + pip install nvidia-cudnn-cu12==9.16.0.29; \ + fi + + +RUN rm -rf /root/.cache/pip /root/flash-attention + +# ====================================== Install sglang-miles ============================================ + +# Install sglang from sglang-miles branch +RUN cd /sgl-workspace/sglang && \ + git fetch origin ${SGLANG_BRANCH} && \ + if [ -n "${SGLANG_COMMIT}" ]; then \ + git checkout ${SGLANG_COMMIT}; \ + else \ + git checkout FETCH_HEAD; \ + fi && \ + pip install -e "python[all]" --no-deps + +# ====================================== Install main package ============================================ + +ARG MILES_COMMIT=main +RUN git clone https://github.com/radixark/miles.git /root/miles && \ + cd /root/miles && \ + git checkout ${MILES_COMMIT} && \ + pip install -e . --no-deps + +# int4_qat +RUN pip install /tmp/wheels/fake_int4_quant_cuda-*.whl + +# ====================================== Install sgl-model-gateway ============================================ +# SGL_ROUTER_USE_WHEELS=0: +# Build from source https://github.com/radixark/sgl-router-for-miles +# SGL_ROUTER_USE_WHEELS=1 (default): +# Install the pre-built sgl-model-gateway wheel + +ARG SGL_ROUTER_USE_WHEELS=1 +ARG SGL_ROUTER_REPO=https://github.com/radixark/sgl-router-for-miles.git +ARG SGL_ROUTER_BRANCH=main + +RUN --mount=type=cache,target=/root/.cache/pip \ + set -eux; \ + if [ "${SGL_ROUTER_USE_WHEELS}" = "1" ]; then \ + pip install --force-reinstall /tmp/wheels/sglang_router-*.whl && \ + tar xzf /tmp/wheels/sgl-model-gateway-linux-*.tar.gz -C /usr/local/bin/ && \ + chmod +x /usr/local/bin/sgl-model-gateway; \ + elif [ "${SGL_ROUTER_USE_WHEELS}" = "0" ]; then \ + git clone --branch "${SGL_ROUTER_BRANCH}" --depth 1 "${SGL_ROUTER_REPO}" /build/sgl-model-gateway && \ + curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.rustup.rs | sh -s -- -y && \ + export PATH="/root/.cargo/bin:${PATH}" && \ + python3 -m pip install maturin && \ + cd /build/sgl-model-gateway/bindings/python && \ + ulimit -n 65536 && \ + maturin build --release --features vendored-openssl --out /build/gateway_wheels && \ + cd /build/sgl-model-gateway && \ + cargo build --release --bin sgl-model-gateway --features vendored-openssl && \ + cp target/release/sgl-model-gateway /usr/local/bin/sgl-model-gateway && \ + chmod +x /usr/local/bin/sgl-model-gateway && \ + pip install --force-reinstall /build/gateway_wheels/sglang_router-*.whl && \ + rm -rf /root/.cargo /root/.rustup /build/sgl-model-gateway /build/gateway_wheels; \ + fi + +RUN rm -rf /tmp/wheels + +# cu130 sglang base ships rust at /root/.cargo/bin but not on PATH; a later CI +# step calls `rustc` directly. Last so cached layers above stay valid. +ENV PATH="/root/.cargo/bin:${PATH}"