Skip to content
Merged
113 changes: 45 additions & 68 deletions docker/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# 1. radixark/miles:dev
# use default settings
#
# 2. radixark/miles:dev-cu13-arm64
# build-arg:ENABLE_CUDA_13=1 \
# build-arg:SGLANG_IMAGE_TAG=v0.5.8.post1-cu130-arm64 \
# build-arg:WHEELS_TAG=cu130-aarch64 \

ARG SGLANG_IMAGE_TAG=v0.5.8.post1
FROM lmsysorg/sglang:${SGLANG_IMAGE_TAG} AS sglang

Expand All @@ -6,12 +14,12 @@ FROM lmsysorg/sglang:${SGLANG_IMAGE_TAG} AS sglang
ARG SGLANG_BRANCH=sglang-miles
ARG SGLANG_COMMIT=""

ARG PATCH_VERSION=dev
ARG MEGATRON_COMMIT=3714d81d418c9f1bca4594fc35f9e8289f652862
ARG MEGATRON_REPO=radixark/Megatron-LM
ARG MEGATRON_BRANCH=miles-main

ARG ENABLE_CUDA_13=0

ARG WHEELS_BASE_URL=https://github.com/yueming-yuan/miles-wheels/releases/download
ARG WHEELS_REPO=yueming-yuan/miles-wheels
ARG WHEELS_TAG=cu129-x86_64

# ======================================== Setup =============================================
Expand All @@ -23,78 +31,59 @@ WORKDIR /root/
RUN apt update
RUN apt install -y nvtop rsync dnsutils

# ====================================== Download pre-built wheels ============================================

RUN mkdir -p /tmp/wheels && \
curl -sL "https://api.github.com/repos/${WHEELS_REPO}/releases/tags/${WHEELS_TAG}" \
| python3 -c "import sys, json, subprocess; \
[subprocess.run(['curl', '-fSL', '-o', '/tmp/wheels/' + a['name'], a['browser_download_url']], check=True) \
for a in json.load(sys.stdin)['assets'] if a['name'].endswith('.whl')]" && \
Comment on lines +38 to +40

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This Python one-liner is a bit dense and has a couple of potential issues:

  1. It will fail with a KeyError if the GitHub API response for the release does not contain an assets key. Using .get('assets', []) would be more robust.
  2. Using a list comprehension for its side effects (calling subprocess.run) is not idiomatic Python. A for loop is more explicit and readable.

Consider refactoring this into a multi-line script within the RUN command for better readability and maintainability, for example:

import sys, json, subprocess

release_data = json.load(sys.stdin)
for asset in release_data.get('assets', []):
    if asset['name'].endswith('.whl'):
        url = asset['browser_download_url']
        filename = '/tmp/wheels/' + asset['name']
        print(f"Downloading {url} to {filename}")
        subprocess.run(['curl', '-fSL', '-o', filename, url], check=True)

ls -lh /tmp/wheels/

# ====================================== Python dependencies ============================================

# Install pre-built wheels from GitHub release (cu12.9 path only)
RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
# CUDA 13: compile from source as before
MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation; \
else \
pip install ${WHEELS_BASE_URL}/${WHEELS_TAG}/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl; \
fi
# flash-attn
RUN pip install /tmp/wheels/flash_attn-*.whl

RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
git clone https://github.com/Dao-AILab/flash-attention.git && \
cd flash-attention/ && git checkout fbf24f67cf7f6442c5cfb2c1057f4bfc57e72d89 && git submodule update --init && cd hopper/ && \
MAX_JOBS=96 python setup.py install && \
export python_path=`python -c "import site; print(site.getsitepackages()[0])"` && \
mkdir -p $python_path/flash_attn_3 && \
cp flash_attn_interface.py $python_path/flash_attn_3/flash_attn_interface.py && \
rm -rf flash-attention/; \
else \
pip install ${WHEELS_BASE_URL}/${WHEELS_TAG}/flash_attn_3-3.0.0b1-cp39-abi3-linux_x86_64.whl && \
python_path=$(python -c "import site; print(site.getsitepackages()[0])") && \
mkdir -p $python_path/flash_attn_3 && \
curl -fSL https://raw.githubusercontent.com/Dao-AILab/flash-attention/fbf24f67cf7f6442c5cfb2c1057f4bfc57e72d89/hopper/flash_attn_interface.py \
-o $python_path/flash_attn_3/flash_attn_interface.py; \
fi
# flash-attn hopper
RUN pip install /tmp/wheels/flash_attn_3-*.whl && \
python_path=$(python -c "import site; print(site.getsitepackages()[0])") && \
mkdir -p $python_path/flash_attn_3 && \
curl -fSL https://raw.githubusercontent.com/Dao-AILab/flash-attention/fbf24f67cf7f6442c5cfb2c1057f4bfc57e72d89/hopper/flash_attn_interface.py \
-o $python_path/flash_attn_3/flash_attn_interface.py

RUN pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps

RUN pip install flash-linear-attention==0.4.0
RUN pip install tilelang -f https://tile-ai.github.io/whl/nightly/cu128/

# TE does not have wheel on cuda 13 yet, thus need to install from source
RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
pip install nvidia-mathdx==26.6.0 && \
pip -v install --no-build-isolation git+https://github.com/NVIDIA/TransformerEngine.git@release_v2.10; \
pip install nvidia-mathdx==25.6.0 && \
pip -v install --no-build-isolation "transformer_engine[core_cu13,pytorch]==2.10.0"; \
else \
pip -v install --no-build-isolation "transformer_engine[pytorch]==2.10.0"; \
fi

RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
NVCC_APPEND_FLAGS="--threads 4" \
pip -v install --disable-pip-version-check --no-cache-dir \
--no-build-isolation \
--config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4; \
else \
pip install ${WHEELS_BASE_URL}/${WHEELS_TAG}/apex-0.1-cp312-cp312-linux_x86_64.whl; \
fi
# apex
RUN pip install /tmp/wheels/apex-*.whl

RUN git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
RUN git clone https://github.com/${MEGATRON_REPO}.git --recursive -b ${MEGATRON_BRANCH} Megatron-LM && \
cd Megatron-LM && \
pip install -e .

RUN pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@dc6876905830430b5054325fa4211ff302169c6b --no-cache-dir --force-reinstall
RUN pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
RUN pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# This patch from masahi will be included in later Triton releases
RUN if [ "$ENABLE_CUDA_13" = "1" ]; then \
(cd /root && git clone -b feat/v350_plus_8045 https://github.com/fzyzcjy/triton.git && cd triton && pip install -r python/requirements.txt && pip install --verbose -e .); \
fi
RUN pip install "nvidia-modelopt[torch]>=0.37.0" --no-build-isolation

COPY requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt

# Temporarily install another sgl-kernel version for GB300 without rebuilding the whole image
RUN if [ "$ENABLE_CUDA_13" = "1" ]; then \
SGL_KERNEL_VERSION=0.3.17.post2 && \
python3 -m pip install https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu130-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps; \
fi

# https://github.com/pytorch/pytorch/issues/168167
RUN pip install nvidia-cudnn-cu12==9.16.0.29
RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
pip install nvidia-cudnn-cu13==9.16.0.29 \
else \
pip install nvidia-cudnn-cu12==9.16.0.29 \
fi

# reinstall numpy 1.x for megatron
RUN pip install "numpy<2"
Expand All @@ -103,17 +92,7 @@ RUN rm -rf /root/.cache/pip /root/flash-attention

# ====================================== Patches ============================================

COPY docker/patch/${PATCH_VERSION}/megatron.patch /root/Megatron-LM/
RUN cd Megatron-LM && \
git update-index --refresh && \
git apply megatron.patch --3way && \
if grep -R -n '^<<<<<<< ' .; then \
echo "Patch failed to apply cleanly. Please resolve conflicts." && \
exit 1; \
fi && \
rm megatron.patch

# Install sglang from sglang-miles branch instead of patching
# Install sglang from sglang-miles branch
RUN cd /sgl-workspace/sglang && \
git fetch origin ${SGLANG_BRANCH} && \
if [ -n "${SGLANG_COMMIT}" ]; then \
Expand All @@ -131,9 +110,7 @@ RUN git clone https://github.com/radixark/miles.git /root/miles && \
git checkout ${MILES_COMMIT} && \
pip install -e . --no-deps

RUN if [ "${ENABLE_CUDA_13}" = "1" ]; then \
cd /root/miles/miles/backends/megatron_utils/kernels/int4_qat && \
pip install . --no-build-isolation; \
else \
pip install ${WHEELS_BASE_URL}/${WHEELS_TAG}/fake_int4_quant_cuda-0.0.0-cp312-cp312-linux_x86_64.whl; \
fi
# int4_qat
RUN pip install /tmp/wheels/fake_int4_quant_cuda-*.whl

RUN rm -rf /tmp/wheels
Loading