diff --git a/.github/workflows/release-ci-docker.yml b/.github/workflows/release-ci-docker.yml index 9303627482..7da8df6d58 100644 --- a/.github/workflows/release-ci-docker.yml +++ b/.github/workflows/release-ci-docker.yml @@ -67,7 +67,7 @@ jobs: - name: Build and push ${{ matrix.cuda }} ${{ matrix.arch }} image uses: docker/build-push-action@v5 with: - context: docker + context: . file: docker/Dockerfile.${{ matrix.cuda }} platforms: linux/${{ matrix.arch }} push: ${{ github.event_name != 'pull_request' }} diff --git a/docker/Dockerfile.cu126 b/docker/Dockerfile.cu126 index 8c839913c1..e67d1dfbd4 100644 --- a/docker/Dockerfile.cu126 +++ b/docker/Dockerfile.cu126 @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \ wget # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /opt/conda py312 # Set home directory @@ -20,7 +20,8 @@ ENV PATH="/opt/conda/bin:$PATH" ENV PATH="/opt/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu126 # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu126.dev b/docker/Dockerfile.cu126.dev index 490a70fa42..9b8b1ccd49 100644 --- a/docker/Dockerfile.cu126.dev +++ b/docker/Dockerfile.cu126.dev @@ -37,7 +37,7 @@ USER $USERNAME WORKDIR /home/$USERNAME # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /home/$USERNAME/conda py312 RUN echo "source activate py312" >> ~/.bashrc @@ -45,7 +45,8 @@ ENV PATH="/home/$USERNAME/conda/bin:$PATH" ENV PATH="/home/$USERNAME/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu126 && pip3 install pre-commit # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu128 b/docker/Dockerfile.cu128 index 4baf09d815..ed1bb194db 100644 --- a/docker/Dockerfile.cu128 +++ b/docker/Dockerfile.cu128 @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \ wget # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /opt/conda py312 # Set home directory @@ -20,7 +20,8 @@ ENV PATH="/opt/conda/bin:$PATH" ENV PATH="/opt/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu128 # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu128.dev b/docker/Dockerfile.cu128.dev index 5d98409515..ad734724d5 100644 --- a/docker/Dockerfile.cu128.dev +++ b/docker/Dockerfile.cu128.dev @@ -37,7 +37,7 @@ USER $USERNAME WORKDIR /home/$USERNAME # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /home/$USERNAME/conda py312 RUN echo "source activate py312" >> ~/.bashrc @@ -45,7 +45,8 @@ ENV PATH="/home/$USERNAME/conda/bin:$PATH" ENV PATH="/home/$USERNAME/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu128 && pip3 install pre-commit # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu129 b/docker/Dockerfile.cu129 index 4474047b61..b395f201c4 100644 --- a/docker/Dockerfile.cu129 +++ b/docker/Dockerfile.cu129 @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \ wget # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /opt/conda py312 # Set home directory @@ -23,7 +23,8 @@ ENV PATH="/opt/conda/envs/py312/bin:$PATH" ENV TRITON_PTXAS_PATH="/usr/local/cuda/bin/ptxas" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu129 # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu129.dev b/docker/Dockerfile.cu129.dev index 3536566cde..59c5368932 100644 --- a/docker/Dockerfile.cu129.dev +++ b/docker/Dockerfile.cu129.dev @@ -37,7 +37,7 @@ USER $USERNAME WORKDIR /home/$USERNAME # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /home/$USERNAME/conda py312 RUN echo "source activate py312" >> ~/.bashrc @@ -45,7 +45,8 @@ ENV PATH="/home/$USERNAME/conda/bin:$PATH" ENV PATH="/home/$USERNAME/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh RUN bash /install/install_python_packages.sh cu129 && pip3 install pre-commit # Install mpi4py in the conda environment diff --git a/docker/Dockerfile.cu130 b/docker/Dockerfile.cu130 index ab14e12433..e9d08511ad 100644 --- a/docker/Dockerfile.cu130 +++ b/docker/Dockerfile.cu130 @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y \ wget # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /opt/conda py312 # Set home directory @@ -23,7 +23,8 @@ ENV PATH="/opt/conda/envs/py312/bin:$PATH" ENV TRITON_PTXAS_PATH="/usr/local/cuda/bin/ptxas" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh # use nightly/cu130 temporarily and change to cu130 when torch releases stable version RUN bash /install/install_python_packages.sh nightly/cu130 diff --git a/docker/Dockerfile.cu130.dev b/docker/Dockerfile.cu130.dev index 7aa5a81c42..2bef85754f 100644 --- a/docker/Dockerfile.cu130.dev +++ b/docker/Dockerfile.cu130.dev @@ -37,7 +37,7 @@ USER $USERNAME WORKDIR /home/$USERNAME # Install python -COPY install/install_python.sh /install/install_python.sh +COPY docker/install/install_python.sh /install/install_python.sh RUN bash /install/install_python.sh /home/$USERNAME/conda py312 RUN echo "source activate py312" >> ~/.bashrc @@ -45,7 +45,8 @@ ENV PATH="/home/$USERNAME/conda/bin:$PATH" ENV PATH="/home/$USERNAME/conda/envs/py312/bin:$PATH" # Install torch and other python packages -COPY install/install_python_packages.sh /install/install_python_packages.sh +COPY requirements.txt /install/requirements.txt +COPY docker/install/install_python_packages.sh /install/install_python_packages.sh # use nightly/cu130 temporarily and change to cu130 when torch releases stable version RUN bash /install/install_python_packages.sh nightly/cu130 && pip3 install pre-commit diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index e74a2b500a..bd35b96a43 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -23,11 +23,8 @@ set -u CUDA_VERSION=${1:-cu128} pip3 install torch --index-url https://download.pytorch.org/whl/${CUDA_VERSION} -pip3 install requests responses ninja pytest numpy scipy build nvidia-ml-py cuda-python einops nvidia-nvshmem-cu12 -pip3 install click -pip3 install 'apache-tvm-ffi==0.1.0b15' -pip3 install nvidia-cutlass-dsl -pip3 install 'nvidia-cudnn-frontend>=1.13.0' +pip3 install -r /install/requirements.txt +pip3 install responses pytest scipy build cuda-python nvidia-nvshmem-cu12 # Install cudnn package based on CUDA version if [[ "$CUDA_VERSION" == *"cu13"* ]]; then diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..a4e391c38d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +apache-tvm-ffi==0.1.0b15 +click +einops +ninja +numpy +nvidia-cudnn-frontend>=1.13.0 +nvidia-cutlass-dsl>=4.2.1 +nvidia-ml-py +packaging>=24.2 +requests +tabulate +torch +tqdm diff --git a/setup.py b/setup.py index 6c663c1cb6..3f021464e9 100755 --- a/setup.py +++ b/setup.py @@ -41,21 +41,21 @@ def generate_build_meta() -> None: ext_modules: List[setuptools.Extension] = [] cmdclass: Mapping[str, type[setuptools.Command]] = {} -install_requires = [ - "numpy", - "torch", - "ninja", - "requests", - "nvidia-ml-py", - "einops", - "click", - "tqdm", - "tabulate", - "apache-tvm-ffi==0.1.0b15", - "packaging>=24.2", - "nvidia-cudnn-frontend>=1.13.0", - "nvidia-cutlass-dsl>=4.2.1", -] + + +def get_install_requires() -> List[str]: + """Read install requirements from requirements.txt.""" + requirements_file = root / "requirements.txt" + if not requirements_file.exists(): + return [] + return [ + line.strip() + for line in requirements_file.read_text().splitlines() + if line.strip() and not line.strip().startswith("#") + ] + + +install_requires = get_install_requires() generate_build_meta()