From 2a81abcf56b3e3bb266c737fcb22a1dc4d95fea6 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:28:18 +0100 Subject: [PATCH 01/14] Update Dockerfile --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index cd66e9d729..020d155ce3 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -57,7 +57,7 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - CUDA_VERSION=121 make cuda12x && \ + cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc . && \ python setup.py develop && \ pip freeze | grep bitsandbytes From 1ac22f99f364181985c65a83d94d4102d38c7f7e Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:28:55 +0100 Subject: [PATCH 02/14] Update Dockerfile --- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index cd39105702..dfcfa9b70d 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -57,7 +57,7 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - CUDA_VERSION=121 make cuda12x && \ + cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc . && \ python setup.py develop && \ pip freeze | grep bitsandbytes From c682b08007a0fb04bc5680521ab5e27fb34744d3 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:29:51 +0100 Subject: [PATCH 03/14] Update build_docker_images.yml --- .github/workflows/build_docker_images.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml index a888a08934..efa9e51cc4 100644 --- a/.github/workflows/build_docker_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -1,10 +1,8 @@ name: Build Docker images (scheduled) on: - workflow_dispatch: - workflow_call: - schedule: - - cron: "0 1 * * *" + push: + - branches: ["younesbelkada-patch-2"] concurrency: group: docker-image-builds From e08683ba6f5e98f4bbd4929a47b5dde22bff88c9 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:30:31 +0100 Subject: [PATCH 04/14] Update build_docker_images.yml --- .github/workflows/build_docker_images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml index efa9e51cc4..931d2434c9 100644 --- a/.github/workflows/build_docker_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -2,7 +2,8 @@ name: Build Docker images (scheduled) on: push: - - branches: ["younesbelkada-patch-2"] + branches: + - younesbelkada-patch-2 concurrency: group: docker-image-builds From b844b78cb252a6e2d0913e25cc899259a156f866 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:37:30 +0100 Subject: [PATCH 05/14] Update Dockerfile --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index 020d155ce3..3a848dbc0a 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -57,7 +57,7 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc . && \ + cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ python setup.py develop && \ pip freeze | grep bitsandbytes From 72152326ede93f88831908b985dd54d104693b46 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:37:42 +0100 Subject: [PATCH 06/14] Update Dockerfile --- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index dfcfa9b70d..8028928a89 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -57,7 +57,7 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc . && \ + cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ python setup.py develop && \ pip freeze | grep bitsandbytes From 88d7be4a8f2f1e7cf013dd830ab6532a3c82d114 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 5 Feb 2024 23:52:59 +0000 Subject: [PATCH 07/14] add cmake to dockerfile --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index 3a848dbc0a..bf6e99f209 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -40,7 +40,7 @@ SHELL ["/bin/bash", "-c"] # Install apt libs RUN apt-get update && \ - apt-get install -y curl git wget && \ + apt-get install -y curl git wget cmake && \ apt-get clean && \ rm -rf /var/lib/apt/lists* diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index 8028928a89..51769ad37b 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -40,7 +40,7 @@ SHELL ["/bin/bash", "-c"] # Install apt libs RUN apt-get update && \ - apt-get install -y curl git wget && \ + apt-get install -y curl git wget cmake && \ apt-get clean && \ rm -rf /var/lib/apt/lists* From 1baaacd6d20c45cc4f7a78bcf756b3a4ee9d8f6e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 5 Feb 2024 23:59:42 +0000 Subject: [PATCH 08/14] use pip install instead --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index bf6e99f209..351c538f6d 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -58,7 +58,7 @@ RUN source activate peft && \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ - python setup.py develop && \ + pip install -e . && \ pip freeze | grep bitsandbytes RUN echo "source activate peft" >> ~/.profile diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index 51769ad37b..f7d8d1870f 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -58,7 +58,7 @@ RUN source activate peft && \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ - python setup.py develop && \ + pip install -e . && \ pip freeze | grep bitsandbytes RUN echo "source activate peft" >> ~/.profile From 46a81daea05c4cf820454133c16c0cfec1118919 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:09:08 +0100 Subject: [PATCH 09/14] Update Dockerfile --- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index f7d8d1870f..2ec5cf0e16 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 AS build-image +FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH From ea7b9410b337f42c10101832336dcdc8a2d43694 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:09:20 +0100 Subject: [PATCH 10/14] Update Dockerfile --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index 351c538f6d..e5e9d783bb 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 AS build-image +FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH From c8ce05b61cffbe182b9f761aaa9ad9cb54fc7ab1 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:21:14 +0100 Subject: [PATCH 11/14] Update Dockerfile --- docker/peft-gpu-bnb-source/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index 2ec5cf0e16..9734de7f7d 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 AS build-image +FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH From 7b81850492274a37508dd03600386cbd6fde9b7a Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 02:21:28 +0100 Subject: [PATCH 12/14] Update Dockerfile --- docker/peft-gpu-bnb-latest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index e5e9d783bb..74f819f869 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -31,7 +31,7 @@ RUN chsh -s /bin/bash SHELL ["/bin/bash", "-c"] # Stage 2 -FROM nvidia/cuda:12.1.0-devel-ubuntu20.04 AS build-image +FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS build-image COPY --from=compile-image /opt/conda /opt/conda ENV PATH /opt/conda/bin:$PATH From de5e6e2e0c641813a0c0f1c250d8bd55ba7b2cba Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 6 Feb 2024 18:46:30 +0000 Subject: [PATCH 13/14] final fix --- docker/peft-gpu-bnb-latest/Dockerfile | 3 ++- docker/peft-gpu-bnb-source/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/peft-gpu-bnb-latest/Dockerfile b/docker/peft-gpu-bnb-latest/Dockerfile index 351c538f6d..cfe2e945e9 100644 --- a/docker/peft-gpu-bnb-latest/Dockerfile +++ b/docker/peft-gpu-bnb-latest/Dockerfile @@ -57,7 +57,8 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ + cmake -B . -DCOMPUTE_BACKEND=cuda -S . && \ + cmake --build . && \ pip install -e . && \ pip freeze | grep bitsandbytes diff --git a/docker/peft-gpu-bnb-source/Dockerfile b/docker/peft-gpu-bnb-source/Dockerfile index f7d8d1870f..7fb611d905 100644 --- a/docker/peft-gpu-bnb-source/Dockerfile +++ b/docker/peft-gpu-bnb-source/Dockerfile @@ -57,7 +57,8 @@ RUN source activate peft && \ optimum \ auto-gptq && \ git clone https://github.com/TimDettmers/bitsandbytes && cd bitsandbytes && \ - cmake -DCOMPUTE_BACKEND=cuda -DNO_CUBLASLT=OFF . && \ + cmake -B . -DCOMPUTE_BACKEND=cuda -S . && \ + cmake --build . && \ pip install -e . && \ pip freeze | grep bitsandbytes From aac412e3cfa61feb866105a860d8696c85f54879 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 6 Feb 2024 20:03:59 +0100 Subject: [PATCH 14/14] Update .github/workflows/build_docker_images.yml --- .github/workflows/build_docker_images.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml index 931d2434c9..a888a08934 100644 --- a/.github/workflows/build_docker_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -1,9 +1,10 @@ name: Build Docker images (scheduled) on: - push: - branches: - - younesbelkada-patch-2 + workflow_dispatch: + workflow_call: + schedule: + - cron: "0 1 * * *" concurrency: group: docker-image-builds