diff --git a/.github/workflows/test-docker-build.yml b/.github/workflows/test-docker-build.yml
index b13465caa5..88bf8a35d4 100644
--- a/.github/workflows/test-docker-build.yml
+++ b/.github/workflows/test-docker-build.yml
@@ -29,9 +29,9 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         id: set-matrix
         env:
-          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+          CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}"
         run: |
-          echo "matrix=${ALL_CHANGED_FILES}" >> $GITHUB_OUTPUT
+          echo "matrix=$(echo ${CHANGED_FILES} | sed -e 's/\\\"/\"/g')" >> $GITHUB_OUTPUT
   build_modified_files:
     needs: get_changed_files
     name: Build Docker images on modified files
diff --git a/docker/peft-gpu/Dockerfile b/docker/peft-gpu/Dockerfile
index 994c647523..0cc9512bdc 100644
--- a/docker/peft-gpu/Dockerfile
+++ b/docker/peft-gpu/Dockerfile
@@ -39,7 +39,10 @@ RUN apt-get update && \
 RUN chsh -s /bin/bash
 SHELL ["/bin/bash", "-c"]
 
-RUN conda run -n peft pip install --no-cache-dir bitsandbytes optimum gptqmodel
+RUN conda run -n peft pip install --no-cache-dir bitsandbytes optimum
+
+# GPTQmodel doesn't find torch without build isolation
+RUN conda run -n peft pip install --no-build-isolation gptqmodel
 
 RUN \
     # Add eetq for quantization testing; needs to run without build isolation since the setup
diff --git a/pyproject.toml b/pyproject.toml
index 05e35c54c8..c1ed90e74f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,5 +52,7 @@ markers = [
 
 filterwarnings = [
     "error::DeprecationWarning:transformers",
+    # in sync with tests/conftest.py regarding BPE deprecation
+    "ignore:.*BPE.__init__ will not create from files anymore.*:DeprecationWarning:transformers",
     "error::FutureWarning:transformers",
 ]
diff --git a/setup.py b/setup.py
index bc70515aeb..168b4e4aed 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,7 @@
 ]
 extras["docs_specific"] = [
     "black",  # doc-builder has an implicit dependency on Black, see huggingface/doc-builder#434
+    "requests",  # doc-builder has an implicit dependency on requests (setup.py doesn't mention it, pyproject does)
     "hf-doc-builder",
 ]
 extras["dev"] = extras["quality"] + extras["docs_specific"]
diff --git a/src/peft/tuners/lora/gptq.py b/src/peft/tuners/lora/gptq.py
index 3101b4ff53..d210e7e45d 100644
--- a/src/peft/tuners/lora/gptq.py
+++ b/src/peft/tuners/lora/gptq.py
@@ -131,11 +131,5 @@ def dispatch_gptq(
         if isinstance(target_base_layer, BaseQuantLinear):
             new_module = GPTQLoraLinear(target, adapter_name, config=config, **kwargs)
             target.qweight = target_base_layer.qweight
-    else:
-        quant_linear = get_auto_gptq_quant_linear(cfg)
-
-        if quant_linear is not None and isinstance(target_base_layer, quant_linear):
-            new_module = GPTQLoraLinear(target, adapter_name, config=config, **kwargs)
-            target.qweight = target_base_layer.qweight
 
     return new_module