diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml
index 6e21f785ea..2bc476add5 100644
--- a/.github/actions/build_cmake/action.yml
+++ b/.github/actions/build_cmake/action.yml
@@ -20,27 +20,35 @@ runs:
       with:
         python-version: '3.11'
         miniconda-version: latest
-    - name: Initialize Conda environment
+    - name: Configure build environment
       shell: bash
       run: |
+        # initialize Conda
         conda config --set solver libmamba
         conda update -y -q conda
-    - name: Configure Conda environment
-      shell: bash
-      run: |
+        echo "$CONDA/bin" >> $GITHUB_PATH
+
+        # install base packages
         conda install -y -q -c conda-forge gxx_linux-64=11.2 sysroot_linux-64=2.28
         conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest
-        echo "$CONDA/bin" >> $GITHUB_PATH
-    - name: Install CUDA
-      if: inputs.gpu == 'ON' && inputs.raft == 'OFF'
-      shell: bash
-      run: |
-        conda install -y -q cuda-toolkit -c "nvidia/label/cuda-11.8.0"
-    - name: Install RAFT
-      if: inputs.raft == 'ON'
-      shell: bash
-      run: |
-        conda install -y -q libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
+
+        # install CUDA packages
+        if [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.raft }}" = "OFF" ]; then
+          conda install -y -q cuda-toolkit -c "nvidia/label/cuda-11.8.0"
+        fi
+
+        # install RAFT packages
+        if [ "${{ inputs.raft }}" = "ON" ]; then
+          conda install -y -q libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
+        fi
+
+        # install test packages
+        conda install -y pytest
+        if [ "${{ inputs.gpu }}" = "ON" ]; then
+          conda install -y -q pytorch pytorch-cuda=11.8 -c pytorch -c nvidia/label/cuda-11.8.0
+        else
+          conda install -y -q pytorch -c pytorch
+        fi
     - name: Build all targets
       shell: bash
       run: |
@@ -69,22 +77,16 @@ runs:
       working-directory: build/faiss/python
       run: |
         $CONDA/bin/python setup.py install
-    - name: Install pytest
-      shell: bash
-      run: |
-        conda install -y pytest
     - name: Python tests (CPU only)
       if: inputs.gpu == 'OFF'
       shell: bash
       run: |
-        conda install -y -q pytorch -c pytorch
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
         pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
     - name: Python tests (CPU + GPU)
       if: inputs.gpu == 'ON'
       shell: bash
       run: |
-        conda install -y -q pytorch pytorch-cuda=11.8 -c pytorch -c nvidia/label/cuda-11.8.0
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
         pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
         cp tests/common_faiss_tests.py faiss/gpu/test