diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bba99143c43..42def2f64a2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -271,11 +271,6 @@ jobs:
           echo "${LLVM_DIR}/bin" >> $GITHUB_PATH
         env:
           LLVM_DIR: .llvm
-      - name: Add `brew` libs to `RUSTFLAGS`
-        if: matrix.metadata.os == 'macos-14'
-        shell: bash
-        run: |
-          echo "RUSTFLAGS=-L/opt/homebrew/lib" >> $GITHUB_ENV
       - name: Configure LLVM (Windows)
         # The Custom Windows build does not contains llvm-config.exe, so need to setup manualy here
         if: startsWith(matrix.build, 'windows-x64') && matrix.llvm_url
@@ -286,6 +281,10 @@ jobs:
           echo LLVM_ENABLE=1 >> $GITHUB_ENV
         env:
           LLVM_DIR: .llvm
+      - name: Add `brew` libraries (Apple Silicon)
+        run: |
+          echo "RUSTFLAGS=-L/opt/homebrew/lib" >> $GITHUB_ENV
+        if: matrix.os == 'macos-14'
       - name: Set up dependencies for Mac OS
         run: |
           brew install automake