diff --git a/.ci/setup.sh b/.ci/setup.sh
index ec9e3f114655..104648789bf8 100755
--- a/.ci/setup.sh
+++ b/.ci/setup.sh
@@ -17,7 +17,9 @@ if [[ $OS_NAME == "macos" ]]; then
             sudo xcode-select -s /Applications/Xcode_11.7.app/Contents/Developer || exit 1
         fi
     else  # gcc
-        sudo xcode-select -s /Applications/Xcode_14.1.app/Contents/Developer || exit 1
+        # Check https://github.com/actions/runner-images/tree/main/images/macos for available
+        # versions of Xcode
+        sudo xcode-select -s /Applications/Xcode_14.3.1.app/Contents/Developer || exit 1
         if [[ $TASK != "mpi" ]]; then
             brew install gcc
         fi
diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1
index 61ef1ae0f2a1..12f90f71cd05 100644
--- a/.ci/test_windows.ps1
+++ b/.ci/test_windows.ps1
@@ -9,6 +9,7 @@ function Check-Output {
 # unify environment variable for Azure DevOps and AppVeyor
 if (Test-Path env:APPVEYOR) {
   $env:APPVEYOR = "true"
+  $env:ALLOW_SKIP_ARROW_TESTS = "1"
 }
 
 if ($env:TASK -eq "r-package") {
diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml
index 27aecf82b419..8b2ed1dcc5c1 100644
--- a/.github/workflows/python_package.yml
+++ b/.github/workflows/python_package.yml
@@ -27,30 +27,32 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macOS-latest
+          - os: macos-13
             task: regular
             python_version: '3.9'
-          - os: macOS-latest
+          - os: macos-13
             task: sdist
             python_version: '3.10'
-          - os: macOS-latest
+          - os: macos-13
             task: bdist
             python_version: '3.7'
-          - os: macOS-latest
+          - os: macos-13
             task: if-else
             python_version: '3.9'
-          - os: macOS-latest
-            task: mpi
-            method: source
-            python_version: '3.10'
-          - os: macOS-latest
-            task: mpi
-            method: pip
-            python_version: '3.11'
-          - os: macOS-latest
-            task: mpi
-            method: wheel
-            python_version: '3.8'
+          # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425
+          # for further details.
+          # - os: macos-13
+          #   task: mpi
+          #   method: source
+          #   python_version: '3.10'
+          # - os: macos-13
+          #   task: mpi
+          #   method: pip
+          #   python_version: '3.11'
+          # - os: macos-13
+          #   task: mpi
+          #   method: wheel
+          #   python_version: '3.8'
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -63,7 +65,7 @@ jobs:
           export TASK="${{ matrix.task }}"
           export METHOD="${{ matrix.method }}"
           export PYTHON_VERSION="${{ matrix.python_version }}"
-          if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then
+          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
               export COMPILER="gcc"
               export OS_NAME="macos"
           elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml
index 3cae5af43908..c0346866deb9 100644
--- a/.github/workflows/r_package.yml
+++ b/.github/workflows/r_package.yml
@@ -63,13 +63,13 @@ jobs:
             r_version: 4.3
             build_type: cmake
             container: 'ubuntu:22.04'
-          - os: macOS-latest
+          - os: macos-13
             task: r-package
             compiler: gcc
             r_version: 4.3
             build_type: cmake
             container: null
-          - os: macOS-latest
+          - os: macos-13
             task: r-package
             compiler: clang
             r_version: 4.3
@@ -128,7 +128,7 @@ jobs:
             r_version: 4.3
             build_type: cran
             container: 'ubuntu:22.04'
-          - os: macOS-latest
+          - os: macos-13
             task: r-package
             compiler: clang
             r_version: 4.3
@@ -184,13 +184,13 @@ jobs:
           CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex
           TINYTEX_INSTALLER: TinyTeX
       - name: Setup and run tests on Linux and macOS
-        if: matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'macos-13' || matrix.os == 'ubuntu-latest'
         shell: bash
         run: |
           export TASK="${{ matrix.task }}"
           export COMPILER="${{ matrix.compiler }}"
           export GITHUB_ACTIONS="true"
-          if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then
+          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
               export OS_NAME="macos"
           elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
               export OS_NAME="linux"
diff --git a/tests/python_package_test/test_arrow.py b/tests/python_package_test/test_arrow.py
index 855300dda1ef..ab871048a949 100644
--- a/tests/python_package_test/test_arrow.py
+++ b/tests/python_package_test/test_arrow.py
@@ -1,16 +1,25 @@
 # coding: utf-8
 import filecmp
+import os
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 import numpy as np
-import pyarrow as pa
 import pytest
 
 import lightgbm as lgb
 
 from .utils import np_assert_array_equal
 
+# NOTE: In the AppVeyor CI, importing pyarrow fails due to an old Visual Studio version. Hence,
+#  we conditionally import pyarrow here (and skip tests if it cannot be imported). However, we
+#  don't want these tests to silently be skipped, hence, we only conditionally import when a
+#  specific env var is set.
+if os.getenv("ALLOW_SKIP_ARROW_TESTS") == "1":
+    pa = pytest.importorskip("pyarrow")
+else:
+    import pyarrow as pa  # type: ignore
+
 # ----------------------------------------------------------------------------------------------- #
 #                                            UTILITIES                                            #
 # ----------------------------------------------------------------------------------------------- #