[ci] restore aarch64 linux builds (fixes #6509) (#6527)

.ci/test.sh

@@ -191,6 +191,8 @@ elif [[ $TASK == "bdist" ]]; then
             PLATFORM="manylinux2014_$ARCH"
         fi
         sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1
+        # rename wheel, to fix scikit-build-core choosing the platform 'linux_aarch64' instead of
+        # a manylinux tag
         mv \
             ./dist/*.whl \
             ./dist/tmp.whl || exit 1

.vsts-ci.yml

@@ -182,71 +182,71 @@ jobs:
     inputs:
       filePath: $(Build.SourcesDirectory)/.ci/test.sh
       targetType: 'filePath'
-# ###########################################
-# - job: QEMU_multiarch
-# ###########################################
-#   variables:
-#     BUILD_DIRECTORY: /LightGBM
-#     COMPILER: gcc
-#     PRODUCES_ARTIFACTS: 'true'
-#   pool:
-#     vmImage: ubuntu-22.04
-#   timeoutInMinutes: 180
-#   strategy:
-#     matrix:
-#       bdist:
-#         TASK: bdist
-#         ARCH: aarch64
-#   steps:
-#   - script: |
-#       sudo apt-get update
-#       sudo apt-get install --no-install-recommends -y \
-#         binfmt-support \
-#         qemu \
-#         qemu-user \
-#         qemu-user-static
-#     displayName: 'Install QEMU'
-#   - script: |
-#       docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-#     displayName: 'Enable Docker multi-architecture support'
-#   - script: |
-#       git clean -d -f -x
-#     displayName: 'Clean source directory'
-#   - script: |
-#       cat > docker-script.sh <<EOF
-#       export CONDA=\$HOME/miniforge
-#       export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
-#       export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
-#       \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
-#       \$BUILD_DIRECTORY/.ci/test.sh || exit 1
-#       EOF
-#       IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
-#       docker pull "${IMAGE_URI}" || exit 1
-#       PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
-#       echo "detected image platform: ${PLATFORM}"
-#       docker run \
-#         --platform "${PLATFORM}" \
-#         --rm \
-#         --env AZURE=true \
-#         --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
-#         --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
-#         --env COMPILER=$COMPILER \
-#         --env METHOD=$METHOD \
-#         --env OS_NAME=linux \
-#         --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
-#         --env PYTHON_VERSION=$PYTHON_VERSION \
-#         --env TASK=$TASK \
-#         -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
-#         -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
-#         "${IMAGE_URI}" \
-#         /bin/bash $BUILD_DIRECTORY/docker-script.sh
-#     displayName: 'Setup and run tests'
-#   - task: PublishBuildArtifacts@1
-#     condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
-#     inputs:
-#       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-#       artifactName: PackageAssets
-#       artifactType: container
+###########################################
+- job: QEMU_multiarch
+###########################################
+  variables:
+    BUILD_DIRECTORY: /LightGBM
+    COMPILER: gcc
+    PRODUCES_ARTIFACTS: 'true'
+  pool:
+    vmImage: ubuntu-22.04
+  timeoutInMinutes: 180
+  strategy:
+    matrix:
+      bdist:
+        TASK: bdist
+        ARCH: aarch64
+  steps:
+  - script: |
+      sudo apt-get update
+      sudo apt-get install --no-install-recommends -y \
+        binfmt-support \
+        qemu \
+        qemu-user \
+        qemu-user-static
+    displayName: 'Install QEMU'
+  - script: |
+      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    displayName: 'Enable Docker multi-architecture support'
+  - script: |
+      git clean -d -f -x
+    displayName: 'Clean source directory'
+  - script: |
+      cat > docker-script.sh <<EOF
+      export CONDA=\$HOME/miniforge
+      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
+      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
+      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
+      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
+      EOF
+      IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
+      docker pull "${IMAGE_URI}" || exit 1
+      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
+      echo "detected image platform: ${PLATFORM}"
+      docker run \
+        --platform "${PLATFORM}" \
+        --rm \
+        --env AZURE=true \
+        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
+        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
+        --env COMPILER=$COMPILER \
+        --env METHOD=$METHOD \
+        --env OS_NAME=linux \
+        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
+        --env PYTHON_VERSION=$PYTHON_VERSION \
+        --env TASK=$TASK \
+        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
+        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
+        "${IMAGE_URI}" \
+        /bin/bash $BUILD_DIRECTORY/docker-script.sh
+    displayName: 'Setup and run tests'
+  - task: PublishBuildArtifacts@1
+    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    inputs:
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      artifactName: PackageAssets
+      artifactType: container
 ###########################################
 - job: macOS
 ###########################################
@@ -376,7 +376,7 @@ jobs:
   dependsOn:
   - Linux
   - Linux_latest
-  # - QEMU_multiarch
+  - QEMU_multiarch
   - macOS
   - Windows
   - R_artifact

python-package/lightgbm/compat.py

@@ -3,75 +3,8 @@
 
 from typing import Any, List
 
-"""pandas"""
-try:
-    from pandas import DataFrame as pd_DataFrame
-    from pandas import Series as pd_Series
-    from pandas import concat
-
-    try:
-        from pandas import CategoricalDtype as pd_CategoricalDtype
-    except ImportError:
-        from pandas.api.types import CategoricalDtype as pd_CategoricalDtype
-    PANDAS_INSTALLED = True
-except ImportError:
-    PANDAS_INSTALLED = False
-
-    class pd_Series:  # type: ignore
-        """Dummy class for pandas.Series."""
-
-        def __init__(self, *args: Any, **kwargs: Any):
-            pass
-
-    class pd_DataFrame:  # type: ignore
-        """Dummy class for pandas.DataFrame."""
-
-        def __init__(self, *args: Any, **kwargs: Any):
-            pass
-
-    class pd_CategoricalDtype:  # type: ignore
-        """Dummy class for pandas.CategoricalDtype."""
-
-        def __init__(self, *args: Any, **kwargs: Any):
-            pass
-
-    concat = None
-
-"""matplotlib"""
-try:
-    import matplotlib  # noqa: F401
-
-    MATPLOTLIB_INSTALLED = True
-except ImportError:
-    MATPLOTLIB_INSTALLED = False
-
-"""graphviz"""
-try:
-    import graphviz  # noqa: F401
-
-    GRAPHVIZ_INSTALLED = True
-except ImportError:
-    GRAPHVIZ_INSTALLED = False
-
-"""datatable"""
-try:
-    import datatable
-
-    if hasattr(datatable, "Frame"):
-        dt_DataTable = datatable.Frame
-    else:
-        dt_DataTable = datatable.DataTable
-    DATATABLE_INSTALLED = True
-except ImportError:
-    DATATABLE_INSTALLED = False
-
-    class dt_DataTable:  # type: ignore
-        """Dummy class for datatable.DataTable."""
-
-        def __init__(self, *args: Any, **kwargs: Any):
-            pass
-
-
+# scikit-learn is intentionally imported first here,
+# see https://github.com/microsoft/LightGBM/issues/6509
 """sklearn"""
 try:
     from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
@@ -141,6 +74,75 @@ class _LGBMRegressorBase:  # type: ignore
     _LGBMCheckClassificationTargets = None
     _LGBMComputeSampleWeight = None
 
+"""pandas"""
+try:
+    from pandas import DataFrame as pd_DataFrame
+    from pandas import Series as pd_Series
+    from pandas import concat
+
+    try:
+        from pandas import CategoricalDtype as pd_CategoricalDtype
+    except ImportError:
+        from pandas.api.types import CategoricalDtype as pd_CategoricalDtype
+    PANDAS_INSTALLED = True
+except ImportError:
+    PANDAS_INSTALLED = False
+
+    class pd_Series:  # type: ignore
+        """Dummy class for pandas.Series."""
+
+        def __init__(self, *args: Any, **kwargs: Any):
+            pass
+
+    class pd_DataFrame:  # type: ignore
+        """Dummy class for pandas.DataFrame."""
+
+        def __init__(self, *args: Any, **kwargs: Any):
+            pass
+
+    class pd_CategoricalDtype:  # type: ignore
+        """Dummy class for pandas.CategoricalDtype."""
+
+        def __init__(self, *args: Any, **kwargs: Any):
+            pass
+
+    concat = None
+
+"""matplotlib"""
+try:
+    import matplotlib  # noqa: F401
+
+    MATPLOTLIB_INSTALLED = True
+except ImportError:
+    MATPLOTLIB_INSTALLED = False
+
+"""graphviz"""
+try:
+    import graphviz  # noqa: F401
+
+    GRAPHVIZ_INSTALLED = True
+except ImportError:
+    GRAPHVIZ_INSTALLED = False
+
+"""datatable"""
+try:
+    import datatable
+
+    if hasattr(datatable, "Frame"):
+        dt_DataTable = datatable.Frame
+    else:
+        dt_DataTable = datatable.DataTable
+    DATATABLE_INSTALLED = True
+except ImportError:
+    DATATABLE_INSTALLED = False
+
+    class dt_DataTable:  # type: ignore
+        """Dummy class for datatable.DataTable."""
+
+        def __init__(self, *args: Any, **kwargs: Any):
+            pass
+
+
 """dask"""
 try:
     from dask import delayed