From 670fcad66ce4d613af14e198b2da1ba18395fb66 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 22:00:56 -0500 Subject: [PATCH 01/14] [ci] restore aarch64 linux builds (fixes #6509) --- .appveyor.yml | 42 ---- .ci/test.sh | 6 - .github/workflows/cuda.yml | 135 ------------ .github/workflows/python_package.yml | 64 +++++- .github/workflows/r_package.yml | 316 --------------------------- .vsts-ci.yml | 66 ------ 6 files changed, 52 insertions(+), 577 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/r_package.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index aae1c3a074fd..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: 4.4.0.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: - - '3.8' - -# only build on 'master' and pull requests targeting it -branches: - only: - - master - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - ps: | - $env:ALLOW_SKIP_ARROW_TESTS = "1" - $env:APPVEYOR = "true" - $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test.sh b/.ci/test.sh index 6e862ba0248f..7e1a9577728d 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -191,12 +191,6 @@ elif [[ $TASK == "bdist" ]]; then PLATFORM="manylinux2014_$ARCH" fi sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1 - mv \ - ./dist/*.whl \ - ./dist/tmp.whl || exit 1 - mv \ - ./dist/tmp.whl \ - ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1 sh .ci/check_python_dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 735bcef4a0d7..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - # Run manually by clicking a button in the UI - workflow_dispatch: - inputs: - restart_docker: - description: 'Restart nvidia-docker on the runner before building?' - required: true - type: boolean - default: false - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Optionally reinstall + restart docker on the runner before building. - # This is safe as long as only 1 of these jobs runs at a time. - restart-docker: - name: set up docker - runs-on: [self-hosted, linux] - timeout-minutes: 30 - steps: - - name: Setup or update software on host machine - if: ${{ inputs.restart_docker }} - run: | - # install core packages - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - lsb-release \ - software-properties-common - # set up nvidia-docker - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: mark job successful - run: | - exit 0 - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - needs: [restart-docker] - container: - image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} - env: - CMAKE_BUILD_PARALLEL_LEVEL: 4 - COMPILER: ${{ matrix.compiler }} - CONDA: /tmp/miniforge - DEBIAN_FRONTEND: noninteractive - METHOD: ${{ matrix.method }} - OS_NAME: linux - PYTHON_VERSION: ${{ matrix.python_version }} - TASK: ${{ matrix.task }} - SKBUILD_STRICT_CONFIG: true - options: --gpus all - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - method: wheel - compiler: gcc - python_version: "3.10" - cuda_version: "11.8.0" - linux_version: "ubuntu20.04" - task: cuda - - method: source - compiler: gcc - python_version: "3.12" - cuda_version: "12.2.0" - linux_version: "ubuntu22.04" - task: cuda - - method: pip - compiler: clang - python_version: "3.11" - cuda_version: "11.8.0" - linux_version: "ubuntu20.04" - task: cuda - steps: - - name: Install latest git - run: | - apt-get update - apt-get install --no-install-recommends -y \ - ca-certificates \ - software-properties-common - add-apt-repository ppa:git-core/ppa -y - apt-get update - apt-get install --no-install-recommends -y \ - git - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export PATH=$CONDA/bin:$PATH - - # check GPU usage - nvidia-smi - - # build and test - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - all-cuda-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index cd16696336c7..1e559cfe9342 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -18,6 +18,7 @@ env: SKBUILD_STRICT_CONFIG: true jobs: + # purpose: test different Python package variants (build options, compiler, Python version) test: name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os }}, Python ${{ matrix.python_version }}) runs-on: ${{ matrix.os }} @@ -38,10 +39,6 @@ jobs: - os: macos-13 task: if-else python_version: '3.9' - - os: macos-14 - task: bdist - method: wheel - python_version: '3.10' # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425 # for further details. # - os: macos-13 @@ -68,16 +65,56 @@ jobs: export TASK="${{ matrix.task }}" export METHOD="${{ matrix.method }}" export PYTHON_VERSION="${{ matrix.python_version }}" - if [[ "${{ matrix.os }}" == "macos-14" ]]; then + export COMPILER="gcc" + export OS_NAME="macos" + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export CONDA=${HOME}/miniforge + export PATH=${CONDA}/bin:${PATH} + $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 + $GITHUB_WORKSPACE/.ci/test.sh || exit 1 + # purpose: build macOS wheels to release + build-and-test-wheels: + name: ${{ matrix.task }} (${{ matrix.artifact-name }}, Python ${{ matrix.python_version }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - os: macos-14 + task: bdist + method: wheel + python_version: '3.10' + container: null + artifact-name: macosx-arm64-wheel + - os: macos-14 + task: bdist + method: wheel + python_version: '3.10' + container: "lightgbm/vsts-agent:manylinux2014_aarch64" + artifact-name: linux-aarch64-wheel + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + shell: bash + run: | + export TASK="${{ matrix.task }}" + export METHOD="${{ matrix.method }}" + export PYTHON_VERSION="${{ matrix.python_version }}" + if [[ "${{ matrix.artifact-name }}" == "macosx-arm64-wheel" ]]; then # use clang when creating macOS release artifacts export COMPILER="clang" export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "macos-13" ]]; then + elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then export COMPILER="gcc" - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export COMPILER="clang" export OS_NAME="linux" + else + echo "Unrecognized artifact name: '${{ matrix.artifact-name }}'" + exit 1 fi export BUILD_DIRECTORY="$GITHUB_WORKSPACE" export CONDA=${HOME}/miniforge @@ -85,11 +122,12 @@ jobs: $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - name: upload wheels - if: ${{ matrix.method == 'wheel' && matrix.os == 'macos-14' }} uses: actions/upload-artifact@v4 with: - name: macosx-arm64-wheel + name: ${{ matrix.artifact-name }} path: dist/*.whl + # purpose: test the Python package against nightlies of its dependencies, + # to catch issues before users experience them test-latest-versions: name: Python - latest versions (ubuntu-latest) runs-on: ubuntu-latest @@ -117,6 +155,8 @@ jobs: -w /opt/lgb-build \ python:3.11 \ /bin/bash ./.ci/test-python-latest.sh + # purpose: test the Python package against the oldest supported versions of its dependencies, + # to prevent changes from accidentally breaking compatibility with those older versions test-oldest-versions: name: Python - oldest supported versions (ubuntu-latest) runs-on: ubuntu-latest @@ -147,7 +187,7 @@ jobs: all-python-package-jobs-successful: if: always() runs-on: ubuntu-latest - needs: [test, test-latest-versions, test-oldest-versions] + needs: [test, build-and-test-wheels, test-latest-versions, test-oldest-versions] steps: - name: Note that all tests succeeded uses: re-actors/alls-green@v1.2.2 diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml deleted file mode 100644 index 3bcbf7ea4f79..000000000000 --- a/.github/workflows/r_package.yml +++ /dev/null @@ -1,316 +0,0 @@ -name: R-package - -on: - push: - branches: - - master - pull_request: - branches: - - master - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # https://github.com/actions/checkout/issues/1590#issuecomment-2207052044 - # - # this could be removed (hopefully) when R 3.6 support is removed - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - CMAKE_BUILD_PARALLEL_LEVEL: 4 - # hack to get around this: - # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html - _R_CHECK_SYSTEM_CLOCK_: 0 - # ignore R CMD CHECK NOTE checking how long it has - # been since the last submission - _R_CHECK_CRAN_INCOMING_REMOTE_: 0 - # CRAN ignores the "installed size is too large" NOTE, - # so our CI can too. Setting to a large value here just - # to catch extreme problems - _R_CHECK_PKG_SIZES_THRESHOLD_: 100 - -jobs: - test: - name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - ################ - # CMake builds # - ################ - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 3.6 - build_type: cmake - container: 'ubuntu:18.04' - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 3.6 - build_type: cmake - container: 'ubuntu:18.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: null - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cmake - container: null - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - container: null - # Visual Studio 2022 - - os: windows-2022 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.3 - build_type: cmake - container: null - ############### - # CRAN builds # - ############### - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cran - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cran - container: null - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - steps: - - name: Prevent conversion of line endings on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh - run: git config --global core.autocrlf false - - name: Install packages used by third-party actions - if: startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - apt-get update -y - apt-get install --no-install-recommends -y \ - ca-certificates \ - dirmngr \ - gpg \ - gpg-agent \ - software-properties-common \ - sudo - # install newest version of git - # ref: - # - https://unix.stackexchange.com/a/170831/550004 - # - https://git-scm.com/download/linux - add-apt-repository ppa:git-core/ppa -y - apt-get update -y - apt-get install --no-install-recommends -y \ - git - - name: Trust git cloning LightGBM - if: startsWith(matrix.os, 'ubuntu') - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - if: matrix.container != 'ubuntu:18.04' - # R 3.6 binary isn't easily available on buntu 18.04, - # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. - # ref: https://github.com/microsoft/LightGBM/issues/6298 - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2.6.0 - if: matrix.container == 'ubuntu:18.04' - - name: install tinytex - if: startsWith(matrix.os, 'windows') - uses: r-lib/actions/setup-tinytex@v2 - env: - CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex - TINYTEX_INSTALLER: TinyTeX - - name: Setup and run tests on Linux and macOS - if: matrix.os == 'macos-13' || matrix.os == 'ubuntu-latest' - shell: bash - run: | - export TASK="${{ matrix.task }}" - export COMPILER="${{ matrix.compiler }}" - if [[ "${{ matrix.os }}" == "macos-13" ]]; then - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export OS_NAME="linux" - export IN_UBUNTU_BASE_CONTAINER="true" - # the default version of cmake provided on Ubuntu 18.04 (v3.10.2), is not supported by LightGBM - # see https://github.com/microsoft/LightGBM/issues/5642 - if [[ "${{ matrix.container }}" == "ubuntu:18.04" ]]; then - export INSTALL_CMAKE_FROM_RELEASES="true" - fi - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export R_VERSION="${{ matrix.r_version }}" - export R_BUILD_TYPE="${{ matrix.build_type }}" - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - - name: Setup and run tests on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh -command ". {0}" - run: | - $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE - $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') - $env:TOOLCHAIN = "${{ matrix.toolchain }}" - $env:R_VERSION = "${{ matrix.r_version }}" - $env:R_BUILD_TYPE = "${{ matrix.build_type }}" - $env:COMPILER = "${{ matrix.compiler }}" - $env:TASK = "${{ matrix.task }}" - & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" - test-r-sanitizers: - name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: wch1/r-debug - strategy: - fail-fast: false - matrix: - include: - - r_customization: san - compiler: gcc - - r_customization: csan - compiler: clang - steps: - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} - RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - exit_code=0 - RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-debian-clang: - name: r-package (debian, R-devel, clang-${{ matrix.clang-version }}) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # list of versions tested in CRAN "Additional Checks": - # https://cran.r-project.org/web/checks/check_issue_kinds.html - clang-version: - - 16 - - 17 - runs-on: ubuntu-latest - container: rhub/debian-clang-devel - env: - DEBIAN_FRONTEND: noninteractive - steps: - - name: Install Git before checkout - shell: bash - run: | - apt-get update --allow-releaseinfo-change - apt-get install --no-install-recommends -y git - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: install clang - run: | - ./.ci/install-clang-devel.sh ${{ matrix.clang-version }} - - name: Install packages and run tests - shell: bash - run: | - export PATH=/opt/R-devel/bin/:${PATH} - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit 1 - echo "" - echo "install logs:" - echo "" - cat lightgbm.Rcheck/00install.out - echo "" - if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then - echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" - exit 1 - fi - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-debian-clang] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 4c0f80eb8c02..073a00af3f35 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -182,71 +182,6 @@ jobs: inputs: filePath: $(Build.SourcesDirectory)/.ci/test.sh targetType: 'filePath' -# ########################################### -# - job: QEMU_multiarch -# ########################################### -# variables: -# BUILD_DIRECTORY: /LightGBM -# COMPILER: gcc -# PRODUCES_ARTIFACTS: 'true' -# pool: -# vmImage: ubuntu-22.04 -# timeoutInMinutes: 180 -# strategy: -# matrix: -# bdist: -# TASK: bdist -# ARCH: aarch64 -# steps: -# - script: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# binfmt-support \ -# qemu \ -# qemu-user \ -# qemu-user-static -# displayName: 'Install QEMU' -# - script: | -# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# displayName: 'Enable Docker multi-architecture support' -# - script: | -# git clean -d -f -x -# displayName: 'Clean source directory' -# - script: | -# cat > docker-script.sh < Date: Sun, 7 Jul 2024 22:04:51 -0500 Subject: [PATCH 02/14] simplify configuration --- .github/workflows/python_package.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 1e559cfe9342..dd3c80c9f0d0 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -74,25 +74,20 @@ jobs: $GITHUB_WORKSPACE/.ci/test.sh || exit 1 # purpose: build macOS wheels to release build-and-test-wheels: - name: ${{ matrix.task }} (${{ matrix.artifact-name }}, Python ${{ matrix.python_version }}) + name: ${{ matrix.task }} (${{ matrix.artifact-name }}, Python 3.10) runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} timeout-minutes: 60 strategy: fail-fast: false matrix: include: - os: macos-14 - task: bdist - method: wheel - python_version: '3.10' - container: null artifact-name: macosx-arm64-wheel + container: null - os: macos-14 - task: bdist - method: wheel - python_version: '3.10' - container: "lightgbm/vsts-agent:manylinux2014_aarch64" artifact-name: linux-aarch64-wheel + container: 'lightgbm/vsts-agent:manylinux2014_aarch64' steps: - name: Checkout repository uses: actions/checkout@v4 @@ -102,9 +97,9 @@ jobs: - name: Setup and run tests shell: bash run: | - export TASK="${{ matrix.task }}" - export METHOD="${{ matrix.method }}" - export PYTHON_VERSION="${{ matrix.python_version }}" + export TASK="bdist" + export METHOD="wheel" + export PYTHON_VERSION="3.10" if [[ "${{ matrix.artifact-name }}" == "macosx-arm64-wheel" ]]; then # use clang when creating macOS release artifacts export COMPILER="clang" From f517e705f8cf16d9525f4a9269679a48ed904dd9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 22:55:47 -0500 Subject: [PATCH 03/14] try running in colima --- .github/workflows/python_package.yml | 37 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index dd3c80c9f0d0..55c7cc67322c 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -76,7 +76,11 @@ jobs: build-and-test-wheels: name: ${{ matrix.task }} (${{ matrix.artifact-name }}, Python 3.10) runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} + env: + CONDA: /tmp/miniforge + METHOD: bdist + PYTHON_VERSION: '3.10' + TASK: wheel timeout-minutes: 60 strategy: fail-fast: false @@ -84,7 +88,6 @@ jobs: include: - os: macos-14 artifact-name: macosx-arm64-wheel - container: null - os: macos-14 artifact-name: linux-aarch64-wheel container: 'lightgbm/vsts-agent:manylinux2014_aarch64' @@ -97,25 +100,35 @@ jobs: - name: Setup and run tests shell: bash run: | - export TASK="bdist" - export METHOD="wheel" - export PYTHON_VERSION="3.10" if [[ "${{ matrix.artifact-name }}" == "macosx-arm64-wheel" ]]; then # use clang when creating macOS release artifacts export COMPILER="clang" export OS_NAME="macos" + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export PATH=${CONDA}/bin:${PATH} + $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 + $GITHUB_WORKSPACE/.ci/test.sh || exit 1 elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then - export COMPILER="gcc" - export OS_NAME="linux" + # 'docker' did not come preinstalled on macOS runners as of when this was added + # ref: https://github.com/actions/runner/issues/1456 + brew install docker colima + colima start --cpu 2 + docker run \ + --rm \ + -v $(pwd):/opt/LightGBM \ + -w /opt/LightGBM \ + --env COMPILER=gcc \ + --env CONDA=${CONDA} \ + --env METHOD=${METHOD} \ + --env OS_NAME=linux \ + --env PYTHON_VERSION="${PYTHON_VERSION}" \ + --env TASK=${TASK} \ + -i lightgbm/vsts-agent:manylinux2014_aarch64 \ + bash -c 'PATH="${CONDA}/bin:${PATH}" .ci/setup.sh && PATH="${CONDA}/bin:${PATH}" .ci/test.sh' else echo "Unrecognized artifact name: '${{ matrix.artifact-name }}'" exit 1 fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export CONDA=${HOME}/miniforge - export PATH=${CONDA}/bin:${PATH} - $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 - $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - name: upload wheels uses: actions/upload-artifact@v4 with: From fffd65d3ed04f69a82d33a948449b841853d56f9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:04:46 -0500 Subject: [PATCH 04/14] do not add a homebrew installation of colima --- .github/workflows/python_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 55c7cc67322c..80fb0e39e422 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -111,7 +111,7 @@ jobs: elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 - brew install docker colima + brew install docker colima start --cpu 2 docker run \ --rm \ From c900e08295eb1da6a6879ced6bc58ba356d0dab6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:09:52 -0500 Subject: [PATCH 05/14] try without colima --- .github/workflows/python_package.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 80fb0e39e422..21b658d7c905 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -111,8 +111,7 @@ jobs: elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 - brew install docker - colima start --cpu 2 + brew install --cask docker docker run \ --rm \ -v $(pwd):/opt/LightGBM \ From 94069364c74c49df319f398097b3bc469646e174 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:15:08 -0500 Subject: [PATCH 06/14] start docker --- .github/workflows/python_package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 21b658d7c905..0a44fd7ff13c 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -112,6 +112,8 @@ jobs: # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 brew install --cask docker + open /Applications/Docker.app & + sleep 5 docker run \ --rm \ -v $(pwd):/opt/LightGBM \ From 9feacf945c03b5452195aa5dcf45f532fcc9f840 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:23:28 -0500 Subject: [PATCH 07/14] more testing --- .github/workflows/python_package.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 0a44fd7ff13c..d4114f49c745 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -27,15 +27,15 @@ jobs: fail-fast: false matrix: include: - - os: macos-13 - task: regular - python_version: '3.10' - - os: macos-13 - task: sdist - python_version: '3.11' - - os: macos-13 - task: bdist - python_version: '3.8' + # - os: macos-13 + # task: regular + # python_version: '3.10' + # - os: macos-13 + # task: sdist + # python_version: '3.11' + # - os: macos-13 + # task: bdist + # python_version: '3.8' - os: macos-13 task: if-else python_version: '3.9' @@ -112,8 +112,10 @@ jobs: # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 brew install --cask docker + echo "starting Docker" open /Applications/Docker.app & - sleep 5 + echo "done starting Docker" + sleep 20 docker run \ --rm \ -v $(pwd):/opt/LightGBM \ From 173c17a5d03f97d8f6d28d5180318840236f0343 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:34:26 -0500 Subject: [PATCH 08/14] try podman --- .github/workflows/python_package.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index d4114f49c745..ca19a7acd95f 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -78,9 +78,9 @@ jobs: runs-on: ${{ matrix.os }} env: CONDA: /tmp/miniforge - METHOD: bdist + METHOD: wheel PYTHON_VERSION: '3.10' - TASK: wheel + TASK: bdist timeout-minutes: 60 strategy: fail-fast: false @@ -111,12 +111,10 @@ jobs: elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 - brew install --cask docker - echo "starting Docker" - open /Applications/Docker.app & - echo "done starting Docker" - sleep 20 - docker run \ + brew install podman + podman machine start + podman machine init + podman run \ --rm \ -v $(pwd):/opt/LightGBM \ -w /opt/LightGBM \ From 02e47d653be472cb6f9b92e4f46e8ac4bfbda772 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:39:31 -0500 Subject: [PATCH 09/14] switch order --- .github/workflows/python_package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index ca19a7acd95f..3be68620df83 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -112,8 +112,8 @@ jobs: # 'docker' did not come preinstalled on macOS runners as of when this was added # ref: https://github.com/actions/runner/issues/1456 brew install podman - podman machine start podman machine init + podman machine start podman run \ --rm \ -v $(pwd):/opt/LightGBM \ From e00e7ffe9e17f02a2ae9ac5326d4fc0e41165e72 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Jul 2024 23:51:54 -0500 Subject: [PATCH 10/14] just restore QEMU-based build --- .github/workflows/python_package.yml | 103 ++++++-------------- .vsts-ci.yml | 66 +++++++++++++ python-package/lightgbm/compat.py | 140 ++++++++++++++------------- 3 files changed, 164 insertions(+), 145 deletions(-) diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index 3be68620df83..cd16696336c7 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -18,7 +18,6 @@ env: SKBUILD_STRICT_CONFIG: true jobs: - # purpose: test different Python package variants (build options, compiler, Python version) test: name: ${{ matrix.task }} ${{ matrix.method }} (${{ matrix.os }}, Python ${{ matrix.python_version }}) runs-on: ${{ matrix.os }} @@ -27,18 +26,22 @@ jobs: fail-fast: false matrix: include: - # - os: macos-13 - # task: regular - # python_version: '3.10' - # - os: macos-13 - # task: sdist - # python_version: '3.11' - # - os: macos-13 - # task: bdist - # python_version: '3.8' + - os: macos-13 + task: regular + python_version: '3.10' + - os: macos-13 + task: sdist + python_version: '3.11' + - os: macos-13 + task: bdist + python_version: '3.8' - os: macos-13 task: if-else python_version: '3.9' + - os: macos-14 + task: bdist + method: wheel + python_version: '3.10' # We're currently skipping MPI jobs on macOS, see https://github.com/microsoft/LightGBM/pull/6425 # for further details. # - os: macos-13 @@ -65,78 +68,28 @@ jobs: export TASK="${{ matrix.task }}" export METHOD="${{ matrix.method }}" export PYTHON_VERSION="${{ matrix.python_version }}" - export COMPILER="gcc" - export OS_NAME="macos" + if [[ "${{ matrix.os }}" == "macos-14" ]]; then + # use clang when creating macOS release artifacts + export COMPILER="clang" + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "macos-13" ]]; then + export COMPILER="gcc" + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export COMPILER="clang" + export OS_NAME="linux" + fi export BUILD_DIRECTORY="$GITHUB_WORKSPACE" export CONDA=${HOME}/miniforge export PATH=${CONDA}/bin:${PATH} $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - # purpose: build macOS wheels to release - build-and-test-wheels: - name: ${{ matrix.task }} (${{ matrix.artifact-name }}, Python 3.10) - runs-on: ${{ matrix.os }} - env: - CONDA: /tmp/miniforge - METHOD: wheel - PYTHON_VERSION: '3.10' - TASK: bdist - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - os: macos-14 - artifact-name: macosx-arm64-wheel - - os: macos-14 - artifact-name: linux-aarch64-wheel - container: 'lightgbm/vsts-agent:manylinux2014_aarch64' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - shell: bash - run: | - if [[ "${{ matrix.artifact-name }}" == "macosx-arm64-wheel" ]]; then - # use clang when creating macOS release artifacts - export COMPILER="clang" - export OS_NAME="macos" - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export PATH=${CONDA}/bin:${PATH} - $GITHUB_WORKSPACE/.ci/setup.sh || exit 1 - $GITHUB_WORKSPACE/.ci/test.sh || exit 1 - elif [[ "${{ matrix.artifact-name }}" == "linux-aarch64-wheel" ]]; then - # 'docker' did not come preinstalled on macOS runners as of when this was added - # ref: https://github.com/actions/runner/issues/1456 - brew install podman - podman machine init - podman machine start - podman run \ - --rm \ - -v $(pwd):/opt/LightGBM \ - -w /opt/LightGBM \ - --env COMPILER=gcc \ - --env CONDA=${CONDA} \ - --env METHOD=${METHOD} \ - --env OS_NAME=linux \ - --env PYTHON_VERSION="${PYTHON_VERSION}" \ - --env TASK=${TASK} \ - -i lightgbm/vsts-agent:manylinux2014_aarch64 \ - bash -c 'PATH="${CONDA}/bin:${PATH}" .ci/setup.sh && PATH="${CONDA}/bin:${PATH}" .ci/test.sh' - else - echo "Unrecognized artifact name: '${{ matrix.artifact-name }}'" - exit 1 - fi - name: upload wheels + if: ${{ matrix.method == 'wheel' && matrix.os == 'macos-14' }} uses: actions/upload-artifact@v4 with: - name: ${{ matrix.artifact-name }} + name: macosx-arm64-wheel path: dist/*.whl - # purpose: test the Python package against nightlies of its dependencies, - # to catch issues before users experience them test-latest-versions: name: Python - latest versions (ubuntu-latest) runs-on: ubuntu-latest @@ -164,8 +117,6 @@ jobs: -w /opt/lgb-build \ python:3.11 \ /bin/bash ./.ci/test-python-latest.sh - # purpose: test the Python package against the oldest supported versions of its dependencies, - # to prevent changes from accidentally breaking compatibility with those older versions test-oldest-versions: name: Python - oldest supported versions (ubuntu-latest) runs-on: ubuntu-latest @@ -196,7 +147,7 @@ jobs: all-python-package-jobs-successful: if: always() runs-on: ubuntu-latest - needs: [test, build-and-test-wheels, test-latest-versions, test-oldest-versions] + needs: [test, test-latest-versions, test-oldest-versions] steps: - name: Note that all tests succeeded uses: re-actors/alls-green@v1.2.2 diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 073a00af3f35..56dda0359435 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -183,6 +183,71 @@ jobs: filePath: $(Build.SourcesDirectory)/.ci/test.sh targetType: 'filePath' ########################################### +- job: QEMU_multiarch +########################################### + variables: + BUILD_DIRECTORY: /LightGBM + COMPILER: gcc + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + cat > docker-script.sh < Date: Sun, 7 Jul 2024 23:52:43 -0500 Subject: [PATCH 11/14] add more CI back --- .appveyor.yml | 42 +++++ .github/workflows/cuda.yml | 135 ++++++++++++++ .github/workflows/r_package.yml | 316 ++++++++++++++++++++++++++++++++ 3 files changed, 493 insertions(+) create mode 100644 .appveyor.yml create mode 100644 .github/workflows/cuda.yml create mode 100644 .github/workflows/r_package.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..aae1c3a074fd --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,42 @@ +version: 4.4.0.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: + - '3.8' + +# only build on 'master' and pull requests targeting it +branches: + only: + - master + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - ps: | + $env:ALLOW_SKIP_ARROW_TESTS = "1" + $env:APPVEYOR = "true" + $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 + $env:MINICONDA = "C:\Miniconda3-x64" + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + +build: false + +test_script: + - conda config --remove channels defaults + - conda config --add channels nodefaults + - conda config --add channels conda-forge + - conda config --set channel_priority strict + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..735bcef4a0d7 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,135 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + # Run manually by clicking a button in the UI + workflow_dispatch: + inputs: + restart_docker: + description: 'Restart nvidia-docker on the runner before building?' + required: true + type: boolean + default: false + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Optionally reinstall + restart docker on the runner before building. + # This is safe as long as only 1 of these jobs runs at a time. + restart-docker: + name: set up docker + runs-on: [self-hosted, linux] + timeout-minutes: 30 + steps: + - name: Setup or update software on host machine + if: ${{ inputs.restart_docker }} + run: | + # install core packages + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + lsb-release \ + software-properties-common + # set up nvidia-docker + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: mark job successful + run: | + exit 0 + test: + name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + needs: [restart-docker] + container: + image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 + COMPILER: ${{ matrix.compiler }} + CONDA: /tmp/miniforge + DEBIAN_FRONTEND: noninteractive + METHOD: ${{ matrix.method }} + OS_NAME: linux + PYTHON_VERSION: ${{ matrix.python_version }} + TASK: ${{ matrix.task }} + SKBUILD_STRICT_CONFIG: true + options: --gpus all + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - method: wheel + compiler: gcc + python_version: "3.10" + cuda_version: "11.8.0" + linux_version: "ubuntu20.04" + task: cuda + - method: source + compiler: gcc + python_version: "3.12" + cuda_version: "12.2.0" + linux_version: "ubuntu22.04" + task: cuda + - method: pip + compiler: clang + python_version: "3.11" + cuda_version: "11.8.0" + linux_version: "ubuntu20.04" + task: cuda + steps: + - name: Install latest git + run: | + apt-get update + apt-get install --no-install-recommends -y \ + ca-certificates \ + software-properties-common + add-apt-repository ppa:git-core/ppa -y + apt-get update + apt-get install --no-install-recommends -y \ + git + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export PATH=$CONDA/bin:$PATH + + # check GPU usage + nvidia-smi + + # build and test + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + all-cuda-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml new file mode 100644 index 000000000000..3bcbf7ea4f79 --- /dev/null +++ b/.github/workflows/r_package.yml @@ -0,0 +1,316 @@ +name: R-package + +on: + push: + branches: + - master + pull_request: + branches: + - master + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # https://github.com/actions/checkout/issues/1590#issuecomment-2207052044 + # + # this could be removed (hopefully) when R 3.6 support is removed + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + CMAKE_BUILD_PARALLEL_LEVEL: 4 + # hack to get around this: + # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html + _R_CHECK_SYSTEM_CLOCK_: 0 + # ignore R CMD CHECK NOTE checking how long it has + # been since the last submission + _R_CHECK_CRAN_INCOMING_REMOTE_: 0 + # CRAN ignores the "installed size is too large" NOTE, + # so our CI can too. Setting to a large value here just + # to catch extreme problems + _R_CHECK_PKG_SIZES_THRESHOLD_: 100 + +jobs: + test: + name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + ################ + # CMake builds # + ################ + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 3.6 + build_type: cmake + container: 'ubuntu:18.04' + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 3.6 + build_type: cmake + container: 'ubuntu:18.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: null + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cmake + container: null + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + container: null + # Visual Studio 2022 + - os: windows-2022 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.3 + build_type: cmake + container: null + ############### + # CRAN builds # + ############### + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cran + container: null + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cran + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null + steps: + - name: Prevent conversion of line endings on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh + run: git config --global core.autocrlf false + - name: Install packages used by third-party actions + if: startsWith(matrix.os, 'ubuntu') + shell: bash + run: | + apt-get update -y + apt-get install --no-install-recommends -y \ + ca-certificates \ + dirmngr \ + gpg \ + gpg-agent \ + software-properties-common \ + sudo + # install newest version of git + # ref: + # - https://unix.stackexchange.com/a/170831/550004 + # - https://git-scm.com/download/linux + add-apt-repository ppa:git-core/ppa -y + apt-get update -y + apt-get install --no-install-recommends -y \ + git + - name: Trust git cloning LightGBM + if: startsWith(matrix.os, 'ubuntu') + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 5 + submodules: true + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2 + if: matrix.container != 'ubuntu:18.04' + # R 3.6 binary isn't easily available on buntu 18.04, + # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. + # ref: https://github.com/microsoft/LightGBM/issues/6298 + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2.6.0 + if: matrix.container == 'ubuntu:18.04' + - name: install tinytex + if: startsWith(matrix.os, 'windows') + uses: r-lib/actions/setup-tinytex@v2 + env: + CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex + TINYTEX_INSTALLER: TinyTeX + - name: Setup and run tests on Linux and macOS + if: matrix.os == 'macos-13' || matrix.os == 'ubuntu-latest' + shell: bash + run: | + export TASK="${{ matrix.task }}" + export COMPILER="${{ matrix.compiler }}" + if [[ "${{ matrix.os }}" == "macos-13" ]]; then + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export OS_NAME="linux" + export IN_UBUNTU_BASE_CONTAINER="true" + # the default version of cmake provided on Ubuntu 18.04 (v3.10.2), is not supported by LightGBM + # see https://github.com/microsoft/LightGBM/issues/5642 + if [[ "${{ matrix.container }}" == "ubuntu:18.04" ]]; then + export INSTALL_CMAKE_FROM_RELEASES="true" + fi + fi + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export R_VERSION="${{ matrix.r_version }}" + export R_BUILD_TYPE="${{ matrix.build_type }}" + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + - name: Setup and run tests on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh -command ". {0}" + run: | + $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE + $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') + $env:TOOLCHAIN = "${{ matrix.toolchain }}" + $env:R_VERSION = "${{ matrix.r_version }}" + $env:R_BUILD_TYPE = "${{ matrix.build_type }}" + $env:COMPILER = "${{ matrix.compiler }}" + $env:TASK = "${{ matrix.task }}" + & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" + test-r-sanitizers: + name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: wch1/r-debug + strategy: + fail-fast: false + matrix: + include: + - r_customization: san + compiler: gcc + - r_customization: csan + compiler: clang + steps: + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} + RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + exit_code=0 + RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 + cat ./tests.log + exit ${exit_code} + test-r-debian-clang: + name: r-package (debian, R-devel, clang-${{ matrix.clang-version }}) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # list of versions tested in CRAN "Additional Checks": + # https://cran.r-project.org/web/checks/check_issue_kinds.html + clang-version: + - 16 + - 17 + runs-on: ubuntu-latest + container: rhub/debian-clang-devel + env: + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install Git before checkout + shell: bash + run: | + apt-get update --allow-releaseinfo-change + apt-get install --no-install-recommends -y git + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: install clang + run: | + ./.ci/install-clang-devel.sh ${{ matrix.clang-version }} + - name: Install packages and run tests + shell: bash + run: | + export PATH=/opt/R-devel/bin/:${PATH} + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + R CMD check --as-cran --run-donttest lightgbm_*.tar.gz || exit 1 + echo "" + echo "install logs:" + echo "" + cat lightgbm.Rcheck/00install.out + echo "" + if grep -q -E "NOTE|WARNING|ERROR" lightgbm.Rcheck/00check.log; then + echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check" + exit 1 + fi + all-r-package-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test, test-r-sanitizers, test-r-debian-clang] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} From e86ff05d7ee3800c7425b81eb03f71165abd1d12 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 11:16:46 -0500 Subject: [PATCH 12/14] put back tag updating --- .ci/test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.ci/test.sh b/.ci/test.sh index 7e1a9577728d..4247854fe8c9 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -191,6 +191,14 @@ elif [[ $TASK == "bdist" ]]; then PLATFORM="manylinux2014_$ARCH" fi sh ./build-python.sh bdist_wheel --integrated-opencl || exit 1 + # rename wheel, to fix scikit-build-core choosing the platform 'linux_aarch64' instead of + # a manylinux tag + mv \ + ./dist/*.whl \ + ./dist/tmp.whl || exit 1 + mv \ + ./dist/tmp.whl \ + ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1 sh .ci/check_python_dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 From 5734b9fa14ef7a1f1a101d7c099387dd8f8750d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 15:23:50 -0500 Subject: [PATCH 13/14] try to re-trigger appveyor From 0e2df3620459ab89ffee8c0b089ff4517cfb3dd4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jul 2024 00:22:31 -0700 Subject: [PATCH 14/14] uncomment missed line requiring QEMU_multiarch for releases --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 56dda0359435..169b04fffe13 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -376,7 +376,7 @@ jobs: dependsOn: - Linux - Linux_latest - # - QEMU_multiarch + - QEMU_multiarch - macOS - Windows - R_artifact