From 63be785e66c4e7feca8c9c87734973c159d1eb1b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 20:33:54 -0600 Subject: [PATCH 01/21] [ci] make shell scripts stricter --- .ci/append_comment.sh | 2 +- .ci/check_python_dists.sh | 2 ++ .ci/lint-cpp.sh | 4 +++- .ci/lint-python.sh | 2 ++ .ci/rerun_workflow.sh | 2 +- .ci/set_commit_status.sh | 2 +- .ci/setup.sh | 2 ++ .ci/test-python-oldest.sh | 2 ++ .ci/test.sh | 2 ++ .ci/test_r_package.sh | 2 ++ .ci/test_r_package_valgrind.sh | 2 ++ .ci/trigger_dispatch_run.sh | 2 +- R-package/recreate-configure.sh | 2 ++ build-cran-package.sh | 2 +- build-python.sh | 2 +- docs/build-docs.sh | 2 ++ 16 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.ci/append_comment.sh b/.ci/append_comment.sh index 14e0111249cf..6d9fd0c0f54b 100755 --- a/.ci/append_comment.sh +++ b/.ci/append_comment.sh @@ -10,7 +10,7 @@ # # BODY: Text that will be appended to the original comment body. -set -e +set -e -u -x pipefail if [ -z "$GITHUB_ACTIONS" ]; then echo "Must be run inside GitHub Actions CI" diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index 1dd19679daae..f0b7e7be706b 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e -u -x pipefail + DIST_DIR=${1} echo "checking Python package distributions in '${DIST_DIR}'" diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 2d91f8e85f00..f2688d85aa00 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -e -u -x pipefail echo "running cpplint" cpplint \ diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index 887bc9fdebf1..82f83a6c22ea 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e -E -u -o pipefail + echo "running ruff" ruff check \ --config=./python-package/pyproject.toml \ diff --git a/.ci/rerun_workflow.sh b/.ci/rerun_workflow.sh index eb098adeeaeb..d0796c168245 100755 --- a/.ci/rerun_workflow.sh +++ b/.ci/rerun_workflow.sh @@ -12,7 +12,7 @@ # # PR_BRANCH: Name of pull request's branch. -set -e +set -e -E -u -o pipefail if [ -z "$GITHUB_ACTIONS" ]; then echo "Must be run inside GitHub Actions CI" diff --git a/.ci/set_commit_status.sh b/.ci/set_commit_status.sh index b10dd3ec27c3..745cb9e0ea6d 100755 --- a/.ci/set_commit_status.sh +++ b/.ci/set_commit_status.sh @@ -14,7 +14,7 @@ # # SHA: SHA of a commit to set a status on. -set -e +set -e -E -u -o pipefail if [ -z "$GITHUB_ACTIONS" ]; then echo "Must be run inside GitHub Actions CI" diff --git a/.ci/setup.sh b/.ci/setup.sh index bc17fee03308..cb08c3b437dd 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp diff --git a/.ci/test-python-oldest.sh b/.ci/test-python-oldest.sh index 40dfd393f1fe..ea9161f0ebcd 100644 --- a/.ci/test-python-oldest.sh +++ b/.ci/test-python-oldest.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + # oldest versions of dependencies published after # minimum supported Python version's first release # diff --git a/.ci/test.sh b/.ci/test.sh index 472fd7d8c6e5..e0aadad3d52d 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then export CXX=g++-11 export CC=gcc-11 diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 6d421f16be56..7f1fc1bb814e 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + # set up R environment CRAN_MIRROR="https://cran.rstudio.com" R_LIB_PATH=~/Rlib diff --git a/.ci/test_r_package_valgrind.sh b/.ci/test_r_package_valgrind.sh index bd8f1f71f9f1..64ae3a959da4 100755 --- a/.ci/test_r_package_valgrind.sh +++ b/.ci/test_r_package_valgrind.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + RDscriptvalgrind -e "install.packages(c('R6', 'data.table', 'jsonlite', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com')" || exit -1 sh build-cran-package.sh \ --r-executable=RDvalgrind \ diff --git a/.ci/trigger_dispatch_run.sh b/.ci/trigger_dispatch_run.sh index 040d6e9a581d..77722ada47cd 100755 --- a/.ci/trigger_dispatch_run.sh +++ b/.ci/trigger_dispatch_run.sh @@ -12,7 +12,7 @@ # # DISPATCH_NAME: Name of a dispatch to be triggered. -set -e +set -e -E -u -o pipefail if [ -z "$GITHUB_ACTIONS" ]; then echo "Must be run inside GitHub Actions CI" diff --git a/R-package/recreate-configure.sh b/R-package/recreate-configure.sh index df3586fd2af2..91f47995b6a7 100755 --- a/R-package/recreate-configure.sh +++ b/R-package/recreate-configure.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + # recreates 'configure' from 'configure.ac' # this script should run on Ubuntu 22.04 AUTOCONF_VERSION=$(cat R-package/AUTOCONF_UBUNTU_VERSION) diff --git a/build-cran-package.sh b/build-cran-package.sh index 9fa0c5877085..8ab4db15a90e 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -27,7 +27,7 @@ # # skip vignette building # sh build-cran-package.sh --no-build-vignettes -set -e +set -e -E -u -o pipefail # Default values of arguments BUILD_VIGNETTES=true diff --git a/build-python.sh b/build-python.sh index 8892ca2e936f..673908876950 100755 --- a/build-python.sh +++ b/build-python.sh @@ -62,7 +62,7 @@ # Install into user-specific instead of global site-packages directory. # Only used with 'install' command. -set -e -u +set -e -E -u -o pipefail echo "building lightgbm" diff --git a/docs/build-docs.sh b/docs/build-docs.sh index 689a30df9962..a88ddcbeaeb5 100644 --- a/docs/build-docs.sh +++ b/docs/build-docs.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e -E -u -o pipefail + rm -f ./_FIRST_RUN.flag export PATH="${CONDA}/bin:${PATH}" From 50a31f7e08d9768d1a70d28b4fd937136d308589 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 20:46:50 -0600 Subject: [PATCH 02/21] more fixes --- .ci/append_comment.sh | 2 +- .ci/check_python_dists.sh | 2 +- .ci/lint-cpp.sh | 2 +- .ci/test.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/append_comment.sh b/.ci/append_comment.sh index 6d9fd0c0f54b..21ed3d88d811 100755 --- a/.ci/append_comment.sh +++ b/.ci/append_comment.sh @@ -10,7 +10,7 @@ # # BODY: Text that will be appended to the original comment body. -set -e -u -x pipefail +set -e -E -u -o pipefail if [ -z "$GITHUB_ACTIONS" ]; then echo "Must be run inside GitHub Actions CI" diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index f0b7e7be706b..05fb33aa7c63 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -1,6 +1,6 @@ #!/bin/sh -set -e -u -x pipefail +set -e -E -u -o pipefail DIST_DIR=${1} diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index f2688d85aa00..70a8d72d5238 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e -u -x pipefail +set -e -E -u -o pipefail echo "running cpplint" cpplint \ diff --git a/.ci/test.sh b/.ci/test.sh index e0aadad3d52d..dd8ebe927c82 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e -E -u -o pipefail +set -e -E -o pipefail if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then export CXX=g++-11 From 6e01e25ceced7b03b565f8e801dc47c413185cbb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 21:29:17 -0600 Subject: [PATCH 03/21] set defaults --- .ci/setup.sh | 3 +++ .ci/test.sh | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index cb08c3b437dd..b0e91045b3a0 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -2,6 +2,9 @@ set -e -E -u -o pipefail +# defaults +SETUP_CONDA=${SETUP_CONDA:-"false"} + if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then brew install libomp diff --git a/.ci/test.sh b/.ci/test.sh index dd8ebe927c82..47030358efa2 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -1,6 +1,9 @@ #!/bin/bash -set -e -E -o pipefail +set -e -E -u -o pipefail + +# defaults +IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then export CXX=g++-11 From dab81482e6fb420a1c599bb2a4bc5ec91626d35f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 21:45:42 -0600 Subject: [PATCH 04/21] more defaults --- .ci/setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/setup.sh b/.ci/setup.sh index b0e91045b3a0..aaec7b1379f3 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -3,6 +3,7 @@ set -e -E -u -o pipefail # defaults +IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} SETUP_CONDA=${SETUP_CONDA:-"false"} if [[ $OS_NAME == "macos" ]]; then From 728d9c461ee0dd0825b510afafe003619b01fa0c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 22:47:44 -0600 Subject: [PATCH 05/21] change default --- .ci/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index aaec7b1379f3..a9daf8208070 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -4,7 +4,7 @@ set -e -E -u -o pipefail # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} -SETUP_CONDA=${SETUP_CONDA:-"false"} +SETUP_CONDA=${SETUP_CONDA:-"true"} if [[ $OS_NAME == "macos" ]]; then if [[ $COMPILER == "clang" ]]; then From 3f6456c813a1545adbf5434773a3ddf9bc324e80 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 10 Jan 2024 22:59:32 -0600 Subject: [PATCH 06/21] fix shebangs, more defaults --- .ci/check_python_dists.sh | 2 +- .ci/lint-python.sh | 2 +- .ci/test_r_package.sh | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index 05fb33aa7c63..de6e5b8ebaab 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e -E -u -o pipefail diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index 82f83a6c22ea..01437f4a0dc8 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e -E -u -o pipefail diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 7f1fc1bb814e..2953d2b04c20 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -2,6 +2,9 @@ set -e -E -u -o pipefail +# defaults +INSTALL_CMAKE_FROM_RELEASES=${INSTALL_CMAKE_FROM_RELEASES:-"false"} + # set up R environment CRAN_MIRROR="https://cran.rstudio.com" R_LIB_PATH=~/Rlib From fe6953c8151c83f936d31774728a72f733a7c662 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 11 Jan 2024 23:15:48 -0600 Subject: [PATCH 07/21] omit -o pipefail from build-cran-package and build-python --- build-cran-package.sh | 2 +- build-python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-cran-package.sh b/build-cran-package.sh index 8ab4db15a90e..538877d22053 100755 --- a/build-cran-package.sh +++ b/build-cran-package.sh @@ -27,7 +27,7 @@ # # skip vignette building # sh build-cran-package.sh --no-build-vignettes -set -e -E -u -o pipefail +set -e -E -u # Default values of arguments BUILD_VIGNETTES=true diff --git a/build-python.sh b/build-python.sh index 673908876950..824fcbb0180d 100755 --- a/build-python.sh +++ b/build-python.sh @@ -62,7 +62,7 @@ # Install into user-specific instead of global site-packages directory. # Only used with 'install' command. -set -e -E -u -o pipefail +set -e -E -u echo "building lightgbm" From b1cc27ad7d9453f9cd9e01eadcc297f5c3c18dfb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 12 Jan 2024 00:14:28 -0600 Subject: [PATCH 08/21] revert shebang changes --- .ci/check_python_dists.sh | 4 ++-- .ci/lint-cpp.sh | 2 +- .ci/lint-python.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index de6e5b8ebaab..270ee0a61e11 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/bin/sh -set -e -E -u -o pipefail +set -e -E -u DIST_DIR=${1} diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 70a8d72d5238..3945e3cabe8d 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e -E -u -o pipefail diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index 01437f4a0dc8..82f83a6c22ea 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e -E -u -o pipefail From 50b7a5de6f1b2caa92302a17c66855e8bb51a58f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 12 Jan 2024 20:03:53 -0600 Subject: [PATCH 09/21] more unbound variables --- .ci/lint-python.sh | 2 +- .ci/setup.sh | 1 + .ci/test.sh | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index 82f83a6c22ea..6839d22f9d22 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -1,6 +1,6 @@ #!/bin/sh -set -e -E -u -o pipefail +set -e -E -u echo "running ruff" ruff check \ diff --git a/.ci/setup.sh b/.ci/setup.sh index a9daf8208070..f7d78e1221ac 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -3,6 +3,7 @@ set -e -E -u -o pipefail # defaults +AZURE=${AZURE:-"false"} IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} SETUP_CONDA=${SETUP_CONDA:-"true"} diff --git a/.ci/test.sh b/.ci/test.sh index 47030358efa2..f79fd110151a 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -4,6 +4,9 @@ set -e -E -u -o pipefail # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} +METHOD=${METHOD:-""} +PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"} +SANITIZERS=${SANITIZERS:-"false"} if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then export CXX=g++-11 From 8efd6de1fbf6d62e52b7a164afa3f2ad0bf6a24f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 12 Jan 2024 21:08:48 -0600 Subject: [PATCH 10/21] more changes --- .ci/test_r_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 2953d2b04c20..de037d493a18 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e -E -u -o pipefail +set -e -E -u # defaults INSTALL_CMAKE_FROM_RELEASES=${INSTALL_CMAKE_FROM_RELEASES:-"false"} From bbc231989a8c5e91c372b23578a225838b0bd006 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 18 Feb 2024 21:11:36 -0600 Subject: [PATCH 11/21] unbound variables --- .ci/check_python_dists.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/check_python_dists.sh b/.ci/check_python_dists.sh index 2629d84709d8..59547722508b 100644 --- a/.ci/check_python_dists.sh +++ b/.ci/check_python_dists.sh @@ -4,6 +4,10 @@ set -e -E -u DIST_DIR=${1} +# defaults +METHOD=${METHOD:-""} +TASK=${TASK:-""} + echo "checking Python package distributions in '${DIST_DIR}'" pip install \ From 91872064cc76c1591afdff3448c94036bbe9ce8e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 18 Feb 2024 21:50:19 -0600 Subject: [PATCH 12/21] more fixes --- .ci/lint-cpp.sh | 2 +- .ci/lint-python.sh | 4 ++-- .ci/test.sh | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 9cc66376e8ba..9a2416ae9058 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e -E -u -o pipefail diff --git a/.ci/lint-python.sh b/.ci/lint-python.sh index 3ed88bcb3542..e1e9e306c883 100755 --- a/.ci/lint-python.sh +++ b/.ci/lint-python.sh @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/bash -set -e -E -u +set -e -E -u -o pipefail echo "running pre-commit checks" pre-commit run --all-files || exit 1 diff --git a/.ci/test.sh b/.ci/test.sh index 288c9bf89c4d..c9199aed0c59 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -6,7 +6,7 @@ set -e -E -u -o pipefail IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} METHOD=${METHOD:-""} PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"} -SANITIZERS=${SANITIZERS:-"false"} +SANITIZERS=${SANITIZERS:-""} if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "gcc" ]]; then export CXX=g++-11 @@ -87,11 +87,11 @@ if [[ $TASK == "lint" ]]; then 'r-lintr>=3.1' source activate $CONDA_ENV echo "Linting Python code" - sh ${BUILD_DIRECTORY}/.ci/lint-python.sh || exit 1 + bash ${BUILD_DIRECTORY}/.ci/lint-python.sh || exit 1 echo "Linting R code" Rscript ${BUILD_DIRECTORY}/.ci/lint_r_code.R ${BUILD_DIRECTORY} || exit 1 echo "Linting C++ code" - sh ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit 1 + bash ${BUILD_DIRECTORY}/.ci/lint-cpp.sh || exit 1 exit 0 fi From 13b32335a46937fb19d78a8648b49a0216ffb8e9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 18 Feb 2024 23:37:01 -0600 Subject: [PATCH 13/21] more fixes --- .ci/lint-cpp.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 9a2416ae9058..0e193e885cbe 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -34,9 +34,12 @@ get_omp_pragmas_without_num_threads() { 'pragma omp parallel' \ | grep -v ' num_threads' } +# https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html +set +o pipefail PROBLEMATIC_LINES=$( get_omp_pragmas_without_num_threads ) +set -o pipefail if test "${PROBLEMATIC_LINES}" != ""; then get_omp_pragmas_without_num_threads echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those." From 52be3c20691100d3daef1d5c6b037b1f291ebacc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Feb 2024 00:30:08 -0600 Subject: [PATCH 14/21] more changes --- .ci/lint-cpp.sh | 4 ++-- .ci/test_r_package.sh | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 0e193e885cbe..2464ab444d80 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -35,11 +35,11 @@ get_omp_pragmas_without_num_threads() { | grep -v ' num_threads' } # https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html -set +o pipefail +set +e +o pipefail PROBLEMATIC_LINES=$( get_omp_pragmas_without_num_threads ) -set -o pipefail +set -e -o pipefail if test "${PROBLEMATIC_LINES}" != ""; then get_omp_pragmas_without_num_threads echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those." diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 0e16c1769ab2..fd9e97e410e1 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -102,19 +102,6 @@ if [[ $OS_NAME == "macos" ]]; then -pkg $(pwd)/R.pkg \ -target / || exit 1 - # Older R versions (<= 4.1.2) on newer macOS (>= 11.0.0) cannot create the necessary symlinks. - # See https://github.com/r-lib/actions/issues/412. - if [[ $(sw_vers -productVersion | head -c2) -ge "11" ]]; then - sudo ln \ - -sf \ - /Library/Frameworks/R.framework/Resources/bin/R \ - /usr/local/bin/R - sudo ln \ - -sf \ - /Library/Frameworks/R.framework/Resources/bin/Rscript \ - /usr/local/bin/Rscript - fi - # Fix "duplicate libomp versions" issue on Mac # by replacing the R libomp.dylib with a symlink to the one installed with brew if [[ $COMPILER == "clang" ]]; then From 5051ef75600a3a22f449ff1ef50db255f9efd336 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 19 Feb 2024 19:58:20 -0600 Subject: [PATCH 15/21] remove more libompy.dylib workarounds --- .ci/test_r_package.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index fd9e97e410e1..a804ab22bd85 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -101,16 +101,6 @@ if [[ $OS_NAME == "macos" ]]; then sudo installer \ -pkg $(pwd)/R.pkg \ -target / || exit 1 - - # Fix "duplicate libomp versions" issue on Mac - # by replacing the R libomp.dylib with a symlink to the one installed with brew - if [[ $COMPILER == "clang" ]]; then - ver_arr=( ${R_MAC_VERSION//./ } ) - R_MAJOR_MINOR="${ver_arr[0]}.${ver_arr[1]}" - sudo ln -sf \ - "$(brew --cellar libomp)"/*/lib/libomp.dylib \ - /Library/Frameworks/R.framework/Versions/${R_MAJOR_MINOR}/Resources/lib/libomp.dylib - fi fi # fix for issue where CRAN was not returning {lattice} when using R 3.6 From 75ed073649df9ea7614e49b45824af619098f062 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 3 Mar 2024 17:47:30 -0600 Subject: [PATCH 16/21] debugging --- .ci/test_r_package.sh | 6 +++--- .github/workflows/r_package.yml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index a804ab22bd85..ed0e1cc9a8fe 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -209,9 +209,9 @@ echo "R CMD check build logs:" BUILD_LOG_FILE=lightgbm.Rcheck/00install.out cat ${BUILD_LOG_FILE} -if [[ $check_succeeded == "no" ]]; then - exit 1 -fi +# if [[ $check_succeeded == "no" ]]; then +# exit 1 +# fi used_correct_r_version=$( cat $LOG_FILE_NAME \ diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index db3e3c8a713b..a56699efff94 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -137,12 +137,12 @@ jobs: ################ # Other checks # ################ - - os: ubuntu-latest - task: r-rchk - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' + # - os: ubuntu-latest + # task: r-rchk + # compiler: gcc + # r_version: 4.3 + # build_type: cran + # container: 'ubuntu:22.04' steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') From 041b4035d0904a9db60c55bb791b0db5475650e8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 3 Mar 2024 20:50:35 -0600 Subject: [PATCH 17/21] fix exit codes --- .ci/test_r_package.sh | 9 ++++++--- .github/workflows/r_package.yml | 9 --------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index ed0e1cc9a8fe..2709d5d28b48 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -209,9 +209,12 @@ echo "R CMD check build logs:" BUILD_LOG_FILE=lightgbm.Rcheck/00install.out cat ${BUILD_LOG_FILE} -# if [[ $check_succeeded == "no" ]]; then -# exit 1 -# fi +if [[ $check_succeeded == "no" ]]; then + exit 1 +fi + +# ensure 'grep --count' doesn't cause failures +set +e used_correct_r_version=$( cat $LOG_FILE_NAME \ diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index a56699efff94..beca9e701a2f 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -134,15 +134,6 @@ jobs: r_version: 4.3 build_type: cran container: null - ################ - # Other checks # - ################ - # - os: ubuntu-latest - # task: r-rchk - # compiler: gcc - # r_version: 4.3 - # build_type: cran - # container: 'ubuntu:22.04' steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') From 96930b060315098c93ff490a11f7aada4fcfb33b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Apr 2024 20:33:01 -0500 Subject: [PATCH 18/21] merge master --- .ci/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/test.sh b/.ci/test.sh index e65cd4b617ff..9cbe55278767 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -2,6 +2,8 @@ set -e -E -u -o pipefail +git clean -d -f -x + # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} METHOD=${METHOD:-""} From 8a2e093dc5c40dca40a148abdb30134256cc3f2f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 19 Apr 2024 21:47:38 -0500 Subject: [PATCH 19/21] tweak scripts --- .ci/test.sh | 2 +- .ci/test_r_package.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index e65cd4b617ff..de3f19727b4b 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e -E -u -o pipefail +set -e -E -o -u pipefail # defaults IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 7d74ca3fad34..7cdc58d3477b 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e -E -u +set -e -E -u -o pipefail # defaults INSTALL_CMAKE_FROM_RELEASES=${INSTALL_CMAKE_FROM_RELEASES:-"false"} From e4fa87d2743ee91ccf8b89c43a54e212f6971183 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Apr 2024 17:18:44 -0500 Subject: [PATCH 20/21] clarify grep exit --- .ci/lint-cpp.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 2464ab444d80..46369b3aa9d2 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -34,7 +34,12 @@ get_omp_pragmas_without_num_threads() { 'pragma omp parallel' \ | grep -v ' num_threads' } -# https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html + +# 'grep' returns a non-0 exit code if any lines were found. +# Turning off '-e -o pipefail' options here so that bash doesn't +# consider this a failure and stop execution of the script. +# +# ref: https://www.gnu.org/software/grep/manual/html_node/Exit-Status.html set +e +o pipefail PROBLEMATIC_LINES=$( get_omp_pragmas_without_num_threads From 9d24415f0423fb4ab4e245bfa0a6575d8af3ad82 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Apr 2024 17:41:00 -0500 Subject: [PATCH 21/21] Update .ci/lint-cpp.sh --- .ci/lint-cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/lint-cpp.sh b/.ci/lint-cpp.sh index 46369b3aa9d2..dae0e1e93cd1 100755 --- a/.ci/lint-cpp.sh +++ b/.ci/lint-cpp.sh @@ -35,7 +35,7 @@ get_omp_pragmas_without_num_threads() { | grep -v ' num_threads' } -# 'grep' returns a non-0 exit code if any lines were found. +# 'grep' returns a non-0 exit code if 0 lines were found. # Turning off '-e -o pipefail' options here so that bash doesn't # consider this a failure and stop execution of the script. #