diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 61c895f7601..e9dda97f160 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -22,8 +22,19 @@ on: jobs: build_universal_wheel: name: Build Universal Wheel (macOS ARM64) - runs-on: macos-13-xlarge + runs-on: macos-14-xlarge steps: + - name: Free up disk space (Initial) + run: | + # Clean Homebrew cache + brew cleanup -s 2>/dev/null || true + rm -rf "$(brew --cache)" 2>/dev/null || true + sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true + sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true + sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true + sudo rm -rf /tmp/* 2>/dev/null || true + echo "=== Disk usage after cleanup ===" + df -h - name: Setup pyenv run: | curl https://pyenv.run | bash @@ -80,6 +91,7 @@ jobs: brew install openssl@3 || echo "OpenSSL install failed, continuing..." brew install --ignore-dependencies llvm@19 brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv + brew install ccache || echo "ccache installation failed, continuing without it" brew install go cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH @@ -102,6 +114,23 @@ jobs: - name: Update submodules run: | git submodule update --init --recursive --jobs 4 + - name: Free up disk space (Before compilation) + run: | + echo "=== Disk usage before compilation cleanup ===" + df -h + brew cleanup -s 2>/dev/null || true + rm -rf "$(brew --cache)" 2>/dev/null || true + rm -rf ~/.cache/pip 2>/dev/null || true + rm -rf ~/.pyenv/.cache 2>/dev/null || true + rm -rf ~/.cargo/registry/cache 2>/dev/null || true + echo "=== Disk usage after cleanup ===" + df -h + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ runner.os }}-arm64-ccache + max-size: 5G + append-timestamp: true - name: Run chdb/build.sh timeout-minutes: 600 run: | @@ -193,6 +222,12 @@ jobs: sudo sysctl kern.corefile=$PWD/tmp/core/core.%P sudo sysctl kern.coredump=1 ulimit -c unlimited + - name: Free up disk space + run: | + # Clean more build artifacts + rm -rf buildlib/contrib 2>/dev/null || true + rm -rf buildlib/base 2>/dev/null || true + rm -rf buildlib/src 2>/dev/null || true - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" @@ -200,9 +235,10 @@ jobs: for version in 3.8 3.9 3.10 3.11 3.12 3.13; do echo "Testing chdb on Python $version" pyenv shell $version - python -m pip install dist/*.whl --force-reinstall + python -m pip install dist/*.whl --force-reinstall --no-cache-dir python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" make test + python -m pip uninstall -y chdb pyenv shell --unset done continue-on-error: false diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 8217b1ad033..dfdaa752816 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -22,7 +22,8 @@ on: jobs: build_universal_wheel: name: Build Universal Wheel (macOS x86_64) - runs-on: macos-14-large + runs-on: macos-15-intel + timeout-minutes: 600 steps: - name: Setup pyenv run: | @@ -30,13 +31,12 @@ jobs: export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv install 3.8:latest pyenv install 3.9:latest pyenv install 3.10:latest pyenv install 3.11:latest pyenv install 3.12:latest pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + pyenv global 3.9 3.10 3.11 3.12 3.13 echo "Installed versions:" pyenv versions @@ -48,7 +48,7 @@ jobs: pyenv versions echo "" echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.9 3.10 3.11 3.12 3.13; do if ! pyenv versions --bare | grep -q "^$version"; then echo "ERROR: Python $version is not installed!" exit 1 @@ -60,7 +60,7 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.9 3.10 3.11 3.12 3.13; do echo "Installing dependencies for Python $version" pyenv shell $version python -m pip install --upgrade pip @@ -122,30 +122,39 @@ jobs: - name: Update submodules run: | git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ runner.os }}-x86_64-ccache + max-size: 10G + append-timestamp: true + env: + CCACHE_NOHASHDIR: "true" - name: Run chdb/build.sh timeout-minutes: 600 run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" source ~/.cargo/env - pyenv shell 3.8 + pyenv shell 3.9 export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@19)/bin/clang export CXX=$(brew --prefix llvm@19)/bin/clang++ bash gen_manifest.sh bash ./chdb/build.sh - pyenv shell 3.8 + pyenv shell 3.9 bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Run chdb/build/build_static_lib.sh timeout-minutes: 600 run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@19)/bin/clang export CXX=$(brew --prefix llvm@19)/bin/clang++ - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 bash ./chdb/build/build_static_lib.sh pyenv shell --unset continue-on-error: false @@ -201,13 +210,13 @@ jobs: export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@19)/bin/clang export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 + pyenv shell 3.9 make wheel - name: Fix wheel platform tag run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl - name: Setup core dump collection run: | @@ -215,11 +224,17 @@ jobs: sudo sysctl kern.corefile=$PWD/tmp/core/core.%P sudo sysctl kern.coredump=1 ulimit -c unlimited + - name: Free up disk space + run: | + # Clean more build artifacts + rm -rf buildlib/contrib 2>/dev/null || true + rm -rf buildlib/base 2>/dev/null || true + rm -rf buildlib/src 2>/dev/null || true - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.9 3.10 3.11 3.12 3.13; do echo "Testing chdb on Python $version" pyenv shell $version python -m pip install dist/*.whl --force-reinstall @@ -291,7 +306,7 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 python -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ diff --git a/chdb/build.sh b/chdb/build.sh index f31e904e964..da078bff541 100755 --- a/chdb/build.sh +++ b/chdb/build.sh @@ -30,7 +30,6 @@ if [ "$(uname)" == "Darwin" ]; then # if Darwin ARM64 (M1, M2), disable AVX if [ "$(uname -m)" == "arm64" ]; then CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0" - COMPILER_CACHE="-DCOMPILER_CACHE=disabled" LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0" else LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0" @@ -110,7 +109,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 ${CPU_FEATURES} \ -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \ -DENABLE_LIBFIU=1 \ - ${COMPILER_CACHE} \ -DCHDB_VERSION=${CHDB_VERSION} \ " @@ -239,6 +237,10 @@ ls -lh ${LIBCHDB} # build chdb python module py_version="3.8" +# check current os type and architecture for py_version +if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then + py_version="3.9" +fi current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") if [ "$current_py_version" != "$py_version" ]; then echo "Error: Current Python version is $current_py_version, but required version is $py_version" diff --git a/chdb/build/build_static_lib.sh b/chdb/build/build_static_lib.sh index dd5eb19c173..ca7b20432bf 100755 --- a/chdb/build/build_static_lib.sh +++ b/chdb/build/build_static_lib.sh @@ -25,9 +25,6 @@ if [ "$(uname)" == "Darwin" ]; then MYSQL="-DENABLE_MYSQL=0" ICU="-DENABLE_ICU=0" CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0" - if [ "$(uname -m)" == "arm64" ]; then - COMPILER_CACHE="-DCOMPILER_CACHE=disabled" - fi elif [ "$(uname)" == "Linux" ]; then GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1" UNWIND="-DUSE_UNWIND=1" @@ -69,7 +66,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 ${CPU_FEATURES} \ -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \ -DENABLE_LIBFIU=1 \ - ${COMPILER_CACHE} \ -DCHDB_VERSION=${CHDB_VERSION} \ " diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh index ccc6eaa7602..0ac65842f24 100755 --- a/chdb/build_pybind11.sh +++ b/chdb/build_pybind11.sh @@ -64,6 +64,11 @@ build_pybind11_nonlimitedapi() { build_all_pybind11_nonlimitedapi() { local python_versions=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13") + + # Skip Python 3.8 for macOS x86_64 + if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then + python_versions=("3.9" "3.10" "3.11" "3.12" "3.13") + fi echo "Building pybind11 nonlimitedapi libraries for all Python versions..."