Use new test instance

.github/scripts/build-musllinux-arm64.sh

@@ -0,0 +1,281 @@
+#!/bin/sh
+set -e
+
+cd /workspace
+
+# Configure git safe directory in container
+apk update
+apk add --no-cache git python3 py3-pip py3-setuptools
+echo "=== Configure git safe directory ==="
+git config --global --add safe.directory /workspace
+git describe --tags
+python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())"
+
+# 1. Check system info
+echo "=== Container System Info ==="
+echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)"
+if [ -f /lib/ld-musl-aarch64.so.1 ]; then
+    echo "musl libc aarch64"
+elif [ -f /lib/libc.musl-aarch64.so.1 ]; then
+    echo "musl libc aarch64"
+else
+    echo "Not musl libc"
+fi
+echo "Workspace mounted at: /workspace"
+ls -la /workspace
+
+# 2. Install build dependencies
+echo "=== Installing build dependencies ==="
+apk add --no-cache make build-base openssl-dev zlib-dev \
+    bzip2-dev readline-dev sqlite-dev wget curl llvm \
+    ncurses-dev xz-dev tk-dev libxml2-dev \
+    libffi-dev linux-headers
+apk add --no-cache make cmake ccache ninja yasm gawk
+apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20
+
+# 3. Scan SQLite vulnerabilities
+echo "=== Scanning SQLite vulnerabilities ==="
+# Install grype
+curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+grype db update
+
+# Check SQLite vulnerabilities
+echo "Scanning SQLite packages for vulnerabilities..."
+GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true)
+echo "Raw grype output:"
+echo "$GRYPE_RAW_OUTPUT"
+
+SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
+if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
+    echo "SQLite vulnerabilities found in packages! Build should be reviewed."
+    echo "SQLite vulnerability details:"
+    echo "$SQLITE_SCAN_OUTPUT"
+else
+    echo "No SQLite vulnerabilities found"
+fi
+
+# 4. Setup Python environments
+echo "=== Setting up Python environments ==="
+# Setup pyenv
+curl https://pyenv.run | bash
+export PATH="$HOME/.pyenv/bin:$PATH"
+eval "$(pyenv init -)"
+
+# Install Python versions
+for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
+    echo "Installing Python $version"
+    pyenv install $version:latest
+done
+pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14
+
+# Verify installations
+echo "Installed versions:"
+pyenv versions
+for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
+    if ! pyenv versions --bare | grep -q "^$version"; then
+    echo "ERROR: Python $version is not installed!"
+    exit 1
+    fi
+    echo "Python $version is installed"
+done
+echo "All Python versions verified successfully!"
+
+# Install Rust
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+source $HOME/.cargo/env
+rustup toolchain install nightly-2025-07-07
+rustup component add --toolchain nightly-2025-07-07 rust-src
+
+# Install Python dependencies
+for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do
+    echo "Installing dependencies for Python $version"
+    pyenv shell $version
+    python -m pip install --upgrade pip
+    if [ "$version" = "3.8" ]; then
+    python -m pip install setuptools tox twine psutil wheel
+    else
+    python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel
+    fi
+    pyenv shell --unset
+done
+
+# Update version for release (if triggered by tag)
+if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then
+    pyenv shell 3.9
+
+    # Install bump-my-version
+    python -m pip install bump-my-version
+    TAG_NAME=${GITHUB_REF#refs/tags/v}
+    bump-my-version replace --new-version $TAG_NAME
+    echo "Version files updated to $TAG_NAME"
+    pyenv shell --unset
+fi
+
+# 5. Build chdb
+echo "=== Building chdb ==="
+echo "Timestamp: $(date)"
+echo "Current directory: $(pwd)"
+echo "Available disk space: $(df -h .)"
+
+# Setup clang
+echo "Setting up clang compiler..."
+ln -sf /usr/bin/clang-20 /usr/bin/clang
+ln -sf /usr/bin/clang++-20 /usr/bin/clang++
+export CC=/usr/bin/clang
+export CXX=/usr/bin/clang++
+echo "Compiler versions:"
+$CC --version
+$CXX --version
+
+# Build
+echo "Starting chdb build with Python 3.8..."
+pyenv shell 3.8
+python --version
+echo "Build start time: $(date)"
+bash ./chdb/build-musl.sh
+echo "Build end time: $(date)"
+
+# Test
+echo "Running smoke test with Python 3.9..."
+pyenv shell 3.9
+python --version
+echo "Test start time: $(date)"
+bash -x ./chdb/test_smoke.sh
+echo "Test end time: $(date)"
+
+# Check build results
+echo "Build results summary:"
+ccache -s
+echo "chdb directory contents:"
+ls -lh chdb
+echo "Build artifacts size:"
+du -sh chdb
+
+# 6. Create and audit wheels
+echo "=== Creating and auditing wheels ==="
+echo "Wheel creation start time: $(date)"
+echo "Available disk space before wheel build: $(df -h .)"
+
+# Build wheels
+echo "Building wheels with Python 3.8..."
+pyenv shell 3.8
+python --version
+echo "Running make wheel..."
+make wheel
+echo "Wheel build completed at: $(date)"
+echo "Initial wheel files:"
+ls -lh dist/ || echo "No dist directory yet"
+
+# Install patchelf
+echo "Installing patchelf for wheel auditing..."
+wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
+tar -xvf patchelf.tar.gz
+cp bin/patchelf /usr/bin/
+chmod +x /usr/bin/patchelf
+echo "patchelf version: $(patchelf --version)"
+
+# Audit wheels
+echo "Auditing wheels with Python 3.13..."
+pyenv shell 3.13
+python --version
+python -m pip install auditwheel
+echo "auditwheel version: $(auditwheel --version)"
+echo "Starting wheel audit at: $(date)"
+auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl
+echo "Wheel audit completed at: $(date)"
+
+# Clean up non-musllinux wheels
+echo "Cleaning up non-musllinux wheels..."
+echo "Before cleanup:"
+ls -lh dist/
+rm -f dist/*-linux_aarch64.whl
+echo "After cleanup:"
+ls -lh dist/
+echo "Final wheel sizes:"
+du -sh dist/*
+
+# 7. Test wheels
+echo "=== Testing wheels ==="
+echo "Wheel testing start time: $(date)"
+echo "Available wheels for testing:"
+ls -lh dist/*.whl
+echo "Wheel file details:"
+file dist/*.whl
+
+TOTAL_TESTS=6
+CURRENT_TEST=0
+TEST_FAILED=false
+
+for version in 3.9 3.10 3.11 3.12 3.13 3.14; do
+    CURRENT_TEST=$((CURRENT_TEST + 1))
+    echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ==="
+    echo "Test start time: $(date)"
+
+    echo "Switching to Python $version..."
+    pyenv shell $version
+    python --version
+    echo "pip version: $(python -m pip --version)"
+
+    echo "Installing chdb wheel..."
+    python -m pip install dist/*.whl --force-reinstall
+    echo "Installation completed at: $(date)"
+
+    echo "Running basic query test..."
+    python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')"
+
+    echo "Running full test suite..."
+    if make test; then
+    echo "Test suite PASSED for Python $version at: $(date)"
+    else
+    echo "Test suite FAILED for Python $version at: $(date)"
+    TEST_FAILED=true
+    break
+    fi
+
+    pyenv shell --unset
+    echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully"
+    echo ""
+done
+
+echo "All wheel tests completed at: $(date)"
+
+# Check if any tests failed
+if [ "$TEST_FAILED" = true ]; then
+    echo "ERROR: One or more test suites failed!"
+    echo "Test failure detected - aborting build process"
+    exit 1
+fi
+
+# Create test success marker file only if all tests passed
+echo "All tests passed successfully!"
+echo "Creating test success marker..."
+touch /workspace/.test_success_marker
+echo "Test success marker created at: $(date)"
+
+# 8. Scan chdb libraries
+echo "=== Scanning chdb libraries ==="
+FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
+SQLITE_VULNERABILITIES_FOUND=false
+
+for file in $FILES_TO_SCAN; do
+    if [ -f "$file" ]; then
+    echo "=== Scanning $file ==="
+    SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+    echo "$SCAN_OUTPUT"
+
+    if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
+        echo "SQLite vulnerability found in $file"
+        SQLITE_VULNERABILITIES_FOUND=true
+    fi
+    fi
+done
+
+if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
+    echo "SQLite vulnerabilities detected in chdb libraries!"
+else
+    echo "No SQLite vulnerabilities found in chdb libraries"
+fi
+
+# Show final results
+echo "=== Final wheel files ==="
+ls -la ./dist/

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (Linux ARM64)
-    runs-on: GH-Linux-ARM64
+    runs-on: [self-hosted, linux, arm64, ubuntu-latest]
     if: ${{ !github.event.pull_request.draft }}
     steps:
       - name: Install Python build dependencies
@@ -33,11 +33,13 @@ jobs:
           sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
             libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
             libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
-            libffi-dev liblzma-dev
+            libffi-dev liblzma-dev golang-go
       - name: Scan SQLite vulnerabilities with grype
         run: |
           # Install grype and required tools
-          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+          mkdir -p $HOME/.local/bin
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $HOME/.local/bin
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
           sudo apt-get update && sudo apt-get install -y jq lsb-release
 
           # Detect OS distribution info
@@ -46,11 +48,11 @@ jobs:
           echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
 
           # Update grype vulnerability database
-          grype db update
+          $HOME/.local/bin/grype db update
 
           # Check SQLite vulnerabilities in installed packages
           echo "Scanning SQLite packages for vulnerabilities..."
-          GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
+          GRYPE_RAW_OUTPUT=$($HOME/.local/bin/grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
           echo "Raw grype output:"
           echo "$GRYPE_RAW_OUTPUT"
 
@@ -67,6 +69,8 @@ jobs:
         continue-on-error: true
       - name: Setup pyenv
         run: |
+          # Remove existing pyenv installation if present
+          rm -rf $HOME/.pyenv
           curl https://pyenv.run | bash
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
@@ -213,7 +217,7 @@ jobs:
           for file in $FILES_TO_SCAN; do
             if [ -f "$file" ]; then
               echo "=== Scanning $file ==="
-              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+              SCAN_OUTPUT=$($HOME/.local/bin/grype "$file" 2>/dev/null || true)
               echo "$SCAN_OUTPUT"
 
               if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
@@ -256,7 +260,10 @@ jobs:
           patchelf --version
       - name: Audit wheels
         run: |
-          python3 -m pip install auditwheel
+          export PATH="$HOME/.pyenv/bin:$PATH"
+          eval "$(pyenv init -)"
+          pyenv shell 3.9
+          python -m pip install auditwheel
           auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
         continue-on-error: false
       - name: Show files