diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 53d2b202700..00000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "Ubuntu Benchmark" - -on: - pull_request: - paths: - - '.github/workflows/benchmark.yml' - - 'scripts/ci/benchmark-requirements.txt' - - 'scripts/ci/setup-ubuntu.sh' - - push: - branches: [main] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - benchmark: - if: github.repository == 'facebookincubator/velox' - runs-on: 8-core-ubuntu-22.04 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - CCACHE_BASEDIR: "${{ github.workspace }}" - BINARY_DIR: "${{ github.workspace }}/benchmarks/" - CONTENDER_OUTPUT_PATH: "${{ github.workspace }}/benchmark-results/contender/" - INSTALL_PREFIX: "${{ github.workspace }}/dependencies" - steps: - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: 'velox' - submodules: 'recursive' - persist-credentials: false - - - name: Restore Dependencies - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - id: restore-deps - with: - path: "${{ env.INSTALL_PREFIX }}" - key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }} - - - name: Install apt dependencies - run: | - source velox/scripts/setup-ubuntu.sh - install_apt_deps - - - name: Install compiled dependencies - if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }} - env: - CCACHE_DISABLE: "true" - run: | - source velox/scripts/setup-ubuntu.sh - run_and_time install_fmt - run_and_time install_protobuf - run_and_time install_boost - run_and_time install_fast_float - run_and_time install_folly - run_and_time install_stemmer - run_and_time install_thrift - run_and_time install_arrow - - # Folly is built with a new flag that the cache may not have. Ensure velox build and folly have - # used the correct flags (disabled the coroutines) - - name: 'TEMP: Force folly rebuild and update cache' - run: | - source velox/scripts/setup-ubuntu.sh - run_and_time install_folly - - - name: Save Dependencies - if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }} - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: "${{ env.INSTALL_PREFIX }}" - key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }} - - - name: Restore ccache - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 - id: restore-cache - with: - clean: true - path: "ccache" - key: ccache-benchmark - - - name: Clear CCache Statistics - run: | - ccache -sz - - - name: Build Contender Benchmarks - working-directory: velox - run: | - n_cores=$(nproc) - make benchmarks-basic-build NUM_THREADS=$n_cores MAX_HIGH_MEM_JOBS=$n_cores MAX_LINK_JOBS=$n_cores - mkdir -p ${BINARY_DIR}/contender/ - cp -r --verbose _build/release/velox/benchmarks/basic/* ${BINARY_DIR}/contender/ - - - name: CCache after - run: | - ccache -vs - - - name: Save ccache" - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 - with: - path: "ccache" - key: ccache-benchmark - - - name: Install benchmark dependencies - run: | - python3 -m pip install -r velox/scripts/ci/benchmark-requirements.txt - - - name: Run Benchmarks - Contender - working-directory: 'velox' - run: | - make benchmarks-basic-run \ - EXTRA_BENCHMARK_FLAGS="--binary_path ${BINARY_DIR}/contender/ --output_path ${CONTENDER_OUTPUT_PATH}" - - - name: Upload result artifact - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - path: "benchmark-results" - name: "benchmark-results" - retention-days: 5 - - upload: - runs-on: ubuntu-latest - needs: benchmark - if: false && github.event_name == 'push' - permissions: - actions: read - statuses: write - steps: - - - name: Download artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 - with: - merge-multiple: true - path: /tmp/artifacts/ - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: velox - persist-credentials: false - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.10' - cache: 'pip' - cache-dependency-path: "velox/scripts/*" - - - name: Install dependencies - run: pip install -r velox/scripts/ci/benchmark-requirements.txt - - - name: Upload results - env: - CONBENCH_URL: "https://velox-conbench.voltrondata.run/" - CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-8-core" - CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" - CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" - CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}" - CONBENCH_PROJECT_COMMIT: "${{ github.sha }}" - RUN_ID: "GHA-${{ github.run_id }}-${{ github.run_attempt }}" - run: | - ./velox/scripts/ci/benchmark-runner.py upload \ - --run_id "$RUN_ID" \ - --sha "$CONBENCH_PROJECT_COMMIT" \ - --output_dir "/tmp/artifacts/contender/" - - - name: Check the status of the upload - # Status functions like failure() only work in `if:` - if: failure() - id: status - run: echo "failed=true" >> $GITHUB_OUTPUT - - - name: Create a GitHub Status on the contender commit (whether the upload was successful) - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }} - with: - script: | - let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}' - let state = 'success' - let description = 'Result upload succeeded!' - - if(${{ steps.status.outputs.failed == 'true' && true || false }}) { - state = 'failure' - description = 'Result upload failed!' - } - - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: state, - target_url: url, - description: description, - context: 'Benchmark Result Upload' - }) - - - name: Create a GitHub Check benchmark report on the merged PR - env: - CONBENCH_URL: "https://velox-conbench.voltrondata.run/" - GITHUB_APP_ID: "${{ secrets.GH_APP_ID }}" - GITHUB_APP_PRIVATE_KEY: "${{ secrets.GH_APP_PRIVATE_KEY }}" - COMMIT_MESSAGE: "${{ github.event.head_commit.message }}" - CONTENDER_SHA: "${{ github.sha }}" - run: | - ./velox/scripts/ci/benchmark-alert.py \ - --contender-sha "$CONTENDER_SHA" \ - --merge-commit-message "$COMMIT_MESSAGE" \ - --z-score-threshold 50 diff --git a/.github/workflows/breeze.yml b/.github/workflows/breeze.yml deleted file mode 100644 index 0d778f6cac8..00000000000 --- a/.github/workflows/breeze.yml +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Breeze Linux Build - -on: - push: - branches: - - "main" - paths: - - "velox/experimental/breeze/**" - - "velox/external/perfetto/**" - - "CMake/**" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/breeze.yml" - - pull_request: - paths: - - "velox/experimental/breeze/**" - - "velox/external/perfetto/**" - - "CMake/**" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/breeze.yml" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - ubuntu-debug: - runs-on: ubuntu-22.04 - # prevent errors when forks ff their main branch - if: ${{ github.repository == 'facebookincubator/velox' }} - name: "Ubuntu debug" - defaults: - run: - shell: bash - working-directory: velox - steps: - - - uses: actions/checkout@v4 - with: - path: velox - - - name: Install Dependencies - run: | - source scripts/setup-ubuntu.sh && install_apt_deps - - - name: Make Debug Build - env: - VELOX_DEPENDENCY_SOURCE: BUNDLED - # OpenMP build with asan+ubsan enabled - run: | - cmake -S velox/experimental/breeze -B _build-breeze/debug \ - -DCMAKE_BUILD_TYPE=Asan \ - -DCMAKE_CXX_FLAGS="-fsanitize=undefined" \ - -DBUILD_GENERATE_TEST_FIXTURES=OFF \ - -DBUILD_OPENMP=ON - cmake --build _build-breeze/debug -j 8 - - - name: Run Tests - run: | - cd _build-breeze/debug && ctest -j 8 --output-on-failure --no-tests=error - - ubuntu-gpu-relwithdebinfo: - runs-on: 4-core-ubuntu-gpu-t4 - # prevent errors when forks ff their main branch - if: ${{ github.repository == 'facebookincubator/velox' }} - name: "Ubuntu GPU debug" - env: - CUDA_VERSION: "12.2" - defaults: - run: - shell: bash - working-directory: velox - steps: - - - uses: actions/checkout@v4 - with: - path: velox - - - name: Install Dependencies - run: | - source scripts/setup-ubuntu.sh && install_apt_deps && install_cuda ${CUDA_VERSION} - sudo chmod 755 -R /usr/local/lib/python3.10/dist-packages - - - name: Make RelWithDebInfo Build - run: | - cmake -S velox/experimental/breeze -B _build-breeze/relwithdebinfo \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DBUILD_GENERATE_TEST_FIXTURES=OFF \ - -DBUILD_CUDA=ON \ - -DCMAKE_NVCC_FLAGS="-arch=native" - cmake --build _build-breeze/relwithdebinfo -j 8 - - - name: Run Tests - run: | - cd _build-breeze/relwithdebinfo && ctest -j 8 --output-on-failure --no-tests=error diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml deleted file mode 100644 index 3677fd20ad9..00000000000 --- a/.github/workflows/build-metrics.yml +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Collect Build Metrics - -on: - pull_request: - paths: - - ".github/workflows/build-metrics.yml" - - "scripts/ci/bm-report/**" - - workflow_dispatch: - inputs: - ref: - description: "ref to check" - required: true - - schedule: - # Run every day at 04:05 - - cron: "5 4 * * *" - -permissions: - contents: read - -jobs: - metrics: - name: Linux ${{ matrix.link-type }} - ${{ matrix.type }} with adapters - if: ${{ github.repository == 'facebookincubator/velox' }} - runs-on: ${{ matrix.runner }} - container: ghcr.io/facebookincubator/velox-dev:adapters - strategy: - fail-fast: false - matrix: - runner: ["16-core-ubuntu"] - type: ["debug", "release"] - link-type: ["shared", "static"] - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Make ${{ matrix.link-type }} - ${{ matrix.type }} Build - env: - MAKEFLAGS: 'MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4' - run: | - EXTRA_CMAKE_FLAGS=( - "-DVELOX_ENABLE_BENCHMARKS=ON" - "-DVELOX_ENABLE_ARROW=ON" - "-DVELOX_ENABLE_PARQUET=ON" - "-DVELOX_ENABLE_HDFS=ON" - "-DVELOX_ENABLE_S3=ON" - "-DVELOX_ENABLE_GCS=ON" - "-DVELOX_ENABLE_ABFS=ON" - "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" - "-DVELOX_MONO_LIBRARY=ON" - "-DVELOX_BUILD_SHARED=${{ matrix.link-type == 'shared' && 'ON' || 'OFF' }}" - ) - - echo "CMake Flags: ${EXTRA_CMAKE_FLAGS[*]}" - make '${{ matrix.type }}' EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" - - - name: Log binary sizes - run: | - mkdir -p /tmp/metrics - sizes_file=/tmp/metrics/object_sizes - echo "sizes_file=$sizes_file" >> $GITHUB_ENV - - pushd '_build/${{ matrix.type }}' - - # . to also check in lib for mono build - find . -type f \( -name 'libvelox*.so' -o -name 'libvelox*.a' \) -exec ls -l -BB {} \; | - awk '{print $5, $9; total += $5} END {print total," total_lib_size"}' > $sizes_file - - find velox -type f -name '*.o' -exec ls -l -BB {} \; | - awk '{print $5, $9; total += $5} END {print total," total_obj_size"}' >> $sizes_file - - find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | - awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file - - echo "::group::Size file for ${{ matrix.link-type }} - ${{ matrix.type }}" - cat $sizes_file - echo "::endgroup::" - - - uses: actions/upload-artifact@v4 - with: - path: ${{ env.sizes_file }} - name: "${{ matrix.type }}-${{ matrix.link-type }}-sizes" - - - name: Copy ninja_log - run: cp _build/${{ matrix.type }}/.ninja_log /tmp/metrics/.ninja_log - - - name: "Install dependencies" - run: | - python3 -m pip install setuptools - python3 -m pip install -r scripts/ci/benchmark-requirements.txt - - - name: "Upload Metrics" - # This disables the upload and report generation on fork PRs but allows it for forks from within the main repo. - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'facebookincubator/velox' }} - env: - CONBENCH_URL: "https://velox-conbench.voltrondata.run/" - CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-${{ matrix.runner }}" - CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" - CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" - # These don't actually work https://github.com/conbench/conbench/issues/1484 - # but have to be there to work regardless?? - CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}" - CONBENCH_PROJECT_COMMIT: "${{ inputs.ref || github.sha }}" - run: | - ./scripts/ci/bm-report/build-metrics.py upload \ - --build_type "${{ matrix.link-type }}-${{ matrix.type }}" \ - --run_id "BM-${{ matrix.link-type }}-${{ matrix.type }}-${{ github.run_id }}-${{ github.run_attempt }}" \ - --pr_number "${{ github.event.number }}" \ - --sha "${{ inputs.ref || github.sha }}" \ - "/tmp/metrics" - - upload-report: - permissions: - contents: write - runs-on: ubuntu-latest - name: Generate and Upload Build Metric Report - needs: metrics - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Setup Git User - run: | - git config --global user.email "velox@users.noreply.github.com" - git config --global user.name "velox" - - - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - with: - nix_path: nixpkgs=channel:nixos-unstable - - - name: Build Environment - run: | - cd scripts/ci/bm-report - nix-build - - - name: Build Documentation - env: - CONBENCH_URL: "https://velox-conbench.voltrondata.run/" - CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}" - CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}" - run: | - cd scripts/ci/bm-report - nix-shell --run "quarto render report.qmd" - - - name: Upload Report Artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: report - path: scripts/ci/bm-report/report.html - retention-days: 5 - - - name: Push Report - # The report only uses conbench data from 'main' - # so any data generated in a PR won't be included - if: ${{ github.event_name != 'pull_request' && github.repository == 'facebookincubator/velox'}} - run: | - git checkout gh-pages - mkdir -p docs/bm-report - cp -R scripts/ci/bm-report/report.html docs/bm-report/index.html - git add docs - - if [ -n "$(git status --porcelain --untracked-files=no)" ] - then - git commit -m "Update build metrics" - git push - fi diff --git a/.github/workflows/build_pyvelox.yml b/.github/workflows/build_pyvelox.yml deleted file mode 100644 index e92f74ef929..00000000000 --- a/.github/workflows/build_pyvelox.yml +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build Pyvelox Wheels - -on: - workflow_dispatch: - inputs: - version: - description: 'pyvelox version' - required: false - ref: - description: 'git ref to build' - required: false - publish: - description: 'publish to PyPI' - required: false - type: boolean - default: false - # schedule: - # - cron: '15 0 * * *' - pull_request: - paths: - - '.github/workflows/build_pyvelox.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [8-core-ubuntu, macos-13, macos-14] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ inputs.ref || github.ref }} - fetch-depth: 0 - submodules: recursive - persist-credentials: false - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.10' - - - name: "Restore ccache" - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 - id: restore-cache - with: - path: "ccache" - key: ccache-wheels-${{ matrix.os }} - - - name: "Restore macOS Dependencies" - if: startsWith(matrix.os, 'macos') - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 - id: restore-deps - with: - path: "dependencies" - key: dependencies-pyvelox-${{ matrix.os }}-${{ hashFiles('scripts/setup-macos.sh') }} - - - name: Install macOS dependencies from brew - if: startsWith(matrix.os, 'macos') - run: | - export INSTALL_PREFIX="$GITHUB_WORKSPACE/dependencies" - echo "CMAKE_PREFIX_PATH=$INSTALL_PREFIX" >> $GITHUB_ENV - echo "DYLD_LIBRARY_PATH=$INSTALL_PREFIX/lib:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV - - source scripts/setup-macos.sh - install_build_prerequisites - install_velox_deps_from_brew - # CMake 4.0 causes issues with arrows bundled dependencies - brew uninstall cmake - pipx install --force cmake==3.31 - - - name: Install macOS dependencies - if: ${{ startsWith(matrix.os, 'macos') && steps.restore-deps.outputs.stash-hit != 'true' }} - env: - MACOSX_DEPLOYMENT_TARGET: "13.0" - run: | - bash scripts/setup-macos.sh - - - name: "Save macOS Dependencies" - if: ${{ startsWith(matrix.os, 'macos') && steps.restore-deps.outputs.stash-hit != 'true' }} - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 - with: - path: "dependencies" - key: dependencies-pyvelox-${{ matrix.os }}-${{ hashFiles('scripts/setup-macos.sh') }} - retention-days: 90 - - - name: "Create sdist" - if: matrix.os == '8-core-ubuntu' - env: - BUILD_VERSION: "${{ inputs.version }}" - run: | - pipx install uv - uv build --sdist --out-dir wheelhouse - - - name: Build wheels - uses: pypa/cibuildwheel@faf86a6ed7efa889faf6996aa23820831055001a # v2 - env: - BUILD_VERSION: "${{ inputs.version }}" - CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*" - CIBW_ENVIRONMENT_PASS_LINUX: "BUILD_VERSION" - CIBW_ENVIRONMENT_LINUX: "CCACHE_DIR=/host${{ github.workspace }}/ccache" - # for macos - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-14' && '14' || '13' }} - CCACHE_DIR: "${{ github.workspace }}/ccache" - with: - output-dir: wheelhouse - - - name: "Save ccache" - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 - with: - path: "ccache" - key: ccache-wheels-${{ matrix.os }} - - - name: "Rename wheel compatibility tag" - if: false #startsWith(matrix.os, 'macos') - run: | - brew install rename - cd wheelhouse - rename 's/11_0/10_15/g' *.whl - - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: wheels-${{ matrix.os }} - retention-days: 5 - path: | - ./wheelhouse/*.whl - ./wheelhouse/*.tar.gz - - publish_wheels: - name: Publish Wheels to PyPI - if: ${{ github.event_name == 'schedule' || inputs.publish }} - needs: build_wheels - runs-on: ubuntu-22.04 - steps: - - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 - with: - pattern: wheels-* - merge-multiple: true - path: ./wheelhouse - - - run: ls wheelhouse - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.10" - - - name: Publish a Python distribution to PyPI - uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages_dir: wheelhouse diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 7a7202498ea..00000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: Build & Push Docker Images - -on: - pull_request: - paths: - - scripts/docker/*.dockfile - - scripts/docker/*.dockerfile - - scripts/setup-*.sh - - .github/workflows/docker.yml - - docker-compose.yml - push: - branches: [main] - paths: - - scripts/docker/*.dockfile - - scripts/docker/*.dockerfile - - scripts/setup-*.sh - - .github/workflows/docker.yml - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -permissions: - contents: read - packages: write - -jobs: - linux: - name: "Build and Push ${{ matrix.name }}" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: Check - file: "scripts/docker/check-container.dockfile" - tags: "ghcr.io/facebookincubator/velox-dev:check" - - name: Centos 9 - file: "scripts/docker/centos.dockerfile" - tags: "ghcr.io/facebookincubator/velox-dev:centos9" - - name: Pyvelox - file: "scripts/docker/pyvelox.dockerfile" - tags: "ghcr.io/facebookincubator/velox-dev:pyvelox" - - name: Dev - file: "scripts/docker/ubuntu-22.04-cpp.dockerfile" - args: "" - tags: "ghcr.io/facebookincubator/velox-dev:ubuntu-22.04" - - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and Push - uses: docker/build-push-action@v6 - with: - file: "${{ matrix.file }}" - build-args: "${{ matrix.args }}" - push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}} - tags: "${{ matrix.tags }}" - - linux-needs: - name: "Build and Push ${{ matrix.name }}" - needs: linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: Adapters - file: "scripts/docker/adapters.dockerfile" - tags: "ghcr.io/facebookincubator/velox-dev:adapters" - - name: Presto Java - file: "scripts/docker/prestojava-container.dockerfile" - args: "PRESTO_VERSION=0.290" - tags: "ghcr.io/facebookincubator/velox-dev:presto-java" - - name: Spark server - file: "scripts/docker/spark-container.dockerfile" - args: "SPARK_VERSION=3.5.1" - tags: "ghcr.io/facebookincubator/velox-dev:spark-server" - - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and Push - uses: docker/build-push-action@v6 - with: - file: "${{ matrix.file }}" - build-args: "${{ matrix.args }}" - push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}} - tags: "${{ matrix.tags }}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1a799f313ed..0a866356bfd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Update Documentation +name: Build Documentation on: push: @@ -26,91 +26,40 @@ on: - ".github/workflows/docs.yml" permissions: - contents: write + contents: read concurrency: group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true + cancel-in-progress: ${{github.ref != 'refs/heads/presto-0.293-clp-connector'}} jobs: build_docs: - name: Build and Push - runs-on: 8-core-ubuntu - container: ghcr.io/facebookincubator/velox-dev:centos9 + name: Build + # Prevent duplicated runs when an internal branch has a PR + if: github.event_name == 'push' || github.event.pull_request == null + runs-on: ubuntu-latest env: CCACHE_DIR: "/tmp/ccache" steps: - - - name: "Restore ccache" - if: false - uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 - id: restore-cache - with: - path: "${{ env.CCACHE_DIR }}" - key: ccache-wheels-8-core-ubuntu - - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 persist-credentials: true - - name: Install System Dependencies run: | - dnf install -y --setopt=install_weak_deps=False pandoc - pip3 install pipx + sudo apt install -y pipx pipx install --force uv cmake==3.31 - echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV - name: Install Python Dependencies run: | which uv make python-venv which uv - uv pip install -r scripts/docs-requirements.txt - - # Install pyvelox to generate it's docs - make python-build - - - name: "Save ccache" - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 - with: - path: "${{ env.CCACHE_DIR }}" - key: ccache-wheels-8-core-ubuntu + uv pip install -r scripts/docs-requirements.txt - name: Build Documentation run: | source .venv/bin/activate cd velox/docs make clean - # pyvelox - mkdir -p bindings/python - pandoc ../../python/README.md --from markdown --to rst -s -o bindings/python/README_generated_pyvelox.rst - # velox make html - - - name: Setup Git User - run: | - git config --global user.email "velox@users.noreply.github.com" - git config --global user.name "velox" - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Push Documentation - if: ${{ github.event_name == 'push' && github.repository == 'facebookincubator/velox'}} - run: | - git checkout gh-pages - cp -R velox/docs/_build/html/* docs - git add docs - - if [ -n "$(git status --porcelain --untracked-files=no)" ] - then - git commit -m "Update documentation" - git push - fi - - - name: Upload Documentation - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - path: velox/docs/_build/html - retention-days: 3 diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml deleted file mode 100644 index 73cf16c05a5..00000000000 --- a/.github/workflows/experimental.yml +++ /dev/null @@ -1,350 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "Experimental Fuzzer Jobs" - -on: - pull_request: - paths: - - ".github/workflows/experimental.yml" - - - workflow_dispatch: - inputs: - ref: - description: 'Ref to checkout out' - default: 'main' - numThreads: - description: 'Number of threads' - default: 16 - maxHighMemJobs: - description: 'Number of high memory jobs' - default: 8 - maxLinkJobs: - description: 'Maximum number of link jobs' - default: 4 - extraCMakeFlags: - description: 'Additional CMake flags' - default: '' - -defaults: - run: - shell: bash - -permissions: - contents: read - -jobs: - compile: - runs-on: 16-core-ubuntu - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "ubuntu" - steps: - - - name: "Restore ccache" - uses: actions/cache@v4 - with: - path: "${{ env.CCACHE_DIR }}" - # We are using the benchmark ccache as it has all - # required features enabled, so no need to create a new one - key: ccache-benchmark-${{ github.sha }} - restore-keys: | - ccache-benchmark- - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref || 'main' }}" - - - name: "Install dependencies" - run: cd velox && source ./scripts/setup-ubuntu.sh && install_apt_deps - - - name: "Build" - run: | - cd velox - make debug NUM_THREADS="${{ inputs.numThreads || 8 }}" MAX_HIGH_MEM_JOBS="${{ inputs.maxHighMemJobs || 8 }}" MAX_LINK_JOBS="${{ inputs.maxLinkJobs || 4 }}" EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_ARROW=ON ${{ inputs.extraCMakeFlags }}" - ccache -s - - - name: Upload aggregation fuzzer - uses: actions/upload-artifact@v4 - with: - name: aggregation - path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test - - - name: Upload spark aggregation fuzzer - uses: actions/upload-artifact@v4 - with: - name: spark_aggregation_fuzzer - path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test - - - name: Upload join fuzzer - uses: actions/upload-artifact@v4 - with: - name: join - path: velox/_build/debug/velox/exec/fuzzer/velox_join_fuzzer - - - name: Upload Presto expression fuzzer - uses: actions/upload-artifact@v4 - with: - name: presto_expression_fuzzer - path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test - - - name: Upload Spark expression fuzzer - uses: actions/upload-artifact@v4 - with: - name: spark_expression_fuzzer - path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test - - presto-java-aggregation-fuzzer-run: - runs-on: 16-core-ubuntu - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: "Restore ccache" - uses: actions/cache@v4 - with: - path: "${{ env.CCACHE_DIR }}" - # We are using the benchmark ccache as it has all - # required features enabled, so no need to create a new one - key: ccache-presto-${{ github.sha }} - restore-keys: | - ccache-presto- - - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref || 'main' }}" - - - - name: "Build" - run: | - cd velox - source /opt/rh/gcc-toolset-12/enable - make debug NUM_THREADS="${{ inputs.numThreads || 8 }}" MAX_HIGH_MEM_JOBS="${{ inputs.maxHighMemJobs || 8 }}" MAX_LINK_JOBS="${{ inputs.maxLinkJobs || 4 }}" EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_ARROW=ON ${{ inputs.extraCMakeFlags }}" - ccache -s - - - name: "Run Aggregate Fuzzer" - run: | - cd velox - cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - mkdir -p /tmp/aggregate_fuzzer_repro/ - rm -rfv /tmp/aggregate_fuzzer_repro/* - chmod -R 777 /tmp/aggregate_fuzzer_repro - _build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test \ - --seed ${RANDOM} \ - --duration_sec 3600 \ - --logtostderr=1 \ - --minloglevel=0 \ - --repro_persist_path=/tmp/aggregate_fuzzer_repro \ - --enable_sorted_aggregations=true \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\nAggregation fuzzer run finished successfully." - - - name: Archive aggregate production artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: aggregate-fuzzer-failure-artifacts - path: | - /tmp/aggregate_fuzzer_repro - /tmp/server.log - - linux-spark-fuzzer-run: - runs-on: ubuntu-latest - needs: compile - timeout-minutes: 120 - steps: - - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - ref: "${{ inputs.ref || 'main' }}" - - - name: "Install dependencies" - run: source ./scripts/setup-ubuntu.sh && install_apt_deps - - - name: Download spark aggregation fuzzer - uses: actions/download-artifact@v4 - with: - name: spark_aggregation_fuzzer - - - name: "Run Spark Aggregate Fuzzer" - run: | - mkdir -p /tmp/spark_aggregate_fuzzer_repro/ - chmod -R 777 /tmp/spark_aggregate_fuzzer_repro - chmod +x spark_aggregation_fuzzer_test - ./spark_aggregation_fuzzer_test \ - --seed ${RANDOM} \ - --duration_sec 1800 \ - --logtostderr=1 \ - --minloglevel=0 \ - --repro_persist_path=/tmp/spark_aggregate_fuzzer_repro \ - --enable_sorted_aggregations=true \ - && echo -e "\n\nSpark Aggregation Fuzzer run finished successfully." - - - name: Archive Spark aggregate production artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: spark-agg-fuzzer-failure-artifacts - path: | - /tmp/spark_aggregate_fuzzer_repro - - linux-join-fuzzer-run: - runs-on: ubuntu-latest - needs: compile - timeout-minutes: 120 - steps: - - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - ref: "${{ inputs.ref || 'main' }}" - - - name: "Install dependencies" - run: source ./scripts/setup-ubuntu.sh && install_apt_deps - - - name: Download join fuzzer - uses: actions/download-artifact@v4 - with: - name: join - - - name: "Run Join Fuzzer" - run: | - ls /lib64 - mkdir -p /tmp/join_fuzzer_repro/ - rm -rfv /tmp/join_fuzzer_repro/* - chmod -R 777 /tmp/join_fuzzer_repro - chmod +x velox_join_fuzzer - ./velox_join_fuzzer \ - --seed ${RANDOM} \ - --duration_sec 1800 \ - --logtostderr=1 \ - --minloglevel=0 \ - && echo -e "\n\nAggregation fuzzer run finished successfully." - - - name: Archive aggregate production artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: join-fuzzer-failure-artifacts - path: | - /tmp/join_fuzzer_repro - - presto-expression-fuzzer-run: - runs-on: ubuntu-latest - needs: compile - timeout-minutes: 120 - steps: - - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - ref: "${{ inputs.ref || 'main' }}" - - - name: "Install dependencies" - run: source ./scripts/setup-ubuntu.sh && install_apt_deps - - - name: Download presto fuzzer - uses: actions/download-artifact@v4 - with: - name: presto_expression_fuzzer - - - name: "Run Presto Fuzzer" - run: | - mkdir -p /tmp/presto_fuzzer_repro/ - rm -rfv /tmp/presto_fuzzer_repro/* - chmod -R 777 /tmp/presto_fuzzer_repro - chmod +x velox_expression_fuzzer_test - ./velox_expression_fuzzer_test \ - --seed ${RANDOM} \ - --enable_variadic_signatures \ - --velox_fuzzer_enable_complex_types \ - --velox_fuzzer_enable_decimal_type \ - --lazy_vector_generation_ratio 0.2 \ - --velox_fuzzer_enable_column_reuse \ - --velox_fuzzer_enable_expression_reuse \ - --max_expression_trees_per_step 2 \ - --retry_with_try \ - --enable_dereference \ - --duration_sec 1800 \ - --logtostderr=1 \ - --minloglevel=1 \ - --repro_persist_path=/tmp/presto_fuzzer_repro \ - && echo -e "\n\nFuzzer run finished successfully." - - - name: Archive Presto expression production artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: presto-fuzzer-failure-artifacts - path: | - /tmp/presto_fuzzer_repro - - spark-expression-fuzzer-run: - runs-on: ubuntu-latest - needs: compile - timeout-minutes: 120 - steps: - - - name: "Checkout Repo" - uses: actions/checkout@v3 - with: - ref: "${{ inputs.ref || 'main' }}" - - - name: "Install dependencies" - run: source ./scripts/setup-ubuntu.sh && install_apt_deps - - - name: Download spark fuzzer - uses: actions/download-artifact@v4 - with: - name: spark_expression_fuzzer - - - name: "Run Spark Fuzzer" - run: | - mkdir -p /tmp/spark_fuzzer_repro/ - rm -rfv /tmp/spark_fuzzer_repro/* - chmod -R 777 /tmp/spark_fuzzer_repro - chmod +x spark_expression_fuzzer_test - ./spark_expression_fuzzer_test \ - --seed ${RANDOM} \ - --duration_sec 1800 \ - --logtostderr=1 \ - --minloglevel=1 \ - --repro_persist_path=/tmp/spark_fuzzer_repro \ - --velox_fuzzer_enable_decimal_type \ - --retry_with_try \ - && echo -e "\n\nSpark Fuzzer run finished successfully." - - - name: Archive Spark expression production artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: spark-fuzzer-failure-artifacts - path: | - /tmp/spark_fuzzer_repro diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index 3b691120531..f1accbfe073 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -24,107 +24,9 @@ on: type: boolean jobs: - adapters: - name: Linux release with adapters - # prevent errors when forks ff their main branch - if: ${{ github.repository == 'facebookincubator/velox' }} - runs-on: 8-core-ubuntu-22.04 - container: ghcr.io/facebookincubator/velox-dev:adapters - defaults: - run: - shell: bash - env: - CCACHE_DIR: "${{ github.workspace }}/ccache" - VELOX_DEPENDENCY_SOURCE: SYSTEM - GTest_SOURCE: BUNDLED - cudf_SOURCE: BUNDLED - CUDA_VERSION: "12.8" - USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}" - steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install Dependencies - run: | - # Allows to install arbitrary cuda-version whithout needing to update - # docker container before. It simplifies testing new/different versions - if ! yum list installed cuda-nvcc-$(echo ${CUDA_VERSION} | tr '.' '-') 1>/dev/null; then - source scripts/setup-centos9.sh - install_cuda ${CUDA_VERSION} - fi - - - name: Install Minio - run: | - MINIO_BINARY="minio-2022-05-26" - if [ ! -f /usr/local/bin/${MINIO_BINARY} ]; then - wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z -O ${MINIO_BINARY} - chmod +x ./${MINIO_BINARY} - mv ./${MINIO_BINARY} /usr/local/bin/ - fi - - - uses: assignUser/stash/restore@v1 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-linux-adapters-${{ inputs.use-clang && 'clang' || 'gcc' }} - - - name: "Zero Ccache Statistics" - run: | - ccache -sz - - - name: Make Release Build - env: - MAKEFLAGS: 'NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4' - CUDA_ARCHITECTURES: 70 - CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc - # Set compiler to GCC 12 - CUDA_FLAGS: "-ccbin /opt/rh/gcc-toolset-12/root/usr/bin" - run: | - EXTRA_CMAKE_FLAGS=( - "-DVELOX_ENABLE_BENCHMARKS=ON" - "-DVELOX_ENABLE_EXAMPLES=ON" - "-DVELOX_ENABLE_ARROW=ON" - "-DVELOX_ENABLE_GEO=ON" - "-DVELOX_ENABLE_PARQUET=ON" - "-DVELOX_ENABLE_HDFS=ON" - "-DVELOX_ENABLE_S3=ON" - "-DVELOX_ENABLE_GCS=ON" - "-DVELOX_ENABLE_ABFS=ON" - "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" - "-DVELOX_ENABLE_CUDF=ON" - "-DVELOX_ENABLE_WAVE=ON" - "-DVELOX_MONO_LIBRARY=ON" - "-DVELOX_BUILD_SHARED=ON" - ) - if [[ "${USE_CLANG}" = "true" ]]; then scripts/setup-centos9.sh install_clang15; export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; CUDA_FLAGS="-ccbin /usr/lib64/llvm15/bin/clang++-15"; fi - make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" - - - name: Ccache after - run: ccache -s - - - uses: assignUser/stash/save@v1 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-linux-adapters-${{ inputs.use-clang && 'clang' || 'gcc' }} - - - name: Run Tests - # Some of the adapters dependencies are in the 'adapters' conda env - shell: mamba run --no-capture-output -n adapters /usr/bin/bash -e {0} - env: - LIBHDFS3_CONF: "${{ github.workspace }}/scripts/hdfs-client.xml" - working-directory: _build/release - run: | - export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob` - ctest -j 8 --label-exclude cuda_driver --output-on-failure --no-tests=error - - ubuntu-debug: - runs-on: 8-core-ubuntu-22.04 - # prevent errors when forks ff their main branch - if: ${{ github.repository == 'facebookincubator/velox' }} - name: "Ubuntu debug with resolve_dependency" + ubuntu-release: + name: "Ubuntu release with resolve_dependency" + runs-on: yscope-gh-runner env: CCACHE_DIR: "${{ github.workspace }}/ccache" USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}" @@ -133,49 +35,31 @@ jobs: shell: bash working-directory: velox steps: - - - name: Get Ccache Stash - uses: assignUser/stash/restore@v1 + - uses: actions/checkout@v4 with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-debug-default-${{ inputs.use-clang && 'clang' || 'gcc' }} - + path: velox - name: Ensure Stash Dirs Exists working-directory: ${{ github.workspace }} run: | mkdir -p '${{ env.CCACHE_DIR }}' - - uses: actions/checkout@v4 - with: - path: velox - - name: Install Dependencies run: | source scripts/setup-ubuntu.sh && install_apt_deps - - name: Clear CCache Statistics - run: | - ccache -sz - - - name: Make Debug Build + - name: Build Artifact env: VELOX_DEPENDENCY_SOURCE: BUNDLED ICU_SOURCE: SYSTEM - MAKEFLAGS: "NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3" - EXTRA_CMAKE_FLAGS: "-DVELOX_ENABLE_ARROW=ON -DVELOX_ENABLE_PARQUET=ON -DVELOX_ENABLE_EXAMPLES=ON" + MAKEFLAGS: "MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3" run: | - if [[ "${USE_CLANG}" = "true" ]]; then export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; fi - make debug - - - name: CCache after - run: | - ccache -vs - - - uses: assignUser/stash/save@v1 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-ubuntu-debug-default-${{ inputs.use-clang && 'clang' || 'gcc' }} + if [[ "${USE_CLANG}" = "true" ]]; then + export CC=/usr/bin/clang-15 + export CXX=/usr/bin/clang++-15 + fi + make release - name: Run Tests + working-directory: velox/_build/release run: | - cd _build/debug && ctest -j 8 --output-on-failure --no-tests=error + ctest -j $(getconf _NPROCESSORS_ONLN) --output-on-failure --no-tests=error diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index bca6e5f9498..9c3fc040793 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -17,7 +17,7 @@ name: Linux Build using GCC on: push: branches: - - "main" + - "presto-0.293-clp-connector" paths: - "velox/**" - "!velox/docs/**" @@ -30,7 +30,6 @@ on: - "scripts/setup-helper-functions.sh" - ".github/workflows/linux-build.yml" - ".github/workflows/linux-build-base.yml" - pull_request: paths: - "velox/**" @@ -50,7 +49,7 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true + cancel-in-progress: ${{github.ref != 'refs/heads/presto-0.293-clp-connector'}} jobs: linux-gcc: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index c383aed9a60..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: macOS Build - -on: - push: - paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-macos.sh" - - ".github/workflows/macos.yml" - - pull_request: - paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-macos.sh" - - ".github/workflows/macos.yml" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - macos-build: - if: ${{ github.repository == 'facebookincubator/velox' }} - name: "${{ matrix.os }}" - strategy: - fail-fast: false - matrix: - # macos-13 = x86_64 Mac - # macos-14 = arm64 Mac - os: [macos-13, macos-14] - runs-on: ${{ matrix.os }} - env: - CCACHE_DIR: '${{ github.workspace }}/ccache' - # The arm runners have only 7GB RAM - BUILD_TYPE: "${{ matrix.os == 'macos-14' && 'Release' || 'Debug' }}" - INSTALL_PREFIX: "/tmp/deps-install" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Install Dependencies - env: - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE" - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - run: | - source scripts/setup-macos.sh - install_build_prerequisites - install_velox_deps_from_brew - install_double_conversion - - echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV - brew unlink protobuf || echo "protobuf not installed" - brew link --force protobuf@21 - brew uninstall cmake - pipx install --force cmake==3.31 - - - name: Cache ccache - uses: assignUser/stash/restore@v1 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-macos-1-${{ matrix.os }} - - - name: Configure Build - env: - fmt_SOURCE: BUNDLED #brew fmt11 is not supported - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - run: | - ccache -sz -M 5Gi - cmake \ - -B _build/$BUILD_TYPE \ - -GNinja \ - -DTREAT_WARNINGS_AS_ERRORS=1 \ - -DENABLE_ALL_WARNINGS=1 \ - -DVELOX_ENABLE_PARQUET=ON \ - -DVELOX_MONO_LIBRARY=ON \ - -DVELOX_BUILD_SHARED=ON \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE - - - name: Build - run: | - cmake --build _build/$BUILD_TYPE -j $NJOBS - ccache -s - - - uses: assignUser/stash/save@v1 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-macos-1-${{ matrix.os }} - - - name: Run Tests - if: false - run: ctest -j $NJOBS --test-dir _build/debug --output-on-failure diff --git a/.github/workflows/preliminary_checks.yml b/.github/workflows/preliminary_checks.yml deleted file mode 100644 index bd094f3abb4..00000000000 --- a/.github/workflows/preliminary_checks.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: Run Checks - -on: - pull_request: - types: - - opened - - reopened - - edited - - synchronize - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - check-matrix: - name: ${{ matrix.config.name }} - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:check - strategy: - fail-fast: false - matrix: - config: - - { name: "License Header", - command: "header-fix", - message: "Found missing License Header(s)", - } - - { name: "Code Format", - command: "format-fix", - message: "Found format issues" - } - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Check ${{ matrix.config.name }} - run: | - make ${{ matrix.config.command }} - - if ! git diff --quiet; then - diff=`git --no-pager diff` - echo "${{ matrix.command.message }} in the following files:" - git --no-pager diff --name-only - echo "Check the Job summary for a copy-pasteable patch." - - echo "> [!IMPORTANT]" >> $GITHUB_STEP_SUMMARY - echo "${{ matrix.config.message }}" >> $GITHUB_STEP_SUMMARY - echo "> Please apply fix using:" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`sh" >> $GITHUB_STEP_SUMMARY - echo "patch -p1 <> $GITHUB_STEP_SUMMARY - echo "$diff" >> $GITHUB_STEP_SUMMARY - echo "EOF" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - exit 1 - fi diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index de383c1ad52..326378a4e05 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: "Fuzzer Jobs" +name: Linux Build using Clang on: pull_request: @@ -31,7 +31,7 @@ on: push: branches: - - "main" + - "presto-0.293-clp-connector" paths: - "velox/**" - "!velox/docs/**" @@ -45,1296 +45,19 @@ on: - ".github/workflows/scheduled.yml" schedule: - - cron: '0 3 * * *' + - cron: '0 6 * * *' workflow_dispatch: inputs: - ref: - description: 'Ref to checkout out' - default: 'main' - numThreads: - description: 'Number of threads' - default: 16 - maxHighMemJobs: - description: 'Number of high memory jobs' - default: 8 - maxLinkJobs: - description: 'Maximum number of link jobs' - default: 4 - extraCMakeFlags: - description: 'Additional CMake flags' - default: '' - duration: - description: 'Duration of fuzzer run in seconds' - default: 1800 - -defaults: - run: - shell: bash permissions: contents: read concurrency: - # This will not cancel fuzzer runs on main (regardless of which trigger) - # by making the commit sha part of the group but will use the branch - # name in PRs to cancel on going runs on a new commit. group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} - cancel-in-progress: true - -env: - # Run for 15 minute on PRs - DURATION: "${{ inputs.duration || ( github.event_name != 'schedule' && 900 || 1800 )}}" - # minimize artifact duration for PRs, keep them a bit longer for nightly runs - RETENTION: "${{ github.event_name == 'pull_request' && 1 || 3 }}" + cancel-in-progress: ${{github.ref != 'refs/heads/presto-0.293-clp-connector'}} jobs: - compile: - name: Build - # prevent errors when forks ff their main branch - if: ${{ github.repository == 'facebookincubator/velox' }} - runs-on: 32-core-ubuntu - container: ghcr.io/facebookincubator/velox-dev:centos9 - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache" - NUM_THREADS: "${{ inputs.numThreads || 16 }}" - MAX_HIGH_MEM_JOBS: "${{ inputs.maxHighMemJobs || 8 }}" - MAX_LINK_JOBS: "${{ inputs.maxLinkJobs || 4 }}" - SKBUILD_BUILD_DIR: "_build/debug" - PYVELOX_LEGACY_ONLY: "ON" - - defaults: - run: - shell: bash - working-directory: velox - outputs: - presto_bias: ${{ steps.sig-check.outputs.presto_functions }} - presto_error: ${{ steps.sig-check.outputs.presto_error }} - spark_bias: ${{ steps.sig-check.outputs.spark_functions }} - spark_error: ${{ steps.sig-check.outputs.spark_error }} - presto_aggregate_bias: ${{ steps.sig-check.outputs.presto_aggregate_functions }} - presto_aggregate_error: ${{ steps.sig-check.outputs.presto_aggregate_error }} - - steps: - - - name: Get latest commit from main - if: ${{ github.event_name != 'schedule' }} - working-directory: ${{ github.workspace }} - env: - GH_TOKEN: ${{ github.token }} - id: get-head - run: | - if [ '${{ github.event_name == 'push' }}' == "true" ]; then - # get the parent commit of the current one to get the relevant function signatures - head_main=$(gh api -q '.parents.[0].sha' '/repos/facebookincubator/velox/commits/${{ github.sha }}') - else - head_main=$(gh api -H "Accept: application/vnd.github.sha" /repos/facebookincubator/velox/commits/heads/main) - fi - echo "head_main=$head_main" >> $GITHUB_OUTPUT - - - name: Get Function Signature Stash - uses: assignUser/stash/restore@v1 - id: get-sig - with: - path: /tmp/signatures - key: function-signatures-${{ steps.get-head.outputs.head_main || github.sha }} - - - name: Restore ccache - uses: assignUser/stash/restore@v1 - with: - path: "${{ env.CCACHE_DIR }}" - key: ccache-fuzzer-centos - - - name: Fix git permissions - working-directory: ${{ github.workspace }} - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox_main - - - name: Ensure Stash Dirs Exists - working-directory: ${{ github.workspace }} - run: | - mkdir -p '$CCACHE_DIR' - mkdir -p /tmp/signatures - - - name: Checkout Main - if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }} - uses: actions/checkout@v4 - with: - ref: ${{ steps.get-head.outputs.head_main || 'main' }} - path: velox_main - - - name: Build PyVelox - if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }} - working-directory: velox_main - run: | - make python-build - - - name: Create Baseline Signatures - if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }} - working-directory: velox_main - run: | - source .venv/bin/activate - python3 -m pip install deepdiff - python3 scripts/ci/signature.py export --spark /tmp/signatures/spark_signatures_main.json - python3 scripts/ci/signature.py export --presto /tmp/signatures/presto_signatures_main.json - python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json - - - name: Save Function Signature Stash - if: ${{ github.event_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' }} - uses: assignUser/stash/save@v1 - with: - path: /tmp/signatures - key: function-signatures-${{ steps.get-head.outputs.head_main }} - - - name: Checkout Contender - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Zero Ccache Statistics - run: | - ccache -sz - - - name: Build - env: - EXTRA_CMAKE_FLAGS: > - -DVELOX_ENABLE_ARROW=ON \ - -DVELOX_MONO_LIBRARY=ON \ - -DVELOX_BUILD_PYTHON_PACKAGE=ON \ - -DVELOX_PYTHON_LEGACY_ONLY=ON \ - ${{ inputs.extraCMakeFlags }} - run: | - make python-venv - source .venv/bin/activate - make debug - - - - name: Ccache after - run: ccache -s - - - name: Save ccache - # see https://github.com/actions/upload-artifact/issues/543 - continue-on-error: true - if: ${{ github.event_name != 'schedule' }} - uses: assignUser/stash/save@v1 - with: - path: "${{ env.CCACHE_DIR }}" - key: ccache-fuzzer-centos - - - name: Build PyVelox - if: ${{ github.event_name != 'schedule' }} - run: | - make python-build - - - name: Create and test new function signatures - if: ${{ github.event_name != 'schedule' }} - id: sig-check - run: | - source .venv/bin/activate - python3 -m pip install deepdiff - python3 scripts/ci/signature.py gh_bias_check presto spark - python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contender.json - python3 scripts/ci/signature.py bias_aggregates /tmp/signatures/presto_aggregate_signatures_main.json \ - /tmp/signatures/presto_aggregate_signatures_contender.json /tmp/signatures/presto_aggregate_bias_functions \ - /tmp/signatures/presto_aggregate_errors - - - name: Upload Signature Artifacts - if: ${{ github.event_name != 'schedule' }} - uses: actions/upload-artifact@v4 - with: - name: signatures - path: /tmp/signatures - retention-days: "${{ env.RETENTION }}" - - - name: Prepare signatures - working-directory: /tmp/signatures - if: ${{ github.event_name == 'push' }} - run: | - # Remove irrelevant artifacts - rm -f *_bias_functions - rm -f *_signatures_main.json - # Rename signature files as 'main' files - for f in *_signatures_contender.json; do - mv "$f" "${f/_contender.json/_main.json}" - done - - - name: Save Function Signature Stash - if: ${{ github.event_name == 'push' }} - uses: assignUser/stash/save@v1 - with: - path: /tmp/signatures - key: function-signatures-${{ github.sha }} - - - name: Upload presto fuzzer - uses: actions/upload-artifact@v4 - with: - name: presto - path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload spark expression fuzzer - uses: actions/upload-artifact@v4 - with: - name: spark_expression_fuzzer - path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload spark aggregation fuzzer - uses: actions/upload-artifact@v4 - with: - name: spark_aggregation_fuzzer - path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload aggregation fuzzer - uses: actions/upload-artifact@v4 - with: - name: aggregation - path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload join fuzzer - uses: actions/upload-artifact@v4 - with: - name: join - path: velox/_build/debug/velox/exec/fuzzer/velox_join_fuzzer - retention-days: "${{ env.RETENTION }}" - - - name: Upload exchange fuzzer - uses: actions/upload-artifact@v4 - with: - name: exchange - path: velox/_build/debug//velox/exec/fuzzer/velox_exchange_fuzzer - retention-days: "${{ env.RETENTION }}" - - - name: Upload window fuzzer - uses: actions/upload-artifact@v4 - with: - name: window - path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_window_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload cache fuzzer - uses: actions/upload-artifact@v4 - with: - name: cache_fuzzer - path: velox/_build/debug/velox/exec/fuzzer/velox_cache_fuzzer - retention-days: "${{ env.RETENTION }}" - - - name: Upload table evolution fuzzer - uses: actions/upload-artifact@v4 - with: - name: table_evolution_fuzzer - path: velox/_build/debug/velox/exec/tests/velox_table_evolution_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload memory arbitration fuzzer - uses: actions/upload-artifact@v4 - with: - name: memory_arbitration_fuzzer - path: velox/_build/debug/velox/exec/tests/velox_memory_arbitration_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - - name: Upload row number fuzzer - uses: actions/upload-artifact@v4 - with: - name: row_number - path: velox/_build/debug//velox/exec/fuzzer/velox_row_number_fuzzer - retention-days: "${{ env.RETENTION }}" - - - name: Upload topn row number fuzzer - uses: actions/upload-artifact@v4 - with: - name: topn_row_number - path: velox/_build/debug//velox/exec/fuzzer/velox_topn_row_number_fuzzer - retention-days: "${{ env.RETENTION }}" - - - name: Upload writer fuzzer - uses: actions/upload-artifact@v4 - with: - name: writer - path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_writer_fuzzer_test - retention-days: "${{ env.RETENTION }}" - - presto-fuzzer-run: - name: Presto Fuzzer - if: ${{ needs.compile.outputs.presto_bias != 'true' }} - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - steps: - - - uses: dorny/paths-filter@v3 - if: github.event_name == 'pull_request' - id: changes - with: - filters: | - presto: - - 'velox/expression/!(test)**' - - 'velox/exec/!(test)**' - - 'velox/common/!(test)**' - - 'velox/core/!(test)**' - - 'velox/vector/!(test)**' - - - name: Set presto specific fuzzer duration - env: - # Run for 30 minutes instead of 15, when files relevant to presto are touched - pr_duration: "${{ steps.changes.outputs.presto == 'true' && 1800 || 900 }}" - # Run for 60 minutes if its a scheduled run - other_duration: "${{ inputs.duration || (github.event_name == 'push' && 1800 || 3600) }}" - is_pr: "${{ github.event_name == 'pull_request' }}" - run: | - - if [ "$is_pr" == "true" ]; then - duration=$pr_duration - else - duration=$other_duration - fi - - echo "DURATION=$duration" >> $GITHUB_ENV - - - name: Download presto fuzzer - uses: actions/download-artifact@v4 - with: - name: presto - - - name: Run Presto Fuzzer - run: | - mkdir -p /tmp/fuzzer_repro/logs/ - chmod -R 777 /tmp/fuzzer_repro - chmod +x velox_expression_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_expression_fuzzer_test \ - --seed ${random_seed} \ - --enable_variadic_signatures \ - --velox_fuzzer_enable_complex_types \ - --lazy_vector_generation_ratio 0.2 \ - --common_dictionary_wraps_generation_ratio=0.3 \ - --velox_fuzzer_enable_column_reuse \ - --velox_fuzzer_enable_expression_reuse \ - --max_expression_trees_per_step 2 \ - --retry_with_try \ - --enable_dereference \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/fuzzer_repro/logs \ - --repro_persist_path=/tmp/fuzzer_repro \ - && echo -e "\n\nFuzzer run finished successfully." - - - name: Archive production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-fuzzer-failure-artifacts - path: | - /tmp/fuzzer_repro - - presto-bias-fuzzer: - name: Presto Bias Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - if: ${{ needs.compile.outputs.presto_bias == 'true' }} - timeout-minutes: 120 - steps: - - - name: Download presto expression fuzzer - uses: actions/download-artifact@v4 - with: - name: presto - - - name: Download Signatures - uses: actions/download-artifact@v4 - with: - name: signatures - path: /tmp/signatures - - - name: Run Presto Expression Fuzzer - run: | - ls /tmp/signatures - mkdir -p /tmp/presto_bias_fuzzer_repro/logs/ - chmod -R 777 /tmp/presto_bias_fuzzer_repro - chmod +x velox_expression_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_expression_fuzzer_test \ - --seed ${random_seed} \ - --lazy_vector_generation_ratio 0.2 \ - --common_dictionary_wraps_generation_ratio=0.3 \ - --assign_function_tickets $(cat /tmp/signatures/presto_bias_functions) \ - --duration_sec 3600 \ - --enable_variadic_signatures \ - --velox_fuzzer_enable_complex_types \ - --velox_fuzzer_enable_column_reuse \ - --velox_fuzzer_enable_expression_reuse \ - --max_expression_trees_per_step 2 \ - --retry_with_try \ - --enable_dereference \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/presto_bias_fuzzer_repro/logs \ - --repro_persist_path=/tmp/presto_bias_fuzzer_repro \ - && echo -e "\n\nPresto Fuzzer run finished successfully." - - - name: Archive Spark expression production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-bias-fuzzer-failure-artifacts - path: | - /tmp/presto_bias_fuzzer_repro - - spark-aggregate-fuzzer-run: - name: Spark Aggregate Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 60 - steps: - - - name: Download spark aggregation fuzzer - uses: actions/download-artifact@v4 - with: - name: spark_aggregation_fuzzer - - - name: Run Spark Aggregate Fuzzer - run: | - mkdir -p /tmp/spark_aggregate_fuzzer_repro/logs/ - chmod -R 777 /tmp/spark_aggregate_fuzzer_repro - chmod +x spark_aggregation_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./spark_aggregation_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/spark_aggregate_fuzzer_repro/logs \ - --repro_persist_path=/tmp/spark_aggregate_fuzzer_repro \ - && echo -e "\n\nSpark Aggregation Fuzzer run finished successfully." - - - name: Archive Spark aggregate production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: spark-agg-fuzzer-failure-artifacts - path: | - /tmp/spark_aggregate_fuzzer_repro - - spark-bias-fuzzer: - name: Spark Bias Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - if: ${{ needs.compile.outputs.spark_bias == 'true' }} - timeout-minutes: 120 - steps: - - - name: Download spark expression fuzzer - uses: actions/download-artifact@v4 - with: - name: spark_expression_fuzzer - - - name: Download Signatures - uses: actions/download-artifact@v4 - with: - name: signatures - path: /tmp/signatures - - - name: Run Spark Expression Fuzzer - run: | - ls /tmp/signatures - mkdir -p /tmp/spark_bias_fuzzer_repro/logs/ - chmod -R 777 /tmp/spark_bias_fuzzer_repro - chmod +x spark_expression_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./spark_expression_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/spark_bias_fuzzer_repro/logs \ - --assign_function_tickets $(cat /tmp/signatures/spark_bias_functions) \ - --repro_persist_path=/tmp/spark_bias_fuzzer_repro \ - && echo -e "\n\nSpark Fuzzer run finished successfully." - - - name: Archive Spark expression production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: spark-fuzzer-failure-artifacts - path: | - /tmp/spark_bias_fuzzer_repro - - spark-fuzzer: - name: Spark Fuzzer - if: ${{ needs.compile.outputs.spark_bias != 'true' }} - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - steps: - - - name: Download spark expression fuzzer - uses: actions/download-artifact@v4 - with: - name: spark_expression_fuzzer - - - name: Run Spark Expression Fuzzer - run: | - mkdir -p /tmp/spark_fuzzer_repro/logs/ - chmod -R 777 /tmp/spark_fuzzer_repro - chmod +x spark_expression_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./spark_expression_fuzzer_test \ - --seed ${random_seed} \ - --enable_variadic_signatures \ - --lazy_vector_generation_ratio 0.2 \ - --velox_fuzzer_enable_column_reuse \ - --velox_fuzzer_enable_expression_reuse \ - --max_expression_trees_per_step 2 \ - --retry_with_try \ - --enable_dereference \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/spark_fuzzer_repro/logs \ - --repro_persist_path=/tmp/spark_fuzzer_repro \ - && echo -e "\n\nSpark Fuzzer run finished successfully." - - - name: Archive Spark expression production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: spark-fuzzer-failure-artifacts - path: | - /tmp/spark_fuzzer_repro - - presto-java-join-fuzzer-run: - name: Join Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - needs: compile - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download join fuzzer - uses: actions/download-artifact@v4 - with: - name: join - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: Run Join Fuzzer - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/join_fuzzer_repro/logs/ - chmod -R 777 /tmp/join_fuzzer_repro - chmod +x velox_join_fuzzer - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_join_fuzzer \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/join_fuzzer_repro/logs \ - --presto_url=http://127.0.0.1:8080 \ - --req_timeout_ms=2000 \ - && echo -e "\n\nJoin fuzzer run finished successfully." - - - name: Archive join production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-sot-join-fuzzer-failure-artifacts - path: | - /tmp/join_fuzzer_repro - /tmp/server.log - - exchange-fuzzer-run: - name: Exchange Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - steps: - - - name: Download exchange fuzzer - uses: actions/download-artifact@v4 - with: - name: exchange - - - name: Run exchange Fuzzer - run: | - cat /proc/sys/vm/max_map_count - mkdir -p /tmp/exchange_fuzzer_repro/logs/ - chmod -R 777 /tmp/exchange_fuzzer_repro - chmod +x velox_exchange_fuzzer - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_exchange_fuzzer \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/exchange_fuzzer_repro/logs \ - --repro_path=/tmp/exchange_fuzzer_repro \ - && echo -e "\n\Exchange fuzzer run finished successfully." - - - name: Archive Exchange production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: exchange-fuzzer-failure-artifacts - path: | - /tmp/exchange_fuzzer_repro - - presto-java-row-number-fuzzer-run: - name: RowNumber Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - needs: compile - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - - steps: - - - name: Download row number fuzzer - uses: actions/download-artifact@v4 - with: - name: row_number - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: Run RowNumber Fuzzer - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - cat /proc/sys/vm/max_map_count - mkdir -p /tmp/row_fuzzer_repro/logs/ - chmod -R 777 /tmp/row_fuzzer_repro - chmod +x velox_row_number_fuzzer - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_row_number_fuzzer \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/row_fuzzer_repro/logs \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\Row number fuzzer run finished successfully." - - - name: Archive row number production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: row-fuzzer-failure-artifacts - path: | - /tmp/row_fuzzer_repro - - presto-java-topn-row-number-fuzzer-run: - name: TopNRowNumber Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - needs: compile - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - - steps: - - - name: Download topn row number fuzzer - uses: actions/download-artifact@v4 - with: - name: topn_row_number - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: Run TopNRowNumber Fuzzer - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - cat /proc/sys/vm/max_map_count - mkdir -p /tmp/topn_row_fuzzer_repro/logs/ - chmod -R 777 /tmp/topn_row_fuzzer_repro - chmod +x velox_topn_row_number_fuzzer - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_topn_row_number_fuzzer \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --req_timeout_ms 50000 \ - --log_dir=/tmp/topn_row_fuzzer_repro/logs \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\TopN Row number fuzzer run finished successfully." - - - name: Archive topn row number production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: topn-row-fuzzer-failure-artifacts - path: | - /tmp/topn_row_fuzzer_repro - - cache-fuzzer-run: - name: Cache Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - # Temporarily disable on PRs till flakiness is fixed #12167 - if: ${{ github.event_name != 'pull_request' }} - steps: - - - name: Download cache fuzzer - uses: actions/download-artifact@v4 - with: - name: cache_fuzzer - - - name: Run Cache Fuzzer - run: | - mkdir -p /tmp/cache_fuzzer/logs/ - chmod -R 777 /tmp/cache_fuzzer - chmod +x velox_cache_fuzzer - ./velox_cache_fuzzer \ - --seed ${RANDOM} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/cache_fuzzer/logs \ - && echo -e "\n\Cache fuzzer run finished successfully." - - - name: Archive Cache production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: cache-fuzzer-logs - path: | - /tmp/cache_fuzzer - - table-evolution-fuzzer-run: - name: Table Evolution Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - steps: - - - name: Download table evolution fuzzer - uses: actions/download-artifact@v4 - with: - name: table_evolution_fuzzer - - - name: Run Table Evolution Fuzzer - run: | - mkdir -p /tmp/table_evolution_fuzzer_test/logs/ - chmod -R 777 /tmp/table_evolution_fuzzer_test - chmod +x velox_table_evolution_fuzzer_test - ./velox_table_evolution_fuzzer_test \ - --seed ${RANDOM} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/table_evolution_fuzzer_test/logs \ - && echo -e "\n\Table evolution fuzzer run finished successfully." - - - name: Archive table evolution production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: table-evolution-fuzzer-test-logs - path: | - /tmp/table_evolution_fuzzer_test - - memory-arbitration-fuzzer-run: - name: Memory Arbitration Fuzzer - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:centos9 - needs: compile - timeout-minutes: 120 - steps: - - - name: Download memory arbitration fuzzer - uses: actions/download-artifact@v4 - with: - name: memory_arbitration_fuzzer - - - name: Run Memory Arbitration Fuzzer - run: | - mkdir -p /tmp/memory_arbitration_fuzzer_test/logs/ - chmod -R 777 /tmp/memory_arbitration_fuzzer_test - chmod +x velox_memory_arbitration_fuzzer_test - ./velox_memory_arbitration_fuzzer_test \ - --seed ${RANDOM} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/memory_arbitration_fuzzer_test/logs \ - && echo -e "\n\Memory arbitration fuzzer run finished successfully." - - - name: Archive memory arbitration production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: memory-arbitration-fuzzer-test-logs - path: | - /tmp/memory_arbitration_fuzzer_test - - presto-java-aggregation-fuzzer-run: - name: Aggregation Fuzzer with Presto as source of truth - needs: compile - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download aggregation fuzzer - uses: actions/download-artifact@v4 - with: - name: aggregation - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - - name: "Run Aggregate Fuzzer" - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/aggregate_fuzzer_repro/logs/ - chmod -R 777 /tmp/aggregate_fuzzer_repro - chmod +x velox_aggregation_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_aggregation_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/aggregate_fuzzer_repro/logs \ - --repro_persist_path=/tmp/aggregate_fuzzer_repro \ - --enable_sorted_aggregations=true \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\nAggregation fuzzer run finished successfully." - - - name: Archive aggregate production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-sot-aggregate-fuzzer-failure-artifacts - path: | - /tmp/aggregate_fuzzer_repro - /tmp/server.log - - - presto-java-only-bias-function-expression-fuzzer-run: - name: Biased Expression Fuzzer with Only Added/Updated Functions and Presto as source of truth - needs: compile - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - if: ${{ needs.compile.outputs.presto_bias == 'true' }} - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download presto expression fuzzer - uses: actions/download-artifact@v4 - with: - name: presto - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: Download Signatures - uses: actions/download-artifact@v4 - with: - name: signatures - path: /tmp/signatures - - - name: Run Bias Expression Fuzzer - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/presto_only_bias_function_fuzzer_repro/logs/ - chmod -R 777 /tmp/presto_only_bias_function_fuzzer_repro - chmod +x velox_expression_fuzzer_test - echo "Biased functions: $(< /tmp/signatures/presto_bias_functions)" - # Convert the list of function names with tickets into a list of function names only. - function_names="" - IFS=',' read -r -a array <<< $(cat /tmp/signatures/presto_bias_functions) - for x in ${array[@]}; do - if [ -n "$function_names" ]; then - function_names+="," - fi - function_names+=$(echo $x | cut -d '=' -f 1) - done - echo "Biased function names: $function_names" - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - echo "Running Fuzzer for $DURATION" - ./velox_expression_fuzzer_test \ - --seed ${random_seed} \ - --lazy_vector_generation_ratio 0.2 \ - --common_dictionary_wraps_generation_ratio=0.3 \ - --only=$function_names \ - --duration_sec $DURATION \ - --enable_variadic_signatures \ - --velox_fuzzer_enable_complex_types \ - --velox_fuzzer_enable_column_reuse \ - --velox_fuzzer_enable_expression_reuse \ - --max_expression_trees_per_step 2 \ - --retry_with_try \ - --batch_size=6 \ - --presto_url=http://127.0.0.1:8080 \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/presto_only_bias_function_fuzzer_repro/logs \ - --repro_persist_path=/tmp/presto_only_bias_function_fuzzer_repro \ - && echo -e "\n\nPresto Fuzzer run finished successfully." - - name: Archive Presto only-bias-function expression fuzzer production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-only-bias-function-fuzzer-failure-artifacts - path: | - /tmp/presto_only_bias_function_fuzzer_repro - - presto-bias-java-aggregation-fuzzer-run: - name: Biased Aggregation Fuzzer with Presto as source of truth - needs: compile - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - if: ${{ needs.compile.outputs.presto_aggregate_bias == 'true' }} - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download aggregation fuzzer - uses: actions/download-artifact@v4 - with: - name: aggregation - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: Download Signatures - uses: actions/download-artifact@v4 - with: - name: signatures - path: /tmp/signatures - - - name: "Run Bias Aggregate Fuzzer" - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/aggregate_fuzzer_repro/logs/ - chmod -R 777 /tmp/aggregate_fuzzer_repro - chmod +x velox_aggregation_fuzzer_test - echo "signatures folder" - ls /tmp/signatures/ - echo "Biased functions: $(< cat /tmp/signatures/presto_aggregate_bias_functions)" - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - echo "Running Fuzzer for $DURATION" - ./velox_aggregation_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/aggregate_fuzzer_repro/logs \ - --repro_persist_path=/tmp/aggregate_fuzzer_repro \ - --enable_sorted_aggregations=true \ - --only=$(cat /tmp/signatures/presto_aggregate_bias_functions) \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\nAggregation fuzzer run finished successfully." - - - name: Archive bias aggregate production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-bias-sot-aggregate-fuzzer-failure-artifacts - path: | - /tmp/aggregate_fuzzer_repro - /tmp/server.log - - surface-signature-errors: - name: Signature Changes - if: ${{ github.event_name != 'schedule' }} - needs: compile - runs-on: ubuntu-latest - steps: - - name: Download Signatures - uses: actions/download-artifact@v4 - with: - name: signatures - path: /tmp/signatures - - - name: Surface Presto function signature errors - if: ${{ needs.compile.outputs.presto_error == 'true' }} - run: | - cat /tmp/signatures/presto_errors - exit 1 - - - name: Surface Aggregate function signature errors - if: ${{ needs.compile.outputs.presto_aggregate_error == 'true' }} - run: | - cat /tmp/signatures/presto_aggregate_errors - exit 1 - - presto-java-window-fuzzer-run: - name: Window Fuzzer with Presto as source of truth - needs: compile - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download window fuzzer - uses: actions/download-artifact@v4 - with: - name: window - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - - name: "Run Window Fuzzer" - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/window_fuzzer_repro/logs/ - chmod -R 777 /tmp/window_fuzzer_repro - chmod +x velox_window_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_window_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --batch_size=50 \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --log_dir=/tmp/window_fuzzer_repro/logs \ - --repro_persist_path=/tmp/window_fuzzer_repro \ - --enable_window_reference_verification \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\nWindow fuzzer run finished successfully." - - - name: Archive window production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-sot-window-fuzzer-failure-artifacts - path: | - /tmp/window_fuzzer_repro - /tmp/server.log - - presto-java-writer-fuzzer-run: - name: Writer Fuzzer with Presto as source of truth - needs: compile - runs-on: ubuntu-latest - container: ghcr.io/facebookincubator/velox-dev:presto-java - timeout-minutes: 120 - env: - CCACHE_DIR: "${{ github.workspace }}/ccache/" - LINUX_DISTRO: "centos" - steps: - - - name: Download writer fuzzer - uses: actions/download-artifact@v4 - with: - name: writer - - - name: "Checkout Repo" - uses: actions/checkout@v4 - with: - path: velox - submodules: 'recursive' - ref: "${{ inputs.ref }}" - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox - - - name: "Run Writer Fuzzer" - run: | - cd velox - cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog - ls -lR $PRESTO_HOME/etc - echo "jvm config content:" - cat $PRESTO_HOME/etc/jvm.config - $PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 & - ls -lR /var/log - # Sleep for 60 seconds to allow Presto server to start. - sleep 60 - /opt/presto-cli --version - /opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;' - cd - - mkdir -p /tmp/writer_fuzzer_repro/logs/ - chmod -R 777 /tmp/writer_fuzzer_repro - chmod +x velox_writer_fuzzer_test - random_seed=${RANDOM} - echo "Random seed: ${random_seed}" - ./velox_writer_fuzzer_test \ - --seed ${random_seed} \ - --duration_sec $DURATION \ - --minloglevel=0 \ - --stderrthreshold=2 \ - --req_timeout_ms 60000 \ - --log_dir=/tmp/writer_fuzzer_repro/logs \ - --presto_url=http://127.0.0.1:8080 \ - && echo -e "\n\Writer fuzzer run finished successfully." - - - name: Archive writer production artifacts - if: ${{ !cancelled() }} - uses: actions/upload-artifact@v4 - with: - name: presto-sot-writer-fuzzer-failure-artifacts - path: | - /tmp/writer_fuzzer_repro - /tmp/server.log - /var/log - linux-clang: if: ${{ github.event_name == 'schedule' }} name: Build with Clang diff --git a/Makefile b/Makefile index 1e12802cff0..adfb2977986 100644 --- a/Makefile +++ b/Makefile @@ -185,33 +185,33 @@ fuzzertest: debug --logtostderr=1 \ --minloglevel=0 -format-fix: #: Fix formatting issues in the main branch +format-fix: #: Fix formatting issues ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format main --fix + source ${PYTHON_VENV}/bin/activate; scripts/check.py format presto-0.293-clp-connector --fix else - scripts/check.py format main --fix + scripts/check.py format presto-0.293-clp-connector --fix endif -format-check: #: Check for formatting issues on the main branch +format-check: #: Check for formatting issues clang-format --version ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format main + source ${PYTHON_VENV}/bin/activate; scripts/check.py format presto-0.293-clp-connector else - scripts/check.py format main + scripts/check.py format presto-0.293-clp-connector endif header-fix: #: Fix license header issues in the current branch ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header main --fix + source ${PYTHON_VENV}/bin/activate; scripts/check.py header presto-0.293-clp-connector --fix else - scripts/check.py header main --fix + scripts/check.py header presto-0.293-clp-connector --fix endif -header-check: #: Check for license header issues on the main branch +header-check: #: Check for license header issues on the presto-0.293-clp-connector branch ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header main + source ${PYTHON_VENV}/bin/activate; scripts/check.py header presto-0.293-clp-connector else - scripts/check.py header main + scripts/check.py header presto-0.293-clp-connector endif circleci-container: #: Build the linux container for CircleCi diff --git a/scripts/check.py b/scripts/check.py index fda77f2b631..9e826205e7b 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -189,7 +189,7 @@ def get_commit(files): if files == "commit": return "HEAD^" - if files == "main" or files == "master": + if files == "main" or files == "master" or files == "presto-0.293-clp-connector": return util.run(f"git merge-base origin/{files} HEAD")[1] return "" @@ -243,6 +243,7 @@ def add_options(parser): branch_parser = add_check_options(files, "main") branch_parser = add_check_options(files, "master") + branch_parser = add_check_options(files, "presto-0.293-clp-connector") commit_parser = add_check_options(files, "commit")