diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..5b36ac93d48 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,96 @@ +Checks: > + *, + -abseil-*, + -android-*, + -cert-err58-cpp, + -cert-err58-cpp, + -clang-analyzer-osx-*, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-goto, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-owning-memory, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-special-member-functions, + -fuchsia-*, + -google-*, + -hicpp-avoid-c-arrays, + -hicpp-avoid-goto, + -hicpp-deprecated-headers, + -hicpp-no-array-decay, + -hicpp-special-member-functions, + -hicpp-use-equals-default, + -hicpp-vararg, + -hicpp-vararg, + -llvm-header-guard, + -llvm-include-order, + -llvmlibc-*, + -misc-no-recursion, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -misc-unused-parameters, + -modernize-avoid-c-arrays, + -modernize-deprecated-headers, + -modernize-use-nodiscard, + -modernize-use-trailing-return-type, + -mpi-*, + -objc-*, + -openmp-*, + -readability-avoid-const-params-in-decls, + -readability-convert-member-functions-to-static, + -readability-implicit-bool-conversion, + -readability-magic-numbers, + -zircon-*, + +HeaderFilterRegex: '.*' + +WarningsAsErrors: '' + +CheckOptions: + # Naming conventions as explicitly stated in CODING_STYLE.md + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.TypeTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: camelBack + - key: readability-identifier-naming.VariableCase + value: camelBack + - key: readability-identifier-naming.ParameterCase + value: camelBack + - key: readability-identifier-naming.PrivateMemberCase + value: camelBack + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.ProtectedMemberCase + value: camelBack + - key: readability-identifier-naming.ProtectedMemberSuffix + value: _ + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantPrefix + value: k + + # Use nullptr instead of NULL or 0 + - key: modernize-use-nullptr.NullMacros + value: 'NULL' + + # Prefer enum class over enum + - key: modernize-use-using.IgnoreUsingStdAllocator + value: 1 diff --git a/.cmake-format.yaml b/.cmake-format.yaml index 91c373bf6b3..bbbd89f433a 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -24,46 +24,46 @@ format: separate_ctrl_name_with_space: false separate_fn_name_with_space: false dangle_parens: false - command_case: "canonical" - keyword_case: "unchanged" + command_case: canonical + keyword_case: unchanged always_wrap: - set_target_properties - target_sources - target_link_libraries parse: - # We define these for our custom + # We define these for our custom # functions so they get formatted correctly additional_commands: velox_add_library: pargs: nargs: 1+ flags: - - OBJECT - - STATIC - - SHARED - - INTERFACE + - OBJECT + - STATIC + - SHARED + - INTERFACE kwargs: {} velox_base_add_library: pargs: nargs: 1+ flags: - - OBJECT - - STATIC - - SHARED - - INTERFACE + - OBJECT + - STATIC + - SHARED + - INTERFACE kwargs: {} velox_compile_definitions: - pargs: 1 + pargs: 1 kwargs: PRIVATE: '*' PUBLIC: '*' INTERFACE: '*' velox_include_directories: - pargs: '1+' + pargs: 1+ flags: - SYSTEM - BEFORE @@ -74,11 +74,10 @@ parse: INTERFACE: '*' velox_link_libraries: - pargs: '1+' + pargs: 1+ kwargs: PRIVATE: '*' PUBLIC: '*' INTERFACE: '*' - markup: first_comment_is_literal: true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c24be384421..03e20d010ca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -16,7 +16,7 @@ # request a review from owners on PRs with changes to matching files. # We currently do not enforce these reviews as required so it's only a tool # for more granular notifications at the moment. For example component maintainers -# can set a rule so that they are pinged on changes to the sections of the +# can set a rule so that they are pinged on changes to the sections of the # codebase that are relevant for their component. # Only users that have write access to the repo can be added as owners. @@ -29,7 +29,7 @@ CMake/ @assignUser @majetideepak scripts/ @assignUser @majetideepak .github/ @assignUser @majetideepak -# Breeze +# Breeze velox/experimental/breeze @dreveman # Parquet @@ -42,4 +42,4 @@ velox/connectors/hive/storage_adapters/ @majetideepak velox/connectors/ @majetideepak # Caching -velox/common/caching/ @majetideepak +velox/common/caching/ @majetideepak diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 9fc52567b5b..0fd30969d32 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -14,7 +14,7 @@ name: Bug Report description: Report a bug or unexpected behavior. -labels: ["bug", "triage"] +labels: [bug, triage] body: - type: markdown attributes: @@ -26,7 +26,7 @@ body: attributes: label: Bug description description: Please describe the issue and the expected behavior. - value: "[Expected behavior] and [actual behavior]." + value: '[Expected behavior] and [actual behavior].' validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/build.yml b/.github/ISSUE_TEMPLATE/build.yml index f02dbc8509d..9b560541c9e 100644 --- a/.github/ISSUE_TEMPLATE/build.yml +++ b/.github/ISSUE_TEMPLATE/build.yml @@ -14,7 +14,7 @@ name: Build problem description: Report an issue when building Velox. -labels: ["build", "triage"] +labels: [build, triage] body: - type: markdown attributes: @@ -26,7 +26,7 @@ body: attributes: label: Problem description description: Please describe the problem. - value: "Please describe how you were trying to build velox and what issue occured" + value: Please describe how you were trying to build velox and what issue occured validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml index 89cd7ac5864..dcad3a4068d 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.yml +++ b/.github/ISSUE_TEMPLATE/enhancement.yml @@ -14,7 +14,7 @@ name: Enhancement description: Raise a potential enhancement. -labels: ["enhancement"] +labels: [enhancement] body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/fuzzer.yml b/.github/ISSUE_TEMPLATE/fuzzer.yml index 8572fcfcad8..8275ae049a2 100644 --- a/.github/ISSUE_TEMPLATE/fuzzer.yml +++ b/.github/ISSUE_TEMPLATE/fuzzer.yml @@ -14,7 +14,7 @@ name: Fuzzer Report description: Report an issue with the fuzzer or found through fuzzing. -labels: ["bug", "fuzzer-found", "fuzzer"] +labels: [bug, fuzzer-found, fuzzer] body: - type: markdown attributes: @@ -26,7 +26,7 @@ body: attributes: label: Description description: Please describe the issue. - placeholder: "[Expected behavior] and [actual behavior]." + placeholder: '[Expected behavior] and [actual behavior].' validations: required: true - type: textarea diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7adedbe9a4b..e00dd70cc9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,9 +14,9 @@ version: 2 updates: - - package-ecosystem: "github-actions" - directory: "/" + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" + interval: weekly commit-message: - prefix: "build(ci): " + prefix: 'build(ci): ' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e47e262f0e9..7a516babbca 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,13 +17,13 @@ name: Build Documentation on: push: paths: - - "velox/docs/**" - - ".github/workflows/docs.yml" + - velox/docs/** + - .github/workflows/docs.yml pull_request: paths: - - "velox/docs/**" - - ".github/workflows/docs.yml" + - velox/docs/** + - .github/workflows/docs.yml permissions: contents: read @@ -37,8 +37,16 @@ jobs: name: Build runs-on: ubuntu-latest env: - CCACHE_DIR: "/tmp/ccache" + CCACHE_DIR: /tmp/ccache steps: + - name: Restore ccache + if: false + uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4 + id: restore-cache + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-wheels-8-core-ubuntu + - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: @@ -57,6 +65,12 @@ jobs: which uv uv pip install -r scripts/docs-requirements.txt + - name: Save ccache + uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-wheels-8-core-ubuntu + - name: Build Documentation run: | source .venv/bin/activate diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index baf4e0283fc..a0b86504a3e 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -18,45 +18,50 @@ on: workflow_call: inputs: use-clang: - description: 'Use Clang to compile the project.' + description: Use Clang to compile the project. default: false required: false type: boolean jobs: ubuntu-release: - name: "Ubuntu release with resolve_dependency" + name: Ubuntu release with resolve_dependency runs-on: yscope-gh-runner env: - CCACHE_DIR: "${{ github.workspace }}/ccache" - USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}" + CCACHE_COMPRESSLEVEL: 2 + CCACHE_MAX_SIZE: 5G + USE_CLANG: ${{ inputs.use-clang && 'true' || 'false' }} defaults: run: shell: bash working-directory: velox steps: - - name: Ensure Stash Dirs Exists - working-directory: ${{ github.workspace }} - run: | - mkdir -p '${{ env.CCACHE_DIR }}' - - uses: actions/checkout@v4 with: path: velox + persist-credentials: false - name: Install Dependencies run: | source scripts/setup-ubuntu.sh && install_apt_deps + - name: Clear CCache Statistics + run: | + ccache -sz + - name: Build Artifact env: VELOX_DEPENDENCY_SOURCE: BUNDLED ICU_SOURCE: SYSTEM - MAKEFLAGS: "MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3" + MAKEFLAGS: MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3 run: | if [[ "${USE_CLANG}" = "true" ]]; then export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; fi make release + - name: CCache after + run: | + ccache -vs + - name: Run Tests run: | cd _build/release && ctest -j $(getconf _NPROCESSORS_ONLN) --output-on-failure --no-tests=error diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index f047cfd0cde..a9417f6546a 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -17,33 +17,31 @@ name: Linux Build using GCC on: push: branches: - - "presto-0.293-clp-connector" + - presto-0.293-clp-connector paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-common.sh" - - "scripts/setup-versions.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/linux-build.yml" - - ".github/workflows/linux-build-base.yml" + - velox/** + - '!velox/docs/**' + - CMakeLists.txt + - CMake/** + - scripts/setup-ubuntu.sh + - scripts/setup-common.sh + - scripts/setup-versions.sh + - scripts/setup-helper-functions.sh + - .github/workflows/linux-build.yml + - .github/workflows/linux-build-base.yml pull_request: paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-common.sh" - - "scripts/setup-versions.sh" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/linux-build.yml" - - ".github/workflows/linux-build-base.yml" + - velox/** + - '!velox/docs/**' + - CMakeLists.txt + - CMake/** + - scripts/setup-ubuntu.sh + - scripts/setup-common.sh + - scripts/setup-versions.sh + - scripts/setup-helper-functions.sh + - .github/workflows/linux-build.yml + - .github/workflows/linux-build-base.yml permissions: contents: read diff --git a/.github/workflows/preliminary_checks.yml b/.github/workflows/preliminary_checks.yml new file mode 100644 index 00000000000..4c37f21d50a --- /dev/null +++ b/.github/workflows/preliminary_checks.yml @@ -0,0 +1,58 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: Run Checks + +on: + pull_request: + types: + - opened + - reopened + - edited + - synchronize + push: {} + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }} + cancel-in-progress: ${{github.ref != 'refs/heads/presto-0.293-clp-connector'}} + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - run: python -m pip install pre-commit + + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + persist-credentials: false + + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit- + + - name: Run pre-commit + env: + GH_TOKEN: ${{ github.token }} + run: | + files=$(git diff --name-only HEAD^1 HEAD) + echo "::group::Changed files" + echo $files | tr ' ' '\n' + echo "::endgroup::" + pre-commit run --show-diff-on-failure --color=always --files $files diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index cd885aa8ddb..a91cf07536d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -17,39 +17,35 @@ name: Linux Build using Clang on: pull_request: paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-common.sh" - - "scripts/setup-versions.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/scheduled.yml" - - "setup.py" + - velox/** + - '!velox/docs/**' + - CMakeLists.txt + - CMake/** + - scripts/setup-ubuntu.sh + - scripts/setup-common.sh + - scripts/setup-versions.sh + - scripts/setup-helper-functions.sh + - .github/workflows/scheduled.yml + - pyproject.toml push: branches: - - "presto-0.293-clp-connector" + - presto-0.293-clp-connector paths: - - "velox/**" - - "!velox/docs/**" - - "CMakeLists.txt" - - "CMake/**" - - "third_party/**" - - "scripts/setup-ubuntu.sh" - - "scripts/setup-common.sh" - - "scripts/setup-versions.sh" - - "scripts/setup-helper-functions.sh" - - ".github/workflows/scheduled.yml" + - velox/** + - '!velox/docs/**' + - CMakeLists.txt + - CMake/** + - scripts/setup-ubuntu.sh + - scripts/setup-common.sh + - scripts/setup-versions.sh + - scripts/setup-helper-functions.sh + - .github/workflows/scheduled.yml + - pyproject.toml schedule: # Run at 6am UTC to avoid interrupting late night work in Toronto. - - cron: '0 6 * * *' - - workflow_dispatch: - inputs: + - cron: 0 6 * * * permissions: contents: read diff --git a/scripts/docker/check-container.dockfile b/.github/zizmor.yml similarity index 82% rename from scripts/docker/check-container.dockfile rename to .github/zizmor.yml index 9240a97dcd8..a1baabf91cf 100644 --- a/scripts/docker/check-container.dockfile +++ b/.github/zizmor.yml @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM amd64/ubuntu:24.04 -COPY scripts/setup-check.sh /root -COPY scripts/setup-helper-functions.sh / -RUN bash /root/setup-check.sh +rules: + use-trusted-publishing: + ignore: + - build_pyvelox.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..c902b08ed1f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,116 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See https://pre-commit.com for more information + +# General excludes, files can also be excluded on a hook level +exclude: .*\.patch|scripts/tests/.*|velox/external/.*|CMake/third-party/.* +default_install_hook_types: [pre-commit, pre-push] +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + + - repo: local + hooks: + - id: cmake-format + name: cmake-format + description: Format CMake files. + entry: cmake-format + language: python + files: (CMakeLists.*|.*\.cmake|.*\.cmake.in)$ + args: [--in-place] + require_serial: false + additional_dependencies: [cmake-format==0.6.13, pyyaml] + + - id: clang-tidy + name: clang-tidy + description: Run clang-tidy on C/C++ files + stages: + - manual # Needs compile_commands.json + entry: clang-tidy + language: python + types_or: [c++, c] + additional_dependencies: [clang-tidy==18.1.8] + require_serial: true + + - id: license-header + name: license-header + description: Add missing license headers. + entry: ./scripts/checks/license-header.py + args: [-i] + language: python + additional_dependencies: [regex] + require_serial: true + exclude: | + (?x)^( + CMake/Find(Snappy|Sodium|Thrift|double-conversion)\.cmake| + velox/docs/affiliations_map.txt| + velox/.*/bitpacking\.(cpp|h)| + velox/.*/Lemire/.*| + velox/.*/gpu/CudaMemMeter.cu| + velox/.*/coverage/data/.*| + velox/tpch/gen/dbgen/.*| + NOTICE.txt + )$ + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.3 + hooks: + - id: clang-format + # types_or: [c++, c, cuda, metal, objective-c] + files: \.(cpp|cc|c|h|hpp|inc|cu|cuh|clcpp|mm|metal)$ + + # Python + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.9 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + + # The following checks mostly target GitHub Actions workflows. + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.37.0 + hooks: + - id: yamllint + args: [--format, parsable, --strict] + exclude: .*\.clang-(tidy|format) + + - repo: https://github.com/google/yamlfmt + rev: v0.16.0 + hooks: + - id: yamlfmt + exclude: .*\.clang-(tidy|format) + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.7.0 + hooks: + - id: zizmor + + - repo: https://github.com/mpalmer/action-validator + rev: 2f8be1d2066eb3687496a156d00b4f1b3ea7b028 + hooks: + - id: action-validator diff --git a/scripts/setup-check.sh b/.yamlfmt.yml similarity index 59% rename from scripts/setup-check.sh rename to .yamlfmt.yml index d3d6573a8ed..f25bdcc6c46 100644 --- a/scripts/setup-check.sh +++ b/.yamlfmt.yml @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,15 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -set -e -set -x - -export DEBIAN_FRONTEND=noninteractive -apt update -apt install --no-install-recommends -y clang-format-18 python3-pip git make ssh -pip3 install --break-system-packages cmake==3.28.3 cmake_format black pyyaml regex -pip3 cache purge -apt purge --auto-remove -y python3-pip -update-alternatives --install /usr/bin/clang-format clang-format "$(command -v clang-format-18)" 18 -apt clean +match_type: doublestar +exclude: + - '**/.clang-format' + - '**/.clang-tidy' +formatter: + type: basic + retain_line_breaks_single: true + scan_folded_as_literal: true + indent: 2 diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 00000000000..390f9f47502 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,46 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 1 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: disable + empty-values: + forbid-in-flow-mappings: true + forbid-in-block-sequences: true + float-values: + forbid-inf: true + forbid-nan: true + forbid-scientific-notation: true + require-numeral-before-decimal: true + indentation: disable + line-length: disable + octal-values: enable + quoted-strings: + required: only-when-needed + extra-allowed: ['.*\$\{\{.*\}\}.*'] + truthy: + allowed-values: ['true', 'false', 'on'] + level: warning diff --git a/CMake/FindSodium.cmake b/CMake/FindSodium.cmake index c486ac112b8..68ea1f96550 100644 --- a/CMake/FindSodium.cmake +++ b/CMake/FindSodium.cmake @@ -267,15 +267,17 @@ if(NOT TARGET sodium) endif() set_target_properties( - sodium PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C") + sodium + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C") if(sodium_USE_STATIC_LIBS) set_target_properties( sodium - PROPERTIES INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" - IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" - IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}") + PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" + IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}") else() if(UNIX) set_target_properties( @@ -292,9 +294,10 @@ else() if(NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) set_target_properties( sodium - PROPERTIES IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" - IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" - IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}") + PROPERTIES + IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}") endif() endif() endif() diff --git a/CMake/Findglog.cmake b/CMake/Findglog.cmake index 752647cb335..81deadb3644 100644 --- a/CMake/Findglog.cmake +++ b/CMake/Findglog.cmake @@ -1,4 +1,17 @@ # Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # - Try to find Glog # Once done, this will define # @@ -9,29 +22,26 @@ include(FindPackageHandleStandardArgs) include(SelectLibraryConfigurations) -find_library(GLOG_LIBRARY_RELEASE glog - PATHS ${GLOG_LIBRARYDIR}) -find_library(GLOG_LIBRARY_DEBUG glogd - PATHS ${GLOG_LIBRARYDIR}) +find_library(GLOG_LIBRARY_RELEASE glog PATHS ${GLOG_LIBRARYDIR}) +find_library(GLOG_LIBRARY_DEBUG glogd PATHS ${GLOG_LIBRARYDIR}) -find_path(GLOG_INCLUDE_DIR glog/logging.h - PATHS ${GLOG_INCLUDEDIR}) +find_path(GLOG_INCLUDE_DIR glog/logging.h PATHS ${GLOG_INCLUDEDIR}) select_library_configurations(GLOG) -find_package_handle_standard_args(glog DEFAULT_MSG - GLOG_LIBRARY - GLOG_INCLUDE_DIR) +find_package_handle_standard_args(glog DEFAULT_MSG GLOG_LIBRARY + GLOG_INCLUDE_DIR) -mark_as_advanced( - GLOG_LIBRARY - GLOG_INCLUDE_DIR) +mark_as_advanced(GLOG_LIBRARY GLOG_INCLUDE_DIR) set(GLOG_LIBRARIES ${GLOG_LIBRARY}) set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) -if (NOT TARGET glog::glog) +if(NOT TARGET glog::glog) add_library(glog::glog UNKNOWN IMPORTED) - set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}") - set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}") + set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${GLOG_INCLUDE_DIRS}") + set_target_properties( + glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${GLOG_LIBRARIES}") endif() diff --git a/CMake/Findlz4.cmake b/CMake/Findlz4.cmake index d49115f1274..d13c951b889 100644 --- a/CMake/Findlz4.cmake +++ b/CMake/Findlz4.cmake @@ -1,4 +1,17 @@ # Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # - Try to find lz4 # Once done, this will define # diff --git a/CMake/Findlzo2.cmake b/CMake/Findlzo2.cmake index c263f5926c0..9f9fbbbe11c 100644 --- a/CMake/Findlzo2.cmake +++ b/CMake/Findlzo2.cmake @@ -1,4 +1,17 @@ # Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # - Try to find lzo2 # Once done, this will define # diff --git a/CMake/Findzstd.cmake b/CMake/Findzstd.cmake index 86c1214492c..a74adb0fbe0 100644 --- a/CMake/Findzstd.cmake +++ b/CMake/Findzstd.cmake @@ -1,4 +1,17 @@ # Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # - Try to find zstd # Once done, this will define # diff --git a/CODING_STYLE.md b/CODING_STYLE.md index 1f540e0c935..43947192cd7 100644 --- a/CODING_STYLE.md +++ b/CODING_STYLE.md @@ -7,34 +7,50 @@ future. ## Code Formatting, Headers, and Licenses -Our Makefile contains targets to help highlight and fix format, header or -license issues. These targets are shortcuts for calling `./scripts/check.py`. +We use [pre-commit](https://pre-commit.com) to manage the installation and +execution of a number of code quality checks, called hooks. -Use `make header-fix` to apply our open source license headers to new files. -Use `make format-fix` to identify and fix formatting issues using clang-format. +### Installation -Formatting issues found on the changed lines in the current commit can be -displayed using `make format-check`. These issues can be fixed by using `make -format-fix`. This command will apply formatting changes to modified lines in -the current commit. +The recommended way to install pre-commit is through either +[`pipx`](https://pipx.pypa.io/stable/) or the newer +[`uv tool`](https://docs.astral.sh/uv/guides/tools/). Once you have +pre-commit available in your environment, you can enable running checks on +each commit by running `pre-commit install` in the root of the repository. -Header issues found on the changed files in the current commit can be displayed -using `make header-check`. These issues can be fixed by using `make header-fix`. -This will apply license header updates to the files in the current commit. +> [!TIP] +> This will take a few minutes the first time you run it, as `pre-commit` will +set up the environment for each hook by installing the required tool and +its dependencies in a separate environment to ensure reproducibility of the +check results. -An entire directory tree of files can be formatted and have license headers -added using the `tree` variant of the format commands: -``` - ./scripts/check.py format tree - ./scripts/check.py format tree --fix +The hooks are defined in `.pre-commit-config.yaml`. - ./scripts/check.py header tree - ./scripts/check.py header tree --fix -``` +After the setup is complete, each time you `git commit`, the hooks will be run +and potential changes applied to your *staged* files. Any unstaged files will +be stashed while the hooks run. If any changes occurred, the commit will *not* +succeed. The same happens when you `git push` but applies to all files that are being +pushed into the repository. + +You will have to review and stage the changed files and commit again. + +To manually run one specific hook, use `pre-commit run `, for example +`pre-commit run clang-format`. You can find the `hookid` in `.pre-commit-config.yaml`. + +By design, `pre-commit` will only be run on the files that are part of the commit +or push. If you want to run the checks on all files (including unstaged files), you can +run `pre-commit run --all-files`. + +The `clang-tidy` hook will *not* be run automatically as it takes a long time and +requires CMake to be run first to create `compile_commands.json`. +It can be run explicitly via `pre-commit run --hook-stage=manual`. + +If you need to *temporarily* skip the checks, you can use the git flag `--no-verify`. -All the available formatting commands can be displayed by using -`./scripts/check.py help`. +> [!Important] +> We also run the hooks as part of our CI, which will flag any issues introduced by +skipping the checks. ## C++ Style diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b16b04e62c..a76dd9e334c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Welcome! Thank you for your interest in the Velox project. Before starting to contribute, please take a moment to review the guidelines outlined below. Contributions are not just about code. Contributing code is great, but that’s -probably not the best place to start. There are many ways in which people can +probably not the best place to start. There are many ways in which people can make contributions to the project and community. ## Code of Conduct @@ -34,28 +34,28 @@ found here](https://velox-lib.io/docs/community/components-and-maintainers). ## Documentation Help the community understand how to use the Velox library by proposing -additions to our [docs](https://facebookincubator.github.io/velox/index.html) or pointing +additions to our [docs](https://facebookincubator.github.io/velox/index.html) or pointing out outdated or missing pieces. ## Bug Reports Found a bug? Help us by filing an issue on GitHub. -Ensure the bug was not already reported by searching +Ensure the bug was not already reported by searching [GitHub Issues](https://github.com/facebookincubator/velox/issues). If you're unable to find an open issue addressing the problem, open a new one. Be sure to include a title and clear description, as much relevant information as possible, and a code sample or an executable test case demonstrating the expected behavior. -Meta has a [bounty program](https://www.facebook.com/whitehat/) for the safe disclosure -of security bugs. In those cases, please go through the process outlined on that page +Meta has a [bounty program](https://www.facebook.com/whitehat/) for the safe disclosure +of security bugs. In those cases, please go through the process outlined on that page and do not file a public issue. ## Code Contribution Process -The code contribution process is designed to reduce the burden on reviewers and -maintainers, allowing them to provide more timely feedback and keeping the +The code contribution process is designed to reduce the burden on reviewers and +maintainers, allowing them to provide more timely feedback and keeping the amount of rework from contributors to a minimum. We encourage new contributors to start with bug fixes and small features so you @@ -105,11 +105,11 @@ The contribution process is outlined below: reviewer(s) by name, stating the comments have been addressed. This is the best way to ensure that the reviewer is notified that the code is ready to be reviewed again. - * As a PR author, please do not "Resolve Conversation" when review comments are + * As a PR author, please do not "Resolve Conversation" when review comments are addressed. Instead, wait for the reviewer to verify the comment has been addressed and resolve the conversation. -7. Iterate on this process until your changes are reviewed and accepted by a +7. Iterate on this process until your changes are reviewed and accepted by a maintainer. At this point, a Meta employee will be notified to merge your PR, due to tooling limitations. @@ -193,35 +193,35 @@ write great commit messages: When submitting code contributions to Velox, make sure to adhere to the following best practices: -1. **Coding Style**: Review and strictly follow our coding style document, +1. **Coding Style**: Review and strictly follow our coding style document, available in [`CODING_STYLE.md`](CODING_STYLE.md). - * Velox favors consistency over personal preference. If there are - technical reasons why a specific guideline should not be followed, + * Velox favors consistency over personal preference. If there are + technical reasons why a specific guideline should not be followed, please start a separate discussion with the community to update the coding style document first. - * If you are simply updating code that did not comply with the coding - style, please do so in a standalone PR isolated from other logic changes. + * If you are simply updating code that did not comply with the coding + style, please do so in a standalone PR isolated from other logic changes. 2. **Small Incremental Changes**: If the change is large, work with the maintainers on a plan to break and submit it as smaller (yet atomic) parts. - * [Research indicates](https://smartbear.com/learn/code-review/best-practices-for-peer-code-review/) - that engineers can only effectively review up to - 400 lines of code at a time. The human brain can only process so much information + * [Research indicates](https://smartbear.com/learn/code-review/best-practices-for-peer-code-review/) + that engineers can only effectively review up to + 400 lines of code at a time. The human brain can only process so much information at a time; beyond that threshold the ability to find bugs and other flaws decreases. - * As larger PRs usually take longer to review and iterate, they - tend to slow down the software development process. As much as possible, - split your work into smaller changes. + * As larger PRs usually take longer to review and iterate, they + tend to slow down the software development process. As much as possible, + split your work into smaller changes. 3. **Unit tests**: With rare exceptions, every PR should contain unit tests covering the logic added/modified. * Unit tests protect our codebase from regressions, promote less coupled APIs, and provide an executable form of documentation that’s useful for new engineers reasoning about the codebase. - * Good unit tests are fast, isolated, repeatable, and exercise all APIs + * Good unit tests are fast, isolated, repeatable, and exercise all APIs including their edge cases. * The lack of existing tests is not a good reason not to add tests to - your PR. If a component or API does not have a corresponding + your PR. If a component or API does not have a corresponding unit test suite, please consider improving the codebase by first adding a new unit test suite to ensure the existing behavior is correct. @@ -232,9 +232,9 @@ following best practices: obvious and remove obscurity. * As a guideline, every file, class, member variable, and member function that is not a getter/setter should be documented. - * As much as possible, try to avoid functions with very large bodies. In the - (rare) cases where large code blocks are needed, a good practice is to group - smaller blocks of related code, and precede them with a blank line and a + * As much as possible, try to avoid functions with very large bodies. In the + (rare) cases where large code blocks are needed, a good practice is to group + smaller blocks of related code, and precede them with a blank line and a high-level comment explaining what the block does. 5. **Benchmarks**: Add micro-benchmarks to support your claims. @@ -242,8 +242,8 @@ following best practices: efficiency trade-offs. 6. **APIs**: Carefully design APIs. - * As a library, Velox APIs should be intentional. External API should only - be deliberately created. + * As a library, Velox APIs should be intentional. External API should only + be deliberately created. * As a rule of thumb, components should be deep and encapsulate as much complexity as possible, and APIs should be narrow, minimizing dependencies across components and preventing implementation details from leaking through @@ -261,20 +261,20 @@ with a benchmark. 2. Use the following template for the PR title: Add xxx [Presto|Spark] function (replace xxx with the function name). * Ensure the PR description contains a link to the function documentation - from Presto or Spark docs. + from Presto or Spark docs. * Describe the function semantics and edge cases clearly. -3. Use Presto or Spark to check the function semantics. +3. Use Presto or Spark to check the function semantics. * When implementing a Spark function, check the function semantics using Spark 3.5 with ANSI OFF. * Try different edge cases to check whether the function returns null, or - throws, etc. + throws, etc. * Make sure to replicate the exact semantics. -4. Add tests exercising common inputs, all possible signatures and corner cases. - * Make sure the test cases are concise and easily readable. +4. Add tests exercising common inputs, all possible signatures and corner cases. + * Make sure the test cases are concise and easily readable. -5. Make sure that obvious inefficiencies are addressed. - * If appropriate, provide micro-benchmarks to support your claims with data. +5. Make sure that obvious inefficiencies are addressed. + * If appropriate, provide micro-benchmarks to support your claims with data. 4. Add documentation for the new function to an .rst file under velox/docs/functions directory. * Functions in documentation are listed in alphabetical order. Make sure to diff --git a/Makefile b/Makefile index 89d66af5785..be290e806ea 100644 --- a/Makefile +++ b/Makefile @@ -185,48 +185,6 @@ fuzzertest: debug --logtostderr=1 \ --minloglevel=0 -format-fix: #: Fix formatting issues -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format presto-0.293-clp-connector --fix -else - scripts/check.py format presto-0.293-clp-connector --fix -endif - -format-check: #: Check for formatting issues - clang-format --version -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format presto-0.293-clp-connector -else - scripts/check.py format presto-0.293-clp-connector -endif - -header-fix: #: Fix license header issues in the current branch -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header presto-0.293-clp-connector --fix -else - scripts/check.py header presto-0.293-clp-connector --fix -endif - -header-check: #: Check for license header issues -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header presto-0.293-clp-connector -else - scripts/check.py header presto-0.293-clp-connector -endif - -circleci-container: #: Build the linux container for CircleCi - $(MAKE) linux-container CONTAINER_NAME=circleci - -check-container: - $(MAKE) linux-container CONTAINER_NAME=check - -linux-container: - rm -rf /tmp/docker && \ - mkdir -p /tmp/docker && \ - cp scripts/setup-helper-functions.sh scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile /tmp/docker && \ - cd /tmp/docker && \ - docker build --build-arg cpu_target=$(CPU_TARGET) --tag "prestocpp/velox-$(CPU_TARGET)-$(CONTAINER_NAME):${USER}-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile . - help: #: Show the help messages @cat $(firstword $(MAKEFILE_LIST)) | \ awk '/^[-a-z]+:/' | \ diff --git a/NOTICE.txt b/NOTICE.txt index 8b812aa41ab..4fb5849fba0 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10,7 +10,7 @@ This product includes software from the QT project (BSD, 3-clause). This product includes software from HowardHinnant's date library (MIT License). * https://github.com/HowardHinnant/date/tree/master -This product includes software from the The Arrow project. +This product includes software from the Arrow project. * https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/io/hdfs_internal.h * https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/io/hdfs_internal.cc Which contain the following NOTICE file: diff --git a/docker-compose.yml b/docker-compose.yml index 7dfc8752d24..d7cc50d1a8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: environment: NUM_THREADS: 8 # default value for NUM_THREADS VELOX_DEPENDENCY_SOURCE: BUNDLED # Build dependencies from source - CCACHE_DIR: "/velox/.ccache" + CCACHE_DIR: /velox/.ccache volumes: - .:/velox:delegated command: scripts/docker/docker-command.sh @@ -51,9 +51,10 @@ services: image: ghcr.io/facebookincubator/velox-dev:centos9 environment: NUM_THREADS: 8 # default value for NUM_THREADS - CCACHE_DIR: "/velox/.ccache" - EXTRA_CMAKE_FLAGS: -DVELOX_ENABLE_PARQUET=ON - -DVELOX_ENABLE_S3=ON + CCACHE_DIR: /velox/.ccache + EXTRA_CMAKE_FLAGS: > + -DVELOX_ENABLE_PARQUET=ON + -DVELOX_ENABLE_S3=ON volumes: - .:/velox:delegated working_dir: /velox @@ -77,9 +78,10 @@ services: image: ghcr.io/facebookincubator/velox-dev:centos9 environment: NUM_THREADS: 8 # default value for NUM_THREADS - CCACHE_DIR: "/velox/.ccache" - EXTRA_CMAKE_FLAGS: -DVELOX_ENABLE_PARQUET=ON - -DVELOX_ENABLE_S3=ON + CCACHE_DIR: /velox/.ccache + EXTRA_CMAKE_FLAGS: > + -DVELOX_ENABLE_PARQUET=ON + -DVELOX_ENABLE_S3=ON privileged: true deploy: resources: @@ -93,7 +95,6 @@ services: working_dir: /velox command: /velox/scripts/docker/docker-command.sh - centos-cpp: # Usage: # docker-compose pull centos-cpp or docker-compose build centos-cpp @@ -109,19 +110,19 @@ services: image: quay.io/centos/centos:stream9 environment: NUM_THREADS: 8 # default value for NUM_THREADS - CCACHE_DIR: "/velox/.ccache" + CCACHE_DIR: /velox/.ccache volumes: - .:/velox:delegated working_dir: /velox command: /velox/scripts/docker/docker-command.sh presto-java: - # Usage: - # docker-compose pull presto-java or docker-compose build presto-java - # docker-compose run --rm presto-java - # or - # docker-compose run -e NUM_THREADS= --rm presto-java - # to set the number of threads used during compilation + # Usage: + # docker-compose pull presto-java or docker-compose build presto-java + # docker-compose run --rm presto-java + # or + # docker-compose run -e NUM_THREADS= --rm presto-java + # to set the number of threads used during compilation image: ghcr.io/facebookincubator/velox-dev:presto-java build: args: @@ -130,19 +131,19 @@ services: dockerfile: scripts/docker/prestojava-container.dockerfile environment: NUM_THREADS: 8 # default value for NUM_THREADS - CCACHE_DIR: "/velox/.ccache" + CCACHE_DIR: /velox/.ccache volumes: - .:/velox:delegated working_dir: /velox command: /velox/scripts/docker/docker-command.sh spark-server: - # Usage: - # docker-compose pull spark-server or docker-compose build spark-server - # docker-compose run --rm spark-server - # or - # docker-compose run -e NUM_THREADS= --rm spark-server - # to set the number of threads used during compilation + # Usage: + # docker-compose pull spark-server or docker-compose build spark-server + # docker-compose run --rm spark-server + # or + # docker-compose run -e NUM_THREADS= --rm spark-server + # to set the number of threads used during compilation image: ghcr.io/facebookincubator/velox-dev:spark-server build: args: @@ -151,7 +152,7 @@ services: dockerfile: scripts/docker/spark-container.dockerfile environment: NUM_THREADS: 8 # default value for NUM_THREADS - CCACHE_DIR: "/velox/.ccache" + CCACHE_DIR: /velox/.ccache volumes: - .:/velox:delegated working_dir: /velox diff --git a/pyproject.toml b/pyproject.toml index adb39ac3634..439d447e583 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/pyvelox/__init__.py b/python/pyvelox/__init__.py index cd7b54a784b..9bfe654c9eb 100644 --- a/python/pyvelox/__init__.py +++ b/python/pyvelox/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .legacy import * -from .legacy import __version__ +from .legacy import * # noqa: F403 +from .legacy import __version__ as __version__ diff --git a/python/pyvelox/utils/__init__.py b/python/pyvelox/utils/__init__.py index ddb2e2b74b0..ef7f461f1d4 100644 --- a/python/pyvelox/utils/__init__.py +++ b/python/pyvelox/utils/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .data_generator import generate_tpch_data +from .data_generator import generate_tpch_data as generate_tpch_data diff --git a/python/pyvelox/utils/data_generator.py b/python/pyvelox/utils/data_generator.py old mode 100644 new mode 100755 index 526acd15071..0ccec166619 --- a/python/pyvelox/utils/data_generator.py +++ b/python/pyvelox/utils/data_generator.py @@ -157,7 +157,7 @@ def main() -> int: result = generate_tpch_data(**vars(args)) logging.info( - f"Written {result.row_count} records to {result.file_count} output files at '{result.output_path}'" # pyre-ignore + f"Written {result.row_count} records to {result.file_count} output files at '{result.output_path}'" # pyre-ignore ) return 0 if result else 1 diff --git a/python/pyvelox/utils/run_queries.py b/python/pyvelox/utils/run_queries.py old mode 100644 new mode 100755 diff --git a/python/test/test_plan_builder.py b/python/test/test_plan_builder.py index 180d0f3d3db..c0c1c8d1ec4 100644 --- a/python/test/test_plan_builder.py +++ b/python/test/test_plan_builder.py @@ -55,7 +55,7 @@ def test_plan_builder(self): self.assertEqual( str(filter_node), - "-- Filter[2]\n" " -- Project[1]\n" " -- TableScan[0]\n", + "-- Filter[2]\n -- Project[1]\n -- TableScan[0]\n", ) def test_multiple_plan_builders(self): diff --git a/python/test/test_vector.py b/python/test/test_vector.py index f829eaaa4bf..a8a97ffd901 100644 --- a/python/test/test_vector.py +++ b/python/test/test_vector.py @@ -188,13 +188,13 @@ def test_array_vector(self): self.assertEqual(expected_firstElements[i], elements[i]) with self.assertRaises(TypeError): - a = pv.from_list([[[1, 2], [3, 4]], [[1.1], [2.3]]]) + _a = pv.from_list([[[1, 2], [3, 4]], [[1.1], [2.3]]]) with self.assertRaises(ValueError): - v = pv.from_list([[None], [None, None, None]]) + _v = pv.from_list([[None], [None, None, None]]) with self.assertRaises(TypeError): - a = pv.from_list([[[1, 2], [3, 4]], [["hello"], ["world"]]]) + _a = pv.from_list([[[1, 2], [3, 4]], [["hello"], ["world"]]]) def test_to_string(self): self.assertEqual( @@ -271,8 +271,8 @@ def test_numeric_limits(self): bigger_than_int32 = pv.from_list([1 << 33]) self.assertEqual(bigger_than_int32[0], 1 << 33) with self.assertRaises(RuntimeError): - bigger_than_int64 = pv.from_list([1 << 63]) - smaller_than_int64 = pv.from_list([(1 << 62) + (1 << 62) - 1]) + _bigger_than_int64 = pv.from_list([1 << 63]) + _smaller_than_int64 = pv.from_list([(1 << 62) + (1 << 62) - 1]) def test_type(self): ints = pv.from_list([1, 2, None]) @@ -334,7 +334,7 @@ def test_slice(self): self.assertEqual(b[i], i + 2) with self.assertRaises(NotImplementedError): - c = a.slice(2, 6, 2) + _c = a.slice(2, 6, 2) d = a[3:6] self.assertEqual(len(d), 3) @@ -342,7 +342,7 @@ def test_slice(self): self.assertEqual(d[i], i + 3) with self.assertRaises(NotImplementedError): - e = a[3:8:3] + _e = a[3:8:3] def test_export_to_arrow(self): test_cases = [ diff --git a/scripts/check.py b/scripts/check.py deleted file mode 100755 index 9e826205e7b..00000000000 --- a/scripts/check.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -from collections import OrderedDict -import os -import regex -import subprocess -import sys - -from util import attrdict -import util - -EXTENSIONS = "cpp,h,inc,prolog" -SCRIPTS = util.script_path() - - -def get_diff(file, formatted): - if not formatted.endswith("\n"): - formatted = formatted + "\n" - - status, stdout, stderr = util.run( - f"diff -u {file} --label {file} --label {file} -", input=formatted - ) - if stdout != "": - stdout = f"diff a/{file} b/{file}\n" + stdout - - return status, stdout, stderr - - -class CppFormatter(str): - def diff(self, commit): - if commit == "": - return get_diff(self, util.run(f"clang-format --style=file {self}")[1]) - else: - return util.run( - f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --diff --style=file {commit} {self}" - ) - - def fix(self, commit): - if commit == "": - return util.run(f"clang-format -i --style=file {self}")[0] == 0 - else: - return ( - util.run( - f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --style=file {commit} {self}" - )[0] - == 0 - ) - - -class CMakeFormatter(str): - def __init__(self, commit) -> None: - super().__init__() - try: - import yaml - except ModuleNotFoundError: - # We need pyyaml so cmake-format can read '.cmake-format.yml' - # otherwise it will run with default - raise SystemExit("Please install 'pyyaml' for the CMake formatter.") - - def diff(self, commit): - return get_diff( - self, util.run(f"cmake-format --first-comment-is-literal True {self}")[1] - ) - - def fix(self, commit): - return ( - util.run(f"cmake-format --first-comment-is-literal True -i {self}")[0] == 0 - ) - - -class PythonFormatter(str): - def diff(self, commit): - return util.run(f"black -q --diff {self}") - - def fix(self, commit): - return util.run(f"black -q {self}")[0] == 0 - - -format_file_types = OrderedDict( - { - "CMakeLists.txt": attrdict({"formatter": CMakeFormatter}), - "*.cmake": attrdict({"formatter": CMakeFormatter}), - "*.cpp": attrdict({"formatter": CppFormatter}), - "*.h": attrdict({"formatter": CppFormatter}), - "*.inc": attrdict({"formatter": CppFormatter}), - "*.prolog": attrdict({"formatter": CppFormatter}), - "*.hpp": attrdict({"formatter": CppFormatter}), - "*.cu": attrdict({"formatter": CppFormatter}), - "*.cuh": attrdict({"formatter": CppFormatter}), - "*.clcpp": attrdict({"formatter": CppFormatter}), - "*.mm": attrdict({"formatter": CppFormatter}), - "*.metal": attrdict({"formatter": CppFormatter}), - "*.py": attrdict({"formatter": PythonFormatter}), - } -) - - -def get_formatter(filename): - if filename in format_file_types: - return format_file_types[filename] - - return format_file_types.get("*" + util.get_fileextn(filename), None) - - -def format_command(commit, files, fix): - ok = 0 - for filepath in files: - filename = util.get_filename(filepath) - filetype = get_formatter(filename) - - if filetype is None: - print("Skip : " + filepath, file=sys.stderr) - continue - - file = filetype.formatter(filepath) - - if fix == "show": - status, diff, stderr = file.diff(commit) - - if stderr != "": - ok = 1 - print(f"Error: {file}", file=sys.stderr) - continue - - if diff != "" and diff != "no modified files to format": - ok = 1 - print(f"Fix : {file}", file=sys.stderr) - print(diff) - else: - print(f"Ok : {file}", file=sys.stderr) - - else: - print(f"Fix : {file}", file=sys.stderr) - if not file.fix(commit): - ok = 1 - print(f"Error: {file}", file=sys.stderr) - - return ok - - -def header_command(commit, files, fix): - options = "-vk" if fix == "show" else "-i" - - status, stdout, stderr = util.run( - f"{SCRIPTS}/license-header.py {options} -", input=files - ) - - if stdout != "": - print(stdout) - - return status - - -def tidy_command(commit, files, fix): - files = [file for file in files if regex.match(r".*\.cpp$", file)] - - if not files: - return 0 - - commit = f"--commit {commit}" if commit != "" else "" - fix = "--fix" if fix == "fix" else "" - - status, stdout, stderr = util.run( - f"{SCRIPTS}/run-clang-tidy.py {commit} {fix} -", input=files - ) - - if stdout != "": - print(stdout) - - return status - - -def get_commit(files): - if files == "commit": - return "HEAD^" - - if files == "main" or files == "master" or files == "presto-0.293-clp-connector": - return util.run(f"git merge-base origin/{files} HEAD")[1] - - return "" - - -def get_files(commit, path): - filelist = [] - - if commit != "": - status, stdout, stderr = util.run( - f"git diff --relative --name-only --diff-filter='ACMR' {commit}" - ) - filelist = stdout.splitlines() - else: - if os.path.isfile(path): - filelist.append(path) - else: - for root, dirs, files in os.walk(path): - for name in files: - filelist.append(os.path.join(root, name)) - - return [ - file - for file in filelist - if "/data/" not in file - and "velox/external/" not in file - and "build/fbcode_builder" not in file - and "build/deps" not in file - and "cmake-build-debug" not in file - and "NOTICE.txt" != file - and "velox/docs/affiliations_map.txt" != file - ] - - -def help(args): - parser.print_help() - return 0 - - -def add_check_options(subparser, name): - parser = subparser.add_parser(name) - parser.add_argument("--fix", action="store_const", default="show", const="fix") - return parser - - -def add_options(parser): - files = parser.add_subparsers(dest="files") - - tree_parser = add_check_options(files, "tree") - tree_parser.add_argument("path", default="") - - branch_parser = add_check_options(files, "main") - branch_parser = add_check_options(files, "master") - branch_parser = add_check_options(files, "presto-0.293-clp-connector") - commit_parser = add_check_options(files, "commit") - - -def add_check_command(parser, name): - subparser = parser.add_parser(name) - add_options(subparser) - - return subparser - - -def parse_args(): - global parser - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, - description="""Check format/header/tidy - - check.py {format,header,tidy} {commit,branch} [--fix] - check.py {format,header,tidy} {tree} [--fix] PATH -""", - ) - command = parser.add_subparsers(dest="command") - command.add_parser("help") - - format_command_parser = add_check_command(command, "format") - header_command_parser = add_check_command(command, "header") - tidy_command_parser = add_check_command(command, "tidy") - - parser.set_defaults(path="") - parser.set_defaults(command="help") - - return parser.parse_args() - - -def run_command(args, command): - commit = get_commit(args.files) - files = get_files(commit, args.path) - - return command(commit, files, args.fix) - - -def format(args): - return run_command(args, format_command) - - -def header(args): - return run_command(args, header_command) - - -def tidy(args): - return run_command(args, tidy_command) - - -def main(): - args = parse_args() - return globals()[args.command](args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/license-header.py b/scripts/checks/license-header.py similarity index 98% rename from scripts/license-header.py rename to scripts/checks/license-header.py index a31df3f815a..6f2ebb52e80 100755 --- a/scripts/license-header.py +++ b/scripts/checks/license-header.py @@ -28,7 +28,9 @@ class attrdict(dict): def parse_args(): parser = argparse.ArgumentParser(description="Update license headers") - parser.add_argument("--header", default="license.header", help="header file") + parser.add_argument( + "--header", default="scripts/checks/license.header", help="header file" + ) parser.add_argument( "--extra", default=80, diff --git a/license.header b/scripts/checks/license.header similarity index 100% rename from license.header rename to scripts/checks/license.header diff --git a/scripts/run-clang-tidy.py b/scripts/checks/run-clang-tidy.py similarity index 99% rename from scripts/run-clang-tidy.py rename to scripts/checks/run-clang-tidy.py index 00a80deeb19..5d7016c3d9e 100755 --- a/scripts/run-clang-tidy.py +++ b/scripts/checks/run-clang-tidy.py @@ -14,7 +14,6 @@ # limitations under the License. import argparse -from itertools import groupby import json import regex import sys diff --git a/scripts/util.py b/scripts/checks/util.py similarity index 98% rename from scripts/util.py rename to scripts/checks/util.py index 3ed836c63db..c0b34532a94 100644 --- a/scripts/util.py +++ b/scripts/checks/util.py @@ -37,7 +37,7 @@ def run(command, compressed=False, **kwargs): if "input" in kwargs: input = kwargs["input"] - if type(input) == list: + if type(input) is list: input = "\n".join(input) + "\n" kwargs["input"] = input.encode("utf-8") diff --git a/scripts/ci/benchmark-runner.py b/scripts/ci/benchmark-runner.py index f49deed1f15..8eba18282f3 100755 --- a/scripts/ci/benchmark-runner.py +++ b/scripts/ci/benchmark-runner.py @@ -81,7 +81,7 @@ def get_retry_name(args, file_name): path = _normalize_path(file_name) try: parent_path = path.relative_to(_normalize_path(args.contender_path)) - except: + except Exception: parent_path = path.relative_to(_normalize_path(args.baseline_path)) return str(parent_path.parent) @@ -455,8 +455,7 @@ def parse_args(): parser_run.add_argument( "--binary_filter", default=None, - help="Filter applied to binary names. " - "By default execute all binaries found.", + help="Filter applied to binary names. By default execute all binaries found.", ) parser_run.add_argument( "--bm_filter", diff --git a/scripts/ci/bm-report/build-metrics.py b/scripts/ci/bm-report/build-metrics.py index 3829d403845..57497f4d5b7 100755 --- a/scripts/ci/bm-report/build-metrics.py +++ b/scripts/ci/bm-report/build-metrics.py @@ -14,7 +14,6 @@ # limitations under the License. import argparse -import sys import uuid from os.path import join, splitext from pathlib import Path @@ -125,8 +124,9 @@ def _transform_results(self) -> List[BenchmarkResult]: else: del log_lines[0] - ms2sec = lambda x: x / 1000 - get_epoch = lambda l: int(l.split()[2]) + def ms2sec(x): + return x / 1000 + totals = { "link_time": 0, "compile_time": 0, diff --git a/scripts/ci/bm-report/report.qmd b/scripts/ci/bm-report/report.qmd index 0c41ae0377a..a91a116a6b9 100644 --- a/scripts/ci/bm-report/report.qmd +++ b/scripts/ci/bm-report/report.qmd @@ -222,7 +222,7 @@ searchable_table( :::: -### Debug +### Debug :::: {layout="[[50, 50],[50, 50]]" } ::: {} @@ -320,7 +320,7 @@ searchable_table( ::: ::: {} -```{r sizes-release} +```{r sizes-release} searchable_table( object_sizes_static, "release", "static", "Size", "Binary Sizes - Static" @@ -331,7 +331,7 @@ searchable_table( :::: -### Debug +### Debug :::: {layout="[50, 50]" } ::: {} @@ -353,4 +353,3 @@ searchable_table( ::: :::: - diff --git a/scripts/ci/hdfs-client.xml b/scripts/ci/hdfs-client.xml index 77b0ce61187..389b5376c0a 100644 --- a/scripts/ci/hdfs-client.xml +++ b/scripts/ci/hdfs-client.xml @@ -4,4 +4,4 @@ dfs.client.log.severity FATAL - \ No newline at end of file + diff --git a/scripts/ci/presto/etc/hive.properties b/scripts/ci/presto/etc/hive.properties index e9a0d05c76a..1ea8272de8c 100644 --- a/scripts/ci/presto/etc/hive.properties +++ b/scripts/ci/presto/etc/hive.properties @@ -1,4 +1,4 @@ connector.name=hive-hadoop2 hive.metastore=file hive.metastore.catalog.dir=file:/opt/presto-server/etc/data -hive.allow-drop-table=true \ No newline at end of file +hive.allow-drop-table=true diff --git a/scripts/ci/presto/start-prestojava.sh b/scripts/ci/presto/start-prestojava.sh index 290e43af8af..4a02636aa82 100755 --- a/scripts/ci/presto/start-prestojava.sh +++ b/scripts/ci/presto/start-prestojava.sh @@ -16,4 +16,3 @@ set -e "$PRESTO_HOME"/bin/launcher --pid-file=/tmp/pidfile run - diff --git a/scripts/ci/signature.py b/scripts/ci/signature.py index 51698a056eb..daa876942f1 100644 --- a/scripts/ci/signature.py +++ b/scripts/ci/signature.py @@ -146,7 +146,7 @@ def diff_signatures(base_signatures, contender_signatures, error_path=""): if "repetition_change" in delta: error_message = "" for rep_change in delta["repetition_change"]: - error_message += f"""'{rep_change.get_root_key()}{rep_change.t1}' is repeated {rep_change.repetition['new_repeat']} times.\n""" + error_message += f"""'{rep_change.get_root_key()}{rep_change.t1}' is repeated {rep_change.repetition["new_repeat"]} times.\n""" show_error(error_message, error_path) exit_status = 1 diff --git a/scripts/docker/pyvelox.dockerfile b/scripts/docker/pyvelox.dockerfile index c700ab371ed..5757345ef9b 100644 --- a/scripts/docker/pyvelox.dockerfile +++ b/scripts/docker/pyvelox.dockerfile @@ -28,4 +28,3 @@ RUN mkdir build && ( cd build && bash /setup-manylinux.sh ) && rm -rf build && \ dnf clean all ENV LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" - diff --git a/scripts/git-clang-format b/scripts/git-clang-format deleted file mode 100755 index 46e7f5cd0ca..00000000000 --- a/scripts/git-clang-format +++ /dev/null @@ -1,622 +0,0 @@ -#!/usr/bin/env python3 -# -#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===# -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -#============================================================================== -#LLVM Release License -#============================================================================== -#University of Illinois/NCSA -#Open Source License -# -#Copyright (c) 2003-2010 University of Illinois at Urbana-Champaign. -#All rights reserved. -# -#Developed by: -# -# LLVM Team -# -# University of Illinois at Urbana-Champaign -# -# http://llvm.org -# -#Permission is hereby granted, free of charge, to any person obtaining a copy of -#this software and associated documentation files (the "Software"), to deal with -#the Software without restriction, including without limitation the rights to -#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -#of the Software, and to permit persons to whom the Software is furnished to do -#so, subject to the following conditions: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimers. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimers in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of the LLVM Team, University of Illinois at -# Urbana-Champaign, nor the names of its contributors may be used to -# endorse or promote products derived from this Software without specific -# prior written permission. -# -#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -#CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE -#SOFTWARE. -#===------------------------------------------------------------------------===# - -r""" -clang-format git integration -============================ - -This file provides a clang-format integration for git. Put it somewhere in your -path and ensure that it is executable. Then, "git clang-format" will invoke -clang-format on the changes in current files or a specific commit. - -For further details, run: -git clang-format -h - -Requires Python 2.7 or Python 3 -""" - -from __future__ import absolute_import, division, print_function -import argparse -import collections -import contextlib -import errno -import os -import re -import subprocess -import sys - -usage = 'git clang-format [OPTIONS] [] [] [--] [...]' - -desc = ''' -If zero or one commits are given, run clang-format on all lines that differ -between the working directory and , which defaults to HEAD. Changes are -only applied to the working directory. - -If two commits are given (requires --diff), run clang-format on all lines in the -second that differ from the first . - -The following git-config settings set the default of the corresponding option: - clangFormat.binary - clangFormat.commit - clangFormat.extension - clangFormat.style -''' - -# Name of the temporary index file in which save the output of clang-format. -# This file is created within the .git directory. -temp_index_basename = 'clang-format-index' - - -Range = collections.namedtuple('Range', 'start, count') - - -def main(): - config = load_git_config() - - # In order to keep '--' yet allow options after positionals, we need to - # check for '--' ourselves. (Setting nargs='*' throws away the '--', while - # nargs=argparse.REMAINDER disallows options after positionals.) - argv = sys.argv[1:] - try: - idx = argv.index('--') - except ValueError: - dash_dash = [] - else: - dash_dash = argv[idx:] - argv = argv[:idx] - - default_extensions = ','.join([ - # From clang/lib/Frontend/FrontendOptions.cpp, all lower case - 'c', 'h', # C - 'm', # ObjC - 'mm', # ObjC++ - 'cc', 'cp', 'cpp', 'c++', 'cxx', 'hpp', # C++ - 'cu', # CUDA - # Other languages that clang-format supports - 'proto', 'protodevel', # Protocol Buffers - 'java', # Java - 'js', # JavaScript - 'ts', # TypeScript - ]) - - p = argparse.ArgumentParser( - usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter, - description=desc) - p.add_argument('--binary', - default=config.get('clangformat.binary', 'clang-format'), - help='path to clang-format'), - p.add_argument('--commit', - default=config.get('clangformat.commit', 'HEAD'), - help='default commit to use if none is specified'), - p.add_argument('--diff', action='store_true', - help='print a diff instead of applying the changes') - p.add_argument('--extensions', - default=config.get('clangformat.extensions', - default_extensions), - help=('comma-separated list of file extensions to format, ' - 'excluding the period and case-insensitive')), - p.add_argument('-f', '--force', action='store_true', - help='allow changes to unstaged files') - p.add_argument('-p', '--patch', action='store_true', - help='select hunks interactively') - p.add_argument('-q', '--quiet', action='count', default=0, - help='print less information') - p.add_argument('--style', - default=config.get('clangformat.style', None), - help='passed to clang-format'), - p.add_argument('-v', '--verbose', action='count', default=0, - help='print extra information') - # We gather all the remaining positional arguments into 'args' since we need - # to use some heuristics to determine whether or not was present. - # However, to print pretty messages, we make use of metavar and help. - p.add_argument('args', nargs='*', metavar='', - help='revision from which to compute the diff') - p.add_argument('ignored', nargs='*', metavar='...', - help='if specified, only consider differences in these files') - opts = p.parse_args(argv) - - opts.verbose -= opts.quiet - del opts.quiet - - commits, files = interpret_args(opts.args, dash_dash, opts.commit) - if len(commits) > 1: - if not opts.diff: - die('--diff is required when two commits are given') - else: - if len(commits) > 2: - die('at most two commits allowed; %d given' % len(commits)) - changed_lines = compute_diff_and_extract_lines(commits, files) - if opts.verbose >= 1: - ignored_files = set(changed_lines) - filter_by_extension(changed_lines, opts.extensions.lower().split(',')) - if opts.verbose >= 1: - ignored_files.difference_update(changed_lines) - if ignored_files: - print('Ignoring changes in the following files (wrong extension):') - for filename in ignored_files: - print(' %s' % filename) - if changed_lines: - print('Running clang-format on the following files:') - for filename in changed_lines: - print(' %s' % filename) - if not changed_lines: - print('no modified files to format') - return - # The computed diff outputs absolute paths, so we must cd before accessing - # those files. - cd_to_toplevel() - if len(commits) > 1: - old_tree = commits[1] - new_tree = run_clang_format_and_save_to_tree(changed_lines, - revision=commits[1], - binary=opts.binary, - style=opts.style) - else: - old_tree = create_tree_from_workdir(changed_lines) - new_tree = run_clang_format_and_save_to_tree(changed_lines, - binary=opts.binary, - style=opts.style) - if opts.verbose >= 1: - print('old tree: %s' % old_tree) - print('new tree: %s' % new_tree) - if old_tree == new_tree: - if opts.verbose >= 0: - print('clang-format did not modify any files') - elif opts.diff: - print_diff(old_tree, new_tree) - else: - changed_files = apply_changes(old_tree, new_tree, force=opts.force, - patch_mode=opts.patch) - if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1: - print('changed files:') - for filename in changed_files: - print(' %s' % filename) - - -def load_git_config(non_string_options=None): - """Return the git configuration as a dictionary. - - All options are assumed to be strings unless in `non_string_options`, in which - is a dictionary mapping option name (in lower case) to either "--bool" or - "--int".""" - if non_string_options is None: - non_string_options = {} - out = {} - for entry in run('git', 'config', '--list', '--null').split('\0'): - if entry: - name, value = entry.split('\n', 1) - if name in non_string_options: - value = run('git', 'config', non_string_options[name], name) - out[name] = value - return out - - -def interpret_args(args, dash_dash, default_commit): - """Interpret `args` as "[commits] [--] [files]" and return (commits, files). - - It is assumed that "--" and everything that follows has been removed from - args and placed in `dash_dash`. - - If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its - left (if present) are taken as commits. Otherwise, the arguments are checked - from left to right if they are commits or files. If commits are not given, - a list with `default_commit` is used.""" - if dash_dash: - if len(args) == 0: - commits = [default_commit] - else: - commits = args - for commit in commits: - object_type = get_object_type(commit) - if object_type not in ('commit', 'tag'): - if object_type is None: - die("'%s' is not a commit" % commit) - else: - die("'%s' is a %s, but a commit was expected" % (commit, object_type)) - files = dash_dash[1:] - elif args: - commits = [] - while args: - if not disambiguate_revision(args[0]): - break - commits.append(args.pop(0)) - if not commits: - commits = [default_commit] - files = args - else: - commits = [default_commit] - files = [] - return commits, files - - -def disambiguate_revision(value): - """Returns True if `value` is a revision, False if it is a file, or dies.""" - # If `value` is ambiguous (neither a commit nor a file), the following - # command will die with an appropriate error message. - run('git', 'rev-parse', value, verbose=False) - object_type = get_object_type(value) - if object_type is None: - return False - if object_type in ('commit', 'tag'): - return True - die('`%s` is a %s, but a commit or filename was expected' % - (value, object_type)) - - -def get_object_type(value): - """Returns a string description of an object's type, or None if it is not - a valid git object.""" - cmd = ['git', 'cat-file', '-t', value] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - if p.returncode != 0: - return None - return convert_string(stdout.strip()) - - -def compute_diff_and_extract_lines(commits, files): - """Calls compute_diff() followed by extract_lines().""" - diff_process = compute_diff(commits, files) - changed_lines = extract_lines(diff_process.stdout) - diff_process.stdout.close() - diff_process.wait() - if diff_process.returncode != 0: - # Assume error was already printed to stderr. - sys.exit(2) - return changed_lines - - -def compute_diff(commits, files): - """Return a subprocess object producing the diff from `commits`. - - The return value's `stdin` file object will produce a patch with the - differences between the working directory and the first commit if a single - one was specified, or the difference between both specified commits, filtered - on `files` (if non-empty). Zero context lines are used in the patch.""" - git_tool = 'diff-index' - if len(commits) > 1: - git_tool = 'diff-tree' - cmd = ['git', git_tool, '-p', '-U0'] + commits + ['--'] - cmd.extend(files) - p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - p.stdin.close() - return p - - -def extract_lines(patch_file): - """Extract the changed lines in `patch_file`. - - The return value is a dictionary mapping filename to a list of (start_line, - line_count) pairs. - - The input must have been produced with ``-U0``, meaning unidiff format with - zero lines of context. The return value is a dict mapping filename to a - list of line `Range`s.""" - matches = {} - for line in patch_file: - line = convert_string(line) - match = re.search(r'^\+\+\+\ [^/]+/(.*)', line) - if match: - filename = match.group(1).rstrip('\r\n') - match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line) - if match: - start_line = int(match.group(1)) - line_count = 1 - if match.group(3): - line_count = int(match.group(3)) - if line_count > 0: - matches.setdefault(filename, []).append(Range(start_line, line_count)) - return matches - - -def filter_by_extension(dictionary, allowed_extensions): - """Delete every key in `dictionary` that doesn't have an allowed extension. - - `allowed_extensions` must be a collection of lowercase file extensions, - excluding the period.""" - allowed_extensions = frozenset(allowed_extensions) - for filename in list(dictionary.keys()): - base_ext = filename.rsplit('.', 1) - if len(base_ext) == 1 and '' in allowed_extensions: - continue - if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions: - del dictionary[filename] - - -def cd_to_toplevel(): - """Change to the top level of the git repository.""" - toplevel = run('git', 'rev-parse', '--show-toplevel') - os.chdir(toplevel) - - -def create_tree_from_workdir(filenames): - """Create a new git tree with the given files from the working directory. - - Returns the object ID (SHA-1) of the created tree.""" - return create_tree(filenames, '--stdin') - - -def run_clang_format_and_save_to_tree(changed_lines, revision=None, - binary='clang-format', style=None): - """Run clang-format on each file and save the result to a git tree. - - Returns the object ID (SHA-1) of the created tree.""" - def iteritems(container): - try: - return container.iteritems() # Python 2 - except AttributeError: - return container.items() # Python 3 - def index_info_generator(): - for filename, line_ranges in iteritems(changed_lines): - if revision: - git_metadata_cmd = ['git', 'ls-tree', - '%s:%s' % (revision, os.path.dirname(filename)), - os.path.basename(filename)] - git_metadata = subprocess.Popen(git_metadata_cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - stdout = git_metadata.communicate()[0] - mode = oct(int(stdout.split()[0], 8)) - else: - mode = oct(os.stat(filename).st_mode) - # Adjust python3 octal format so that it matches what git expects - if mode.startswith('0o'): - mode = '0' + mode[2:] - blob_id = clang_format_to_blob(filename, line_ranges, - revision=revision, - binary=binary, - style=style) - yield '%s %s\t%s' % (mode, blob_id, filename) - return create_tree(index_info_generator(), '--index-info') - - -def create_tree(input_lines, mode): - """Create a tree object from the given input. - - If mode is '--stdin', it must be a list of filenames. If mode is - '--index-info' is must be a list of values suitable for "git update-index - --index-info", such as " ". Any other mode - is invalid.""" - assert mode in ('--stdin', '--index-info') - cmd = ['git', 'update-index', '--add', '-z', mode] - with temporary_index_file(): - p = subprocess.Popen(cmd, stdin=subprocess.PIPE) - for line in input_lines: - p.stdin.write(to_bytes('%s\0' % line)) - p.stdin.close() - if p.wait() != 0: - die('`%s` failed' % ' '.join(cmd)) - tree_id = run('git', 'write-tree') - return tree_id - - -def clang_format_to_blob(filename, line_ranges, revision=None, - binary='clang-format', style=None): - """Run clang-format on the given file and save the result to a git blob. - - Runs on the file in `revision` if not None, or on the file in the working - directory if `revision` is None. - - Returns the object ID (SHA-1) of the created blob.""" - clang_format_cmd = [binary] - if style: - clang_format_cmd.extend(['-style='+style]) - clang_format_cmd.extend([ - '-lines=%s:%s' % (start_line, start_line+line_count-1) - for start_line, line_count in line_ranges]) - if revision: - clang_format_cmd.extend(['-assume-filename='+filename]) - git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)] - git_show = subprocess.Popen(git_show_cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - git_show.stdin.close() - clang_format_stdin = git_show.stdout - else: - clang_format_cmd.extend([filename]) - git_show = None - clang_format_stdin = subprocess.PIPE - try: - clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin, - stdout=subprocess.PIPE) - if clang_format_stdin == subprocess.PIPE: - clang_format_stdin = clang_format.stdin - except OSError as e: - if e.errno == errno.ENOENT: - die('cannot find executable "%s"' % binary) - else: - raise - clang_format_stdin.close() - hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin'] - hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout, - stdout=subprocess.PIPE) - clang_format.stdout.close() - stdout = hash_object.communicate()[0] - if hash_object.returncode != 0: - die('`%s` failed' % ' '.join(hash_object_cmd)) - if clang_format.wait() != 0: - die('`%s` failed' % ' '.join(clang_format_cmd)) - if git_show and git_show.wait() != 0: - die('`%s` failed' % ' '.join(git_show_cmd)) - return convert_string(stdout).rstrip('\r\n') - - -@contextlib.contextmanager -def temporary_index_file(tree=None): - """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting - the file afterward.""" - index_path = create_temporary_index(tree) - old_index_path = os.environ.get('GIT_INDEX_FILE') - os.environ['GIT_INDEX_FILE'] = index_path - try: - yield - finally: - if old_index_path is None: - del os.environ['GIT_INDEX_FILE'] - else: - os.environ['GIT_INDEX_FILE'] = old_index_path - os.remove(index_path) - - -def create_temporary_index(tree=None): - """Create a temporary index file and return the created file's path. - - If `tree` is not None, use that as the tree to read in. Otherwise, an - empty index is created.""" - gitdir = run('git', 'rev-parse', '--git-dir') - path = os.path.join(gitdir, temp_index_basename) - if tree is None: - tree = '--empty' - run('git', 'read-tree', '--index-output='+path, tree) - return path - - -def print_diff(old_tree, new_tree): - """Print the diff between the two trees to stdout.""" - # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output - # is expected to be viewed by the user, and only the former does nice things - # like color and pagination. - # - # We also only print modified files since `new_tree` only contains the files - # that were modified, so unmodified files would show as deleted without the - # filter. - subprocess.check_call(['git', 'diff', '--diff-filter=M', old_tree, new_tree, - '--']) - - -def apply_changes(old_tree, new_tree, force=False, patch_mode=False): - """Apply the changes in `new_tree` to the working directory. - - Bails if there are local changes in those files and not `force`. If - `patch_mode`, runs `git checkout --patch` to select hunks interactively.""" - changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z', - '--name-only', old_tree, - new_tree).rstrip('\0').split('\0') - if not force: - unstaged_files = run('git', 'diff-files', '--name-status', *changed_files) - if unstaged_files: - print('The following files would be modified but ' - 'have unstaged changes:', file=sys.stderr) - print(unstaged_files, file=sys.stderr) - print('Please commit, stage, or stash them first.', file=sys.stderr) - sys.exit(2) - if patch_mode: - # In patch mode, we could just as well create an index from the new tree - # and checkout from that, but then the user will be presented with a - # message saying "Discard ... from worktree". Instead, we use the old - # tree as the index and checkout from new_tree, which gives the slightly - # better message, "Apply ... to index and worktree". This is not quite - # right, since it won't be applied to the user's index, but oh well. - with temporary_index_file(old_tree): - subprocess.check_call(['git', 'checkout', '--patch', new_tree]) - index_tree = old_tree - else: - with temporary_index_file(new_tree): - run('git', 'checkout-index', '-a', '-f') - return changed_files - - -def run(*args, **kwargs): - stdin = kwargs.pop('stdin', '') - verbose = kwargs.pop('verbose', True) - strip = kwargs.pop('strip', True) - for name in kwargs: - raise TypeError("run() got an unexpected keyword argument '%s'" % name) - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE) - stdout, stderr = p.communicate(input=stdin) - - stdout = convert_string(stdout) - stderr = convert_string(stderr) - - if p.returncode == 0: - if stderr: - if verbose: - print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr) - print(stderr.rstrip(), file=sys.stderr) - if strip: - stdout = stdout.rstrip('\r\n') - return stdout - if verbose: - print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr) - if stderr: - print(stderr.rstrip(), file=sys.stderr) - sys.exit(2) - - -def die(message): - print('error:', message, file=sys.stderr) - sys.exit(2) - - -def to_bytes(str_input): - # Encode to UTF-8 to get binary data. - if isinstance(str_input, bytes): - return str_input - return str_input.encode('utf-8') - - -def to_string(bytes_input): - if isinstance(bytes_input, str): - return bytes_input - return bytes_input.encode('utf-8') - - -def convert_string(bytes_input): - try: - return to_string(bytes_input.decode('utf-8')) - except AttributeError: # 'str' object has no attribute 'decode'. - return str(bytes_input) - except UnicodeError: - return str(bytes_input) - -if __name__ == '__main__': - main() diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 008f9302162..c0cd353fb66 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -194,4 +194,3 @@ function install_velox_deps { dnf clean all fi ) - diff --git a/scripts/setup-classpath.sh b/scripts/setup-classpath.sh index e52184d9213..bfd7066dc63 100644 --- a/scripts/setup-classpath.sh +++ b/scripts/setup-classpath.sh @@ -1,4 +1,3 @@ -#!/bin/bash # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh index 7f7cdf10640..d5867e73678 100755 --- a/scripts/setup-helper-functions.sh +++ b/scripts/setup-helper-functions.sh @@ -234,4 +234,3 @@ function cmake_install { cmake --build "${BINARY_DIR}" "-j ${NPROC}" || { echo 'build failed' ; exit 1; } ${SUDO} cmake --install "${BINARY_DIR}" } - diff --git a/scripts/setup-manylinux.sh b/scripts/setup-manylinux.sh old mode 100644 new mode 100755 diff --git a/scripts/velox_env_linux.yml b/scripts/velox_env_linux.yml index 59ceeb0adb4..58e722cee5a 100644 --- a/scripts/velox_env_linux.yml +++ b/scripts/velox_env_linux.yml @@ -22,7 +22,7 @@ variables: CXX: clang++ dependencies: -# tools + # tools - binutils - bison - clangxx=14 @@ -37,7 +37,7 @@ dependencies: - openjdk=8.* - python=3.8 - sysroot_linux-64=2.17 -# dependencies + # dependencies - aws-sdk-cpp - azure-identity-cpp - azure-storage-blobs-cpp diff --git a/scripts/velox_env_mac.yml b/scripts/velox_env_mac.yml index 776247a41f0..8c24af8e31a 100644 --- a/scripts/velox_env_mac.yml +++ b/scripts/velox_env_mac.yml @@ -22,7 +22,7 @@ variables: CXX: clang++ dependencies: -# tools + # tools - binutils - bison - clangxx=14 # pin to something recent'ish to avoid warings on upgrade @@ -36,7 +36,7 @@ dependencies: - openjdk=8.* - python=3.8 - sysroot_linux-64=2.17 -# dependencies + # dependencies - aws-sdk-cpp - azure-identity-cpp - azure-storage-blobs-cpp @@ -64,4 +64,3 @@ dependencies: - xz - zlib - zstd - diff --git a/static/icon.svg b/static/icon.svg index d3e7a794ddc..8ceed2d92a5 100644 --- a/static/icon.svg +++ b/static/icon.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/static/logo.svg b/static/logo.svg index 67627d4cfd2..0db7eec5f1f 100644 --- a/static/logo.svg +++ b/static/logo.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/velox/docs/bindings/python/arrow.rst b/velox/docs/bindings/python/arrow.rst index 716617a7833..888c91679dd 100644 --- a/velox/docs/bindings/python/arrow.rst +++ b/velox/docs/bindings/python/arrow.rst @@ -5,4 +5,4 @@ Pyvelox Arrow Api .. autofunction:: pyvelox.arrow.to_velox -.. autofunction:: pyvelox.arrow.to_arrow \ No newline at end of file +.. autofunction:: pyvelox.arrow.to_arrow diff --git a/velox/docs/bindings/python/index.rst b/velox/docs/bindings/python/index.rst index 4bf8155e62c..7240f77719d 100644 --- a/velox/docs/bindings/python/index.rst +++ b/velox/docs/bindings/python/index.rst @@ -13,6 +13,3 @@ Pyvelox Documentation runners file legacy - - - diff --git a/velox/docs/bindings/python/legacy.rst b/velox/docs/bindings/python/legacy.rst index f4c8a746a80..dddcdbe2d30 100644 --- a/velox/docs/bindings/python/legacy.rst +++ b/velox/docs/bindings/python/legacy.rst @@ -9,4 +9,4 @@ Pyvelox Legacy Api .. autoclass:: pyvelox.legacy::BaseVector :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/velox/docs/bindings/python/runners.rst b/velox/docs/bindings/python/runners.rst index a8f7251aae4..401b2f45261 100644 --- a/velox/docs/bindings/python/runners.rst +++ b/velox/docs/bindings/python/runners.rst @@ -5,4 +5,4 @@ Pyvelox Runners .. autoclass:: pyvelox.runner.LocalRunner :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/velox/docs/bindings/python/vector.rst b/velox/docs/bindings/python/vector.rst index 4c8005b76aa..b4fca9604c7 100644 --- a/velox/docs/bindings/python/vector.rst +++ b/velox/docs/bindings/python/vector.rst @@ -5,4 +5,4 @@ Pyvelox Vectors .. autoclass:: pyvelox.vector.Vector :members: - :special-members: \ No newline at end of file + :special-members: diff --git a/velox/docs/conf.py b/velox/docs/conf.py index 1fecf2f68e6..d9dd62e7450 100644 --- a/velox/docs/conf.py +++ b/velox/docs/conf.py @@ -30,7 +30,7 @@ try: sys.dont_write_bytecode = True -except: +except: # noqa E722 pass sys.path.insert(0, os.path.abspath("ext")) diff --git a/velox/docs/credits.py b/velox/docs/credits.py old mode 100644 new mode 100755 diff --git a/velox/docs/develop/aggregations.rst b/velox/docs/develop/aggregations.rst index bb776e9cefe..934252f7c14 100644 --- a/velox/docs/develop/aggregations.rst +++ b/velox/docs/develop/aggregations.rst @@ -118,9 +118,9 @@ Push-Down into Table Scan HashAggregation operator supports pushing down aggregations into table scan. Pushdown is enabled when all of the following conditions are met: -* the aggregation function takes a single argument, -* the argument is a column read directly from the table without any transformations, -* that column is not used anywhere else in the query. +* the aggregation function takes a single argument, +* the argument is a column read directly from the table without any transformations, +* that column is not used anywhere else in the query. For example, pushdown is possible in the following query: @@ -134,9 +134,9 @@ enabled in the following query: .. code-block:: sql - SELECT a, sum(b) - FROM t - WHERE a > 100 + SELECT a, sum(b) + FROM t + WHERE a > 100 GROUP BY 1 In these queries, TableScan operator produces "b" column as a LazyVector @@ -294,7 +294,7 @@ After receiving at least abandon_partial_aggregation_min_rows input rows, the operator checks the percentage of input rows that are unique, e.g. compares number of groups with number of input rows. If percentage of unique rows exceeds abandon_partial_aggregation_min_pct, the operator abandons partial -aggregation. +aggregation. It is not possible to simply stop aggregating inputs and pass these as is to shuffle and final aggregation because final aggregation expects data type that diff --git a/velox/docs/develop/debugging/print-plan-with-stats.rst b/velox/docs/develop/debugging/print-plan-with-stats.rst index 547feb93d47..3bb482763c6 100644 --- a/velox/docs/develop/debugging/print-plan-with-stats.rst +++ b/velox/docs/develop/debugging/print-plan-with-stats.rst @@ -283,4 +283,3 @@ TableScan operator shows how many rows were processed by pushing down aggregatio .. code-block:: loadedToValueHook sum: 50000, count: 5, min: 10000, max: 10000 - diff --git a/velox/docs/develop/dynamic-loading.rst b/velox/docs/develop/dynamic-loading.rst index c6ff23420da..d720eee5ce3 100644 --- a/velox/docs/develop/dynamic-loading.rst +++ b/velox/docs/develop/dynamic-loading.rst @@ -12,7 +12,7 @@ Getting Started 1. **Create a C++ file for your dynamic library** - For dynamically loaded function registration, the format followed mirrors that of built-in function registration with some noted differences. Using `DynamicTestFunction.cpp` as an example, the function uses the `extern "C"` keyword to protect against name mangling. + For dynamically loaded function registration, the format followed mirrors that of built-in function registration with some noted differences. Using `DynamicTestFunction.cpp` as an example, the function uses the `extern "C"` keyword to protect against name mangling. The `registrationFunctionName` function here acts as the entrypoint for the dynamic library for loading symbols. The `registrationFunctionName` function name is customizable and defaults to `registerExtensions` when not specified in the library loading call. Make sure to also include the necessary header file: @@ -77,4 +77,3 @@ Notes - In Velox, a function's signature is determined solely by its name and argument types. The return type is not taken into account. As a result, if a function with an identical signature is added but with a different return type, it will overwrite the existing function. - Function overloading is supported. Therefore, multiple functions can share the same name as long as they differ in the number or types of arguments. - diff --git a/velox/docs/develop/operators.rst b/velox/docs/develop/operators.rst index b8bbf02fa3e..9af21bcdacd 100644 --- a/velox/docs/develop/operators.rst +++ b/velox/docs/develop/operators.rst @@ -316,7 +316,7 @@ constructor within the Project operation. * - names - A list of new column names. -ExpandNode is typically used to compute GROUPING SETS, CUBE, ROLLUP and COUNT DISTINCT. +ExpandNode is typically used to compute GROUPING SETS, CUBE, ROLLUP and COUNT DISTINCT. To illustrate how ExpandNode works lets examine the following SQL query: @@ -347,7 +347,7 @@ After the computation by the ExpandNode, each row will generate 3 rows of data. .. code-block:: - l_suppkey l_orderkey l_partkey grouping_id_0 + l_suppkey l_orderkey l_partkey grouping_id_0 93 1 673 0 93 1 null 1 93 null null 3 @@ -389,15 +389,15 @@ For example, if the input rows are: .. code-block:: l_suppkey l_partkey - 93 673 - 75 674 + 93 673 + 75 674 38 22 After the computation by the ExpandNode, each row will generate 2 rows of data. So there will be a total of 6 rows: .. code-block:: - l_suppkey l_partkey grouping_id_0 + l_suppkey l_partkey grouping_id_0 93 null 1 null 673 2 75 null 1 @@ -409,7 +409,7 @@ Aggregation operator that follows, groups these rows by (l_suppkey, l_partkey, g .. code-block:: - l_suppkey l_partkey grouping_id_0 + l_suppkey l_partkey grouping_id_0 93 null 1 75 null 1 38 null 1 diff --git a/velox/docs/ext/issue.py b/velox/docs/ext/issue.py index ac95246c607..b80be34c95a 100644 --- a/velox/docs/ext/issue.py +++ b/velox/docs/ext/issue.py @@ -13,7 +13,7 @@ # limitations under the License. # noinspection PyUnresolvedReferences -from docutils import nodes, utils +from docutils import nodes # noinspection PyDefaultArgument,PyUnusedLocal diff --git a/velox/docs/ext/pr.py b/velox/docs/ext/pr.py index bba2e50dd1d..745499b262c 100644 --- a/velox/docs/ext/pr.py +++ b/velox/docs/ext/pr.py @@ -13,7 +13,7 @@ # limitations under the License. # noinspection PyUnresolvedReferences -from docutils import nodes, utils +from docutils import nodes # noinspection PyDefaultArgument,PyUnusedLocal diff --git a/velox/docs/ext/spark.py b/velox/docs/ext/spark.py index eede6354374..134a882db99 100644 --- a/velox/docs/ext/spark.py +++ b/velox/docs/ext/spark.py @@ -653,7 +653,7 @@ def add_target_and_index( text = _("%s() (in module %s)") % (name, modname) self.indexnode["entries"].append(("single", text, node_id, "", None)) else: - text = f'{pairindextypes["builtin"]}; {name}()' + text = f"{pairindextypes['builtin']}; {name}()" self.indexnode["entries"].append(("pair", text, node_id, "", None)) def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str | None: diff --git a/velox/docs/functions/presto/binary.rst b/velox/docs/functions/presto/binary.rst index 334b8d91431..1e9ee252b13 100644 --- a/velox/docs/functions/presto/binary.rst +++ b/velox/docs/functions/presto/binary.rst @@ -77,12 +77,12 @@ Binary Functions .. function:: lpad(binary, size, padbinary) -> varbinary :noindex: - + Left pads ``binary`` to ``size`` bytes with ``padbinary``. If ``size`` is less than the length of ``binary``, the result is truncated to ``size`` characters. ``size`` must not be negative and ``padbinary`` must be non-empty. ``size`` has a maximum value of 1 MiB. - In the case of ``size`` being smaller than the length of ``binary``, + In the case of ``size`` being smaller than the length of ``binary``, ``binary`` will be truncated from the right to fit the ``size``. .. function:: md5(binary) -> varbinary @@ -96,9 +96,9 @@ Binary Functions If ``size`` is less than the length of ``binary``, the result is truncated to ``size`` characters. ``size`` must not be negative and ``padbinary`` must be non-empty. ``size`` has a maximum value of 1 MiB. - In the case of ``size`` being smaller than the length of ``binary``, + In the case of ``size`` being smaller than the length of ``binary``, ``binary`` will be truncated from the right to fit the ``size``. - + .. function:: sha1(binary) -> varbinary Computes the SHA-1 hash of ``binary``. diff --git a/velox/docs/functions/presto/window.rst b/velox/docs/functions/presto/window.rst index 6d25073cb67..56c6b90bbee 100644 --- a/velox/docs/functions/presto/window.rst +++ b/velox/docs/functions/presto/window.rst @@ -168,4 +168,4 @@ Aggregate functions ___________________ All aggregate functions can be used as window functions by adding the OVER clause. The aggregate function is computed -for each row over the rows within the current row's window frame. \ No newline at end of file +for each row over the rows within the current row's window frame. diff --git a/velox/docs/functions/spark/aggregate.rst b/velox/docs/functions/spark/aggregate.rst index 3abf81411eb..a9ac2bd001c 100644 --- a/velox/docs/functions/spark/aggregate.rst +++ b/velox/docs/functions/spark/aggregate.rst @@ -26,7 +26,7 @@ General Aggregate Functions Creates bloom filter from input hashes and returns it serialized into VARBINARY. The caller is expected to apply xxhash64 function to input data before calling bloom_filter_agg. - For example, + For example, bloom_filter_agg(xxhash64(x), 100, 1024) In Spark implementation, ``estimatedNumItems`` and ``numBits`` are used to decide the number of hash functions and bloom filter capacity. In Velox implementation, ``estimatedNumItems`` is not used. @@ -48,7 +48,7 @@ General Aggregate Functions But Spark allows for changing the defaults while Velox does not. .. spark:function:: bloom_filter_agg(hash) -> varbinary - + A version of ``bloom_filter_agg`` that use the value of spark.bloom_filter.max_num_bits configuration property as ``numBits``. ``hash`` cannot be null. @@ -162,11 +162,11 @@ General Aggregate Functions Returns the most frequent value for the values within ``x``. NULL values are ignored. If all the values are NULL, or there are 0 rows, returns NULL. - If multiple values have the same greatest frequency, the + If multiple values have the same greatest frequency, the return value could be any one of them. Example:: - + SELECT mode(x) FROM ( VALUES diff --git a/velox/docs/functions/spark/array.rst b/velox/docs/functions/spark/array.rst index 09eedad9fd2..88689787c3b 100644 --- a/velox/docs/functions/spark/array.rst +++ b/velox/docs/functions/spark/array.rst @@ -262,7 +262,7 @@ Array Functions .. spark:function:: shuffle(array(E), seed) -> array(E) - Generates a random permutation of the given ``array`` using a seed derived + Generates a random permutation of the given ``array`` using a seed derived from the parameter ``seed`` and the configuration `spark.partition_id`. ``seed`` must be constant. :: diff --git a/velox/docs/functions/spark/binary.rst b/velox/docs/functions/spark/binary.rst index 48ce669467f..ab594134d87 100644 --- a/velox/docs/functions/spark/binary.rst +++ b/velox/docs/functions/spark/binary.rst @@ -32,9 +32,9 @@ Binary Functions .. spark:function:: might_contain(bloomFilter, value) -> boolean - Returns TRUE if ``bloomFilter`` might contain ``value``. + Returns TRUE if ``bloomFilter`` might contain ``value``. - ``bloomFilter`` is a VARBINARY computed using ::spark:function::`bloom_filter_agg` aggregate function. + ``bloomFilter`` is a VARBINARY computed using ::spark:function::`bloom_filter_agg` aggregate function. ``value`` is a BIGINT. .. spark:function:: sha1(x) -> varchar diff --git a/velox/docs/functions/spark/bitwise.rst b/velox/docs/functions/spark/bitwise.rst index 5924b36aa42..77828486424 100644 --- a/velox/docs/functions/spark/bitwise.rst +++ b/velox/docs/functions/spark/bitwise.rst @@ -4,7 +4,7 @@ Bitwise Functions .. spark:function:: bitwise_and(x, y) -> [same as input] - Returns the bitwise AND of ``x`` and ``y`` in 2's complement representation. + Returns the bitwise AND of ``x`` and ``y`` in 2's complement representation. Corresponds to Spark's operator ``&``. Supported types are: TINYINT, SMALLINT, INTEGER and BIGINT. @@ -47,4 +47,4 @@ Bitwise Functions .. spark:function:: shiftright(x, n) -> [same as x] - Returns x bitwise right shifted by n bits. Supported types for 'x' are INTEGER and BIGINT. \ No newline at end of file + Returns x bitwise right shifted by n bits. Supported types for 'x' are INTEGER and BIGINT. diff --git a/velox/docs/functions/spark/comparison.rst b/velox/docs/functions/spark/comparison.rst index 9b62e68c918..7dc95393029 100644 --- a/velox/docs/functions/spark/comparison.rst +++ b/velox/docs/functions/spark/comparison.rst @@ -27,7 +27,7 @@ Comparison Functions Returns true if x is equal to y. Supports all scalar and complex types. The types of x and y must be the same. Corresponds to Spark's operators ``=`` and ``==``. Returns NULL for any NULL input, but nested nulls are compared as values. :: - + SELECT equalto(null, null); -- null SELECT equalto(null, ARRAY[1]); -- null SELECT equalto(ARRAY[1, null], ARRAY[1, null]); -- true @@ -44,7 +44,7 @@ Comparison Functions .. spark:function:: greatest(value1, value2, ..., valueN) -> [same as input] - Returns the largest of the provided values ignoring nulls. Supports all scalar types. + Returns the largest of the provided values ignoring nulls. Supports all scalar types. The types of all arguments must be the same. :: SELECT greatest(10, 9, 2, 4, 3); -- 10 diff --git a/velox/docs/functions/spark/coverage.rst b/velox/docs/functions/spark/coverage.rst index 4bacd7f1ec2..f721415c82a 100644 --- a/velox/docs/functions/spark/coverage.rst +++ b/velox/docs/functions/spark/coverage.rst @@ -81,73 +81,73 @@ ========================================= ========================================= ========================================= ========================================= ========================================= == ========================================= == ========================================= Scalar Functions Aggregate Functions Window Functions ===================================================================================================================================================================================================================== == ========================================= == ========================================= - :spark:func:`abs` count_if inline nvl sqrt any cume_dist - :spark:func:`acos count_min_sketch inline_outer nvl2 stack approx_count_distinct dense_rank - :spark:func:`acosh` covar_pop input_file_block_length octet_length std approx_percentile first_value - add_months covar_samp input_file_block_start or stddev array_agg lag - :spark:func:`aggregate` crc32 input_file_name overlay stddev_pop avg last_value - and cume_dist :spark:func:`instr` parse_url stddev_samp bit_and lead - any current_catalog int percent_rank str_to_map bit_or :spark:func:`nth_value` - approx_count_distinct current_database isnan percentile string :spark:func:`bit_xor` ntile - approx_percentile current_date :spark:func:`isnotnull` percentile_approx struct bool_and percent_rank - :spark:func:`array` current_timestamp :spark:func:`isnull` pi substr bool_or rank - :spark:func:`array_contains` current_timezone java_method :spark:func:`pmod` :spark:func:`substring` collect_list row_number - array_distinct current_user json_array_length posexplode substring_index collect_set - array_except date json_object_keys posexplode_outer sum corr - :spark:func:`array_intersect` date_add json_tuple position tan count - array_join date_format kurtosis positive tanh count_if - array_max date_from_unix_date lag pow timestamp count_min_sketch - array_min date_part last :spark:func:`power` timestamp_micros covar_pop - array_position date_sub last_day printf timestamp_millis covar_samp - array_remove date_trunc last_value quarter timestamp_seconds every - array_repeat datediff lcase radians tinyint :spark:func:`first` - :spark:func:`array_sort` day lead raise_error to_csv first_value - array_union dayofmonth :spark:func:`least` :spark:func:`rand` to_date grouping - arrays_overlap dayofweek :spark:func:`left` randn to_json grouping_id - arrays_zip dayofyear :spark:func:`length` random to_timestamp histogram_numeric - :spark:func:`ascii` decimal levenshtein range :spark:func:`to_unix_timestamp` kurtosis - asin decode like rank to_utc_timestamp :spark:func:`last` - :spark:func:`asinh` degrees ln reflect :spark:func:`transform` last_value - assert_true dense_rank locate regexp transform_keys max - atan div log :spark:func:`regexp_extract` transform_values max_by - atan2 double log10 regexp_extract_all translate mean - :spark:func:`atanh` e :spark:func:`log1p` regexp_like :spark:func:`trim` min - avg :spark:func:`element_at` log2 regexp_replace trunc min_by - base64 elt :spark:func:`lower` repeat try_add percentile - :spark:func:`between` encode lpad :spark:func:`replace` try_divide percentile_approx - bigint every :spark:func:`ltrim` reverse typeof regr_avgx - :spark:func:`bin` exists make_date right ucase regr_avgy - binary :spark:func:`exp` make_dt_interval rint unbase64 regr_count - bit_and explode make_interval :spark:func:`rlike` unhex regr_r2 - bit_count explode_outer make_timestamp :spark:func:`round` unix_date skewness - bit_get expm1 make_ym_interval row_number unix_micros some - bit_length extract :spark:func:`map` rpad unix_millis std - bit_or factorial map_concat :spark:func:`rtrim` unix_seconds stddev - bit_xor :spark:func:`filter` map_entries schema_of_csv :spark:func:`unix_timestamp` stddev_pop - bool_and find_in_set :spark:func:`map_filter` schema_of_json :spark:func:`upper` stddev_samp - bool_or first :spark:func:`map_from_arrays` second uuid sum - boolean first_value map_from_entries sentences var_pop try_avg - bround flatten map_keys sequence var_samp try_sum - btrim float map_values session_window variance var_pop - cardinality :spark:func:`floor` map_zip_with sha version var_samp - case forall max :spark:func:`sha1` weekday variance - cast format_number max_by :spark:func:`sha2` weekofyear - cbrt format_string :spark:func:`md5` :spark:func:`shiftleft` when - :spark:func:`ceil` from_csv mean :spark:func:`shiftright` width_bucket - ceiling from_json min shiftrightunsigned window - char from_unixtime min_by shuffle xpath - char_length from_utc_timestamp minute sign xpath_boolean - character_length :spark:func:`get_json_object` mod signum xpath_double - :spark:func:`chr` getbit monotonically_increasing_id sin xpath_float - coalesce :spark:func:`greatest` month :spark:func:`sinh` xpath_int - collect_list grouping months_between :spark:func:`size` xpath_long - collect_set grouping_id named_struct skewness xpath_number - :spark:func:`concat` :spark:func:`hash` nanvl slice xpath_short - concat_ws hex negative smallint xpath_string - conv hour next_day some :spark:func:`xxhash64` - corr :spark:func:`hypot` :spark:func:`not` :spark:func:`sort_array` :spark:func:`year` - cos if now soundex zip_with - cosh ifnull nth_value space - cot :spark:func:`in` ntile spark_partition_id - count initcap nullif :spark:func:`split` + :spark:func:`abs` count_if inline nvl sqrt any cume_dist + :spark:func:`acos count_min_sketch inline_outer nvl2 stack approx_count_distinct dense_rank + :spark:func:`acosh` covar_pop input_file_block_length octet_length std approx_percentile first_value + add_months covar_samp input_file_block_start or stddev array_agg lag + :spark:func:`aggregate` crc32 input_file_name overlay stddev_pop avg last_value + and cume_dist :spark:func:`instr` parse_url stddev_samp bit_and lead + any current_catalog int percent_rank str_to_map bit_or :spark:func:`nth_value` + approx_count_distinct current_database isnan percentile string :spark:func:`bit_xor` ntile + approx_percentile current_date :spark:func:`isnotnull` percentile_approx struct bool_and percent_rank + :spark:func:`array` current_timestamp :spark:func:`isnull` pi substr bool_or rank + :spark:func:`array_contains` current_timezone java_method :spark:func:`pmod` :spark:func:`substring` collect_list row_number + array_distinct current_user json_array_length posexplode substring_index collect_set + array_except date json_object_keys posexplode_outer sum corr + :spark:func:`array_intersect` date_add json_tuple position tan count + array_join date_format kurtosis positive tanh count_if + array_max date_from_unix_date lag pow timestamp count_min_sketch + array_min date_part last :spark:func:`power` timestamp_micros covar_pop + array_position date_sub last_day printf timestamp_millis covar_samp + array_remove date_trunc last_value quarter timestamp_seconds every + array_repeat datediff lcase radians tinyint :spark:func:`first` + :spark:func:`array_sort` day lead raise_error to_csv first_value + array_union dayofmonth :spark:func:`least` :spark:func:`rand` to_date grouping + arrays_overlap dayofweek :spark:func:`left` randn to_json grouping_id + arrays_zip dayofyear :spark:func:`length` random to_timestamp histogram_numeric + :spark:func:`ascii` decimal levenshtein range :spark:func:`to_unix_timestamp` kurtosis + asin decode like rank to_utc_timestamp :spark:func:`last` + :spark:func:`asinh` degrees ln reflect :spark:func:`transform` last_value + assert_true dense_rank locate regexp transform_keys max + atan div log :spark:func:`regexp_extract` transform_values max_by + atan2 double log10 regexp_extract_all translate mean + :spark:func:`atanh` e :spark:func:`log1p` regexp_like :spark:func:`trim` min + avg :spark:func:`element_at` log2 regexp_replace trunc min_by + base64 elt :spark:func:`lower` repeat try_add percentile + :spark:func:`between` encode lpad :spark:func:`replace` try_divide percentile_approx + bigint every :spark:func:`ltrim` reverse typeof regr_avgx + :spark:func:`bin` exists make_date right ucase regr_avgy + binary :spark:func:`exp` make_dt_interval rint unbase64 regr_count + bit_and explode make_interval :spark:func:`rlike` unhex regr_r2 + bit_count explode_outer make_timestamp :spark:func:`round` unix_date skewness + bit_get expm1 make_ym_interval row_number unix_micros some + bit_length extract :spark:func:`map` rpad unix_millis std + bit_or factorial map_concat :spark:func:`rtrim` unix_seconds stddev + bit_xor :spark:func:`filter` map_entries schema_of_csv :spark:func:`unix_timestamp` stddev_pop + bool_and find_in_set :spark:func:`map_filter` schema_of_json :spark:func:`upper` stddev_samp + bool_or first :spark:func:`map_from_arrays` second uuid sum + boolean first_value map_from_entries sentences var_pop try_avg + bround flatten map_keys sequence var_samp try_sum + btrim float map_values session_window variance var_pop + cardinality :spark:func:`floor` map_zip_with sha version var_samp + case forall max :spark:func:`sha1` weekday variance + cast format_number max_by :spark:func:`sha2` weekofyear + cbrt format_string :spark:func:`md5` :spark:func:`shiftleft` when + :spark:func:`ceil` from_csv mean :spark:func:`shiftright` width_bucket + ceiling from_json min shiftrightunsigned window + char from_unixtime min_by shuffle xpath + char_length from_utc_timestamp minute sign xpath_boolean + character_length :spark:func:`get_json_object` mod signum xpath_double + :spark:func:`chr` getbit monotonically_increasing_id sin xpath_float + coalesce :spark:func:`greatest` month :spark:func:`sinh` xpath_int + collect_list grouping months_between :spark:func:`size` xpath_long + collect_set grouping_id named_struct skewness xpath_number + :spark:func:`concat` :spark:func:`hash` nanvl slice xpath_short + concat_ws hex negative smallint xpath_string + conv hour next_day some :spark:func:`xxhash64` + corr :spark:func:`hypot` :spark:func:`not` :spark:func:`sort_array` :spark:func:`year` + cos if now soundex zip_with + cosh ifnull nth_value space + cot :spark:func:`in` ntile spark_partition_id + count initcap nullif :spark:func:`split` ========================================= ========================================= ========================================= ========================================= ========================================= == ========================================= == ========================================= diff --git a/velox/docs/functions/spark/datetime.rst b/velox/docs/functions/spark/datetime.rst index aef9f340478..e94004cd56b 100644 --- a/velox/docs/functions/spark/datetime.rst +++ b/velox/docs/functions/spark/datetime.rst @@ -428,4 +428,3 @@ returned for invalid format; otherwise, exception is thrown. :: SELECT from_unixtime(100, '!@#$%^&*'); -- throws exception) (for Joda date formatter) SELECT get_timestamp('1970-01-01', '!@#$%^&*'); -- NULL (parsing error) (for Simple date formatter) SELECT get_timestamp('1970-01-01', '!@#$%^&*'); -- throws exception) (for Joda date formatter) - diff --git a/velox/docs/functions/spark/decimal.rst b/velox/docs/functions/spark/decimal.rst index 630d580346f..3d1c1c18fc2 100644 --- a/velox/docs/functions/spark/decimal.rst +++ b/velox/docs/functions/spark/decimal.rst @@ -160,12 +160,12 @@ Decimal Special Forms Returns ``decimal`` rounded to a new scale using HALF_UP rounding mode. In HALF_UP rounding, the digit 5 is rounded up. ``scale`` is the new scale to be rounded to. It is 0 by default, and integer in [INT_MIN, INT_MAX] is allowed to be its value. - When the absolute value of scale exceeds the maximum precision of long decimal (38), the round logic is equivalent to the case where it is 38 as we cannot exceed the maximum precision. + When the absolute value of scale exceeds the maximum precision of long decimal (38), the round logic is equivalent to the case where it is 38 as we cannot exceed the maximum precision. The result precision and scale are decided with the precision and scale of input ``decimal`` and ``scale``. After rounding we may need one more digit in the integral part. - + :: - + SELECT (round(cast (9.9 as decimal(2, 1)), 0)); -- decimal 10 SELECT (round(cast (99 as decimal(2, 0)), -1)); -- decimal 100 diff --git a/velox/docs/functions/spark/json.rst b/velox/docs/functions/spark/json.rst index d06a77b8eee..5ffe3af0fca 100644 --- a/velox/docs/functions/spark/json.rst +++ b/velox/docs/functions/spark/json.rst @@ -21,17 +21,17 @@ JSON Functions .. spark:function:: from_json(jsonString) -> array / map / row - Casts ``jsonString`` to an ARRAY, MAP, or ROW type, with the output type + Casts ``jsonString`` to an ARRAY, MAP, or ROW type, with the output type determined by the expression. Returns NULL, if the input string is unparsable. - Supported element types include BOOLEAN, TINYINT, SMALLINT, INTEGER, BIGINT, - REAL, DOUBLE, DATE, VARCHAR, ARRAY, MAP and ROW. When casting to ARRAY or MAP, - the element type of the array or the value type of the map must be one of - these supported types, and for maps, the key type must be VARCHAR. Casting - to ROW supports only JSON objects. - Note that since the result type can be inferred from the expression, in Velox we - do not need to provide the ``schema`` parameter as required by Spark's from_json + Supported element types include BOOLEAN, TINYINT, SMALLINT, INTEGER, BIGINT, + REAL, DOUBLE, DATE, VARCHAR, ARRAY, MAP and ROW. When casting to ARRAY or MAP, + the element type of the array or the value type of the map must be one of + these supported types, and for maps, the key type must be VARCHAR. Casting + to ROW supports only JSON objects. + Note that since the result type can be inferred from the expression, in Velox we + do not need to provide the ``schema`` parameter as required by Spark's from_json function. :: - + SELECT from_json('{"a": true}', 'a BOOLEAN'); -- {'a'=true} SELECT from_json('{"a": 1}', 'a INT'); -- {'a'=1} SELECT from_json('{"a": 1.0}', 'a DOUBLE'); -- {'a'=1.0} @@ -53,7 +53,7 @@ JSON Functions * Does not support schemas that include a corrupt record column, for example, the Spark function below is not supported. :: - from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE, _corrupt_record STRING') + from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE, _corrupt_record STRING') .. spark:function:: get_json_object(jsonString, path) -> varchar diff --git a/velox/docs/functions/spark/misc.rst b/velox/docs/functions/spark/misc.rst index f36a3362a3c..de826e35b14 100644 --- a/velox/docs/functions/spark/misc.rst +++ b/velox/docs/functions/spark/misc.rst @@ -4,7 +4,7 @@ Miscellaneous Functions .. spark:function:: at_least_n_non_nulls(n, value1, value2, ..., valueN) -> bool - Returns true if there are at least ``n`` non-null and non-NaN values, + Returns true if there are at least ``n`` non-null and non-NaN values, or false otherwise. ``value1, value2, ..., valueN`` are evaluated lazily. If ``n`` non-null and non-NaN values are found, the function will stop evaluating the remaining arguments. If ``n <= 0``, the result is true. null diff --git a/velox/docs/functions/spark/string.rst b/velox/docs/functions/spark/string.rst index 7ad7141595b..e788f1d2ddb 100644 --- a/velox/docs/functions/spark/string.rst +++ b/velox/docs/functions/spark/string.rst @@ -3,9 +3,9 @@ String Functions ==================================== .. note:: - + Unless specified otherwise, all functions return NULL if at least one of the arguments is NULL. - + These functions assume that input strings contain valid UTF-8 encoded Unicode code points. The behavior is undefined if they are not. @@ -16,7 +16,7 @@ String Functions .. spark:function:: bit_length(string/binary) -> integer Returns the bit length for the specified string column. :: - + SELECT bit_length('123'); -- 24 .. spark:function:: chr(n) -> varchar @@ -47,7 +47,7 @@ String Functions .. spark:function:: contains(left, right) -> boolean Returns true if 'right' is found in 'left'. Otherwise, returns false. :: - + SELECT contains('Spark SQL', 'Spark'); -- true SELECT contains('Spark SQL', 'SPARK'); -- false SELECT contains('Spark SQL', null); -- NULL @@ -167,7 +167,7 @@ String Functions SELECT lower('SparkSql'); -- sparksql .. spark:function:: lpad(string, len, pad) -> string - + Returns ``string``, left-padded with pad to a length of ``len``. If ``string`` is longer than ``len``, the return value is shortened to ``len`` characters or bytes. If ``pad`` is not specified, ``string`` will be padded to the left with space characters @@ -249,7 +249,7 @@ String Functions .. spark:function:: repeat(input, n) -> varchar - Returns the string which repeats ``input`` ``n`` times. + Returns the string which repeats ``input`` ``n`` times. Result size must be less than or equal to 1MB. If ``n`` is less than or equal to 0, empty string is returned. :: @@ -277,15 +277,15 @@ String Functions Returns input string with characters in reverse order. .. spark:function:: rpad(string, len, pad) -> string - - Returns ``string``, right-padded with ``pad`` to a length of ``len``. + + Returns ``string``, right-padded with ``pad`` to a length of ``len``. If ``string`` is longer than ``len``, the return value is shortened to ``len`` characters. If ``pad`` is not specified, ``string`` will be padded to the right with space characters if it is a character string, and with zeros if it is a binary string. :: SELECT lpad('hi', 5, '??'); -- ???hi SELECT lpad('hi', 1, '??'); -- h - SELECT lpad('hi', 4); -- hi + SELECT lpad('hi', 4); -- hi .. spark:function:: rtrim(string) -> varchar @@ -318,7 +318,7 @@ String Functions contain all input beyond the last matched regex. When ``limit`` <= 0, ``regex`` will be applied as many times as possible, and the resulting array can be of any size. When ``delimiter`` is empty, if ``limit`` is smaller than the size of ``string``, the resulting array only contains ``limit`` number of single characters - splitting from ``string``, if ``limit`` is not provided or is larger than the size of ``string``, the resulting + splitting from ``string``, if ``limit`` is not provided or is larger than the size of ``string``, the resulting array contains all the single characters of ``string`` and does not include an empty tail character. The split function align with vanilla spark 3.4+ split function. :: @@ -357,7 +357,7 @@ String Functions Returns the rest of ``string`` from the starting position ``start``. Positions start with ``1``. A negative starting position is interpreted as being relative to the end of the string. When the starting position is 0, - the meaning is to refer to the first character.Type of 'start' must be an INTEGER. + the meaning is to refer to the first character.Type of 'start' must be an INTEGER. .. spark:function:: substring(string, start, length) -> varchar :noindex: @@ -410,8 +410,8 @@ String Functions size is larger than ``replace's``, the extra characters in ``match`` will be removed from ``string``. In addition, this function only considers the first occurrence of a character in ``match`` and uses its corresponding character in - ``replace`` for translation. - Any invalid UTF-8 characters present in the input string will be treated as a + ``replace`` for translation. + Any invalid UTF-8 characters present in the input string will be treated as a single character.:: SELECT translate('spark', 'sa', '12'); -- "1p2rk" @@ -438,3 +438,22 @@ String Functions Returns string with all characters changed to uppercase. :: SELECT upper('SparkSql'); -- SPARKSQL +<<<<<<< HEAD +======= + +.. spark:function:: varchar_type_write_side_check(string, limit) -> varchar + + Removes trailing space characters (ASCII 32) that exceed the length ``limit`` from the end of input ``string``. ``limit`` is the maximum length of characters that can be allowed. + Throws exception when ``string`` still exceeds ``limit`` after trimming trailing spaces or when ``limit`` is not greater than 0. + Empty strings are returned as-is since they always satisfy any length ``limit`` greater than 0. + Note: This function is not directly callable in Spark SQL, but internally used for length check when writing string type columns. :: + + -- Function call examples (this function is not directly callable in Spark SQL). + varchar_type_write_side_check("abc", 3) -- "abc" + varchar_type_write_side_check("abc ", 3) -- "abc" + varchar_type_write_side_check("abcd", 3) -- VeloxUserError: "Exceeds allowed length limitation: '3'" + varchar_type_write_side_check("中国", 3) -- "中国" + varchar_type_write_side_check("中文中国", 3) -- VeloxUserError: "Exceeds allowed length limitation: '3'" + varchar_type_write_side_check(" ", 0) -- VeloxUserError: "The length limit must be greater than 0." + varchar_type_write_side_check("", 3) -- "" +>>>>>>> 7c73c1106 (misc: Use pre-commit for quality checks (#13361)) diff --git a/velox/docs/monitoring.rst b/velox/docs/monitoring.rst index b92026f9b63..e8f0c86c339 100644 --- a/velox/docs/monitoring.rst +++ b/velox/docs/monitoring.rst @@ -6,4 +6,4 @@ Monitoring :maxdepth: 1 monitoring/metrics.rst - monitoring/stats.rst \ No newline at end of file + monitoring/stats.rst diff --git a/velox/docs/monthly-updates/2021/december-2021.rst b/velox/docs/monthly-updates/2021/december-2021.rst index 76097bbeb18..5ec088f6853 100644 --- a/velox/docs/monthly-updates/2021/december-2021.rst +++ b/velox/docs/monthly-updates/2021/december-2021.rst @@ -35,4 +35,4 @@ Aditi Pandit, Alex Hornby, Amit Dutta, Andres Suarez, Andrew Gallagher, Chao Chen, Cheng Su, Deepak Majeti, Huameng Jiang, Jack Qiao, Kevin Wilfong, Krishna Pai, Laith Sakka, Marc Fisher, Masha Basmanova, Michael Shang, Naresh Kumar, Orri Erling, Pedro Eugenio Rocha Pedreira, Sergey Pershin, -Wei He, Wei Zheng, Xavier Deguillard, Yating Zhou, Yuan Chao Chou, Zhenyuan Zhao +Wei He, Wei Zheng, Xavier Deguillard, Yating Zhou, Yuan Chao Chou, Zhenyuan Zhao diff --git a/velox/docs/monthly-updates/2021/november-2021.rst b/velox/docs/monthly-updates/2021/november-2021.rst index 6626c5bf5c5..beb09b1110b 100644 --- a/velox/docs/monthly-updates/2021/november-2021.rst +++ b/velox/docs/monthly-updates/2021/november-2021.rst @@ -35,4 +35,4 @@ Chao Chen, Darren Fu, David Kang, Deepak Majeti, Huameng Jiang, Jake Jung, Jialiang Tan, Jialing Zhou, Justin Yang, Kevin Wilfong, Konstantin Tsoy, Krishna Pai, Laith Sakka, MJ Deng, Masha Basmanova, Michael Shang, Naresh Kumar, Orri Erling, Pedro Eugenio Rocha Pedreira, Thomas Orozco, Wei He, Yating -Zhou, Yuan Chao Chou, Zhenyuan Zhao, frankobe, ienkovich. \ No newline at end of file +Zhou, Yuan Chao Chou, Zhenyuan Zhao, frankobe, ienkovich. diff --git a/velox/docs/monthly-updates/2022/april-2022.rst b/velox/docs/monthly-updates/2022/april-2022.rst index dd44b388de8..758b0d66fdf 100644 --- a/velox/docs/monthly-updates/2022/april-2022.rst +++ b/velox/docs/monthly-updates/2022/april-2022.rst @@ -56,4 +56,4 @@ Majeti, Ge Gao, Huameng Jiang, James Xu, Jialiang Tan, Jimmy Lu, Jon Janzen, Jun Wu, Katie Mancini, Kevin Wilfong, Krishna Pai, Laith Sakka, Li Yazhou, MJ Deng, Masha Basmanova, Orri Erling, Pedro Eugenio Rocha Pedreira, Pyre Bot Jr, Richard Barnes, Sergey Pershin, Victor Zverovich, Wei He, Wenlei Xie, Xiang Xu, -Zeyi (Rice) Fan, qiaoyi.dingqy \ No newline at end of file +Zeyi (Rice) Fan, qiaoyi.dingqy diff --git a/velox/docs/monthly-updates/2022/august-2022.rst b/velox/docs/monthly-updates/2022/august-2022.rst index 46e21ff8a05..8208794d092 100644 --- a/velox/docs/monthly-updates/2022/august-2022.rst +++ b/velox/docs/monthly-updates/2022/august-2022.rst @@ -85,4 +85,4 @@ Orvid King, Parvez Shaikh, Paul Saab, Pedro Eugenio Rocha Pedreira, Pramod, Pyre Bot Jr, Raúl Cumplido, Serge Druzkin, Sergey Pershin, Shiyu Gan, Shrikrishna (Shri) Khare, Taras Boiko, Victor Zverovich, Wei He, Wei Zheng, Xiaoxuan Meng, Yuan Chao Chou, Zhenyuan Zhao, erdembilegt.j, jiyu.cy, leoluan2009, -muniao, tanjialiang, usurai, yingsu00, 学东栾. \ No newline at end of file +muniao, tanjialiang, usurai, yingsu00, 学东栾. diff --git a/velox/docs/monthly-updates/2022/july-2022.rst b/velox/docs/monthly-updates/2022/july-2022.rst index 9580b11ba7f..842a3aa9078 100644 --- a/velox/docs/monthly-updates/2022/july-2022.rst +++ b/velox/docs/monthly-updates/2022/july-2022.rst @@ -55,4 +55,4 @@ Jialiang Tan, Jie1 Zhang, Jimmy Lu, Jonathan Mendoza, Karteek Murthy, Kevin Wilf Kimberly Yang, Krishna Pai, Laith Sakka, Masha Basmanova, Michael Shang, Naresh Kumar, Orri Erling, Orvid King, Pedro Eugenio Rocha Pedreira, PenghuiJiao, Pramod, Prasoon Telang, Scott Wolchok, Victor Zverovich, Wei He, Xavier Deguillard, Xiaoxuan Meng, Yoav Helfman, -Zeyi (Rice) Fan, Zhenyuan Zhao, usurai, yingsu00 \ No newline at end of file +Zeyi (Rice) Fan, Zhenyuan Zhao, usurai, yingsu00 diff --git a/velox/docs/monthly-updates/2022/june-2022.rst b/velox/docs/monthly-updates/2022/june-2022.rst index 8898e950f2e..dc2dc0aa06d 100644 --- a/velox/docs/monthly-updates/2022/june-2022.rst +++ b/velox/docs/monthly-updates/2022/june-2022.rst @@ -63,4 +63,4 @@ Katie Mancini, Ke Jia, Kevin Wilfong, Krishna Pai, Laith Sakka, Masha Basmanova, Michael Shang, Mindaugas Rukas, Orri Erling, Patrick Stuedi, Paul Saab, Pedro Eugenio Rocha Pedreira, Pramod Sathyanarayana, Sahana CB, Sergey Pershin, Wei He, Xavier Deguillard, Xiaoxuan Meng, Yating Zhou, Yoav Helfman, Zeyi (Rice) Fan, -Zhenyuan Zhao, artem.malyshev, benitakbritto, frankobe, usurai, yingsu00, zhaozhenhui \ No newline at end of file +Zhenyuan Zhao, artem.malyshev, benitakbritto, frankobe, usurai, yingsu00, zhaozhenhui diff --git a/velox/docs/monthly-updates/2022/may-2022.rst b/velox/docs/monthly-updates/2022/may-2022.rst index b76e9452484..4cd772ed4bb 100644 --- a/velox/docs/monthly-updates/2022/may-2022.rst +++ b/velox/docs/monthly-updates/2022/may-2022.rst @@ -64,4 +64,4 @@ Jialiang Tan, Jie1 Zhang, Jimmy Lu, Jing Zhu, John Reese, Karteek Murthy, Kevin Wilfong, Krishna Pai, Laith Sakka, MJ Deng, Masha Basmanova, Muir Manders, Orri Erling, Patrick Stuedi, Pedro Eugenio Rocha Pedreira, Pyre Bot Jr, Rui Mo, Sergey Pershin, TJ Yin, Wei He, Zhenyuan Zhao, artem.malyshev, rui-mo, usurai, -xuedongluan, yeyuqiang, yingsu00 \ No newline at end of file +xuedongluan, yeyuqiang, yingsu00 diff --git a/velox/docs/monthly-updates/2022/october-2022.rst b/velox/docs/monthly-updates/2022/october-2022.rst index c8663d6dd3b..39ee8752ccc 100644 --- a/velox/docs/monthly-updates/2022/october-2022.rst +++ b/velox/docs/monthly-updates/2022/october-2022.rst @@ -61,4 +61,4 @@ Shang, Mike Decker, Milosz Linkiewicz, Open Source Bot, Orri Erling, Patrick Somaru, Pavel Solodovnikov, Pedro Eugenio Rocha Pedreira, Pedro Pedreira, Pramod, Qitian Zeng, Randeep Singh, Raúl Cumplido, Sergey Pershin, Uhyon Chung, Vinti Pandey, Wei He, Weile Wei, Zeyi (Rice) Fan, Zhenyuan Zhao, mwish, -shengxuan.liu, tanjialiang, xiaoxmeng, yingsu00, zhejiangxiaomai \ No newline at end of file +shengxuan.liu, tanjialiang, xiaoxmeng, yingsu00, zhejiangxiaomai diff --git a/velox/docs/monthly-updates/2023/august-2023.rst b/velox/docs/monthly-updates/2023/august-2023.rst index 33e574fcf5f..3ab81921ace 100644 --- a/velox/docs/monthly-updates/2023/august-2023.rst +++ b/velox/docs/monthly-updates/2023/august-2023.rst @@ -82,4 +82,4 @@ Build Systems Credits ======= -Alexander Yermolovich, Amit Dutta, Ann Rose Benny, Arun D. Panicker, Ashwin Krishna Kumar, Austin Dickey, Bikramjeet Vig, Chengcheng Jin, Christian Zentgraf, Daniel Munoz, David Tolnay, Deepak Majeti, Ebe Janchivdorj, Ge Gao, Giuseppe Ottaviano, Harsha Rastogi, Hongze Zhang, Jacob Wujciak-Jens, Jia Ke, Jialiang Tan, Jimmy Lu, Karteek Murthy Samba Murthy, Karteekmurthys, Ke, Kevin Wilfong, Krishna Pai, Laith Sakka, Luca Niccolini, Ma-Jian1, Mack Ward, Mahadevuni Naveen Kumar, Masha Basmanova, Mike Lui, Nick Terrell, Open Source Bot, Orri Erling, Patrick Sullivan, Pedro Eugenio Rocha Pedreira, Pedro Pedreira, Pramod, Pranjal Shankhdhar, Richard Barnes, Rong Ma, Sandino Flores, Sanjiban Sengupta, Shiyu Gan, Wei He, Zac, Zhe Wan, aditi-pandit, duanmeng, ericyuliu, generatedunixname89002005287564, generatedunixname89002005325676, jackylee-ch, leesf, root, rui-mo, wangxinshuo.db, wypb, xiaoxmeng, yingsu00, yiweiHeOSS, zhejiangxiaomai, 陈旭 \ No newline at end of file +Alexander Yermolovich, Amit Dutta, Ann Rose Benny, Arun D. Panicker, Ashwin Krishna Kumar, Austin Dickey, Bikramjeet Vig, Chengcheng Jin, Christian Zentgraf, Daniel Munoz, David Tolnay, Deepak Majeti, Ebe Janchivdorj, Ge Gao, Giuseppe Ottaviano, Harsha Rastogi, Hongze Zhang, Jacob Wujciak-Jens, Jia Ke, Jialiang Tan, Jimmy Lu, Karteek Murthy Samba Murthy, Karteekmurthys, Ke, Kevin Wilfong, Krishna Pai, Laith Sakka, Luca Niccolini, Ma-Jian1, Mack Ward, Mahadevuni Naveen Kumar, Masha Basmanova, Mike Lui, Nick Terrell, Open Source Bot, Orri Erling, Patrick Sullivan, Pedro Eugenio Rocha Pedreira, Pedro Pedreira, Pramod, Pranjal Shankhdhar, Richard Barnes, Rong Ma, Sandino Flores, Sanjiban Sengupta, Shiyu Gan, Wei He, Zac, Zhe Wan, aditi-pandit, duanmeng, ericyuliu, generatedunixname89002005287564, generatedunixname89002005325676, jackylee-ch, leesf, root, rui-mo, wangxinshuo.db, wypb, xiaoxmeng, yingsu00, yiweiHeOSS, zhejiangxiaomai, 陈旭 diff --git a/velox/docs/monthly-updates/2023/december-2023.rst b/velox/docs/monthly-updates/2023/december-2023.rst index 8f4df40a953..021d32e6f77 100644 --- a/velox/docs/monthly-updates/2023/december-2023.rst +++ b/velox/docs/monthly-updates/2023/december-2023.rst @@ -96,4 +96,4 @@ Patrick Sullivan, Pedro Eugenio Rocha Pedreira, Pedro Pedreira, Pramod,Ravi Rahm Richard Barnes, Sergey Pershin, Srikrishna Gopu, Wei He, Xiaoxuan Meng, Yangyang Gao, Yedidya Feldblum, Zac, aditi-pandit, binwei, duanmeng, hengjiang.ly, joey.ljy, rui-mo, shangjing.cxw, soumyaduriseti, xiaoxmeng, xiyu.zk, xumingming, yan ma, yangchuan ,yingsu00, -zhli, zhli1142015, 高阳阳 \ No newline at end of file +zhli, zhli1142015, 高阳阳 diff --git a/velox/docs/monthly-updates/2023/november-2023.rst b/velox/docs/monthly-updates/2023/november-2023.rst index 40bbf2525bc..536ea83d167 100644 --- a/velox/docs/monthly-updates/2023/november-2023.rst +++ b/velox/docs/monthly-updates/2023/november-2023.rst @@ -102,4 +102,4 @@ Daniel Munoz, Deepak Majeti, Ge Gao, Genevieve (Genna) Helsel, Harvey Hunt, Jake Jimmy Lu, John Elliott, Karteekmurthys, Ke, Kevin Wilfong, Krishna Pai, Laith Sakka, Masha Basmanova, Orri Erling, PHILO-HE, Patrick Sullivan, Pedro Eugenio Rocha Pedreira, Pramod, Richard Barnes, Schierbeck, Cody, Sergey Pershin, Wei He, Zhenyuan Zhao, aditi-pandit, curt, duanmeng, joey.ljy, lingbin, rui-mo, usurai, vibhatha, wypb, xiaoxmeng, -xumingming, yangchuan, yaqi-zhao, yingsu00, yiweiHeOSS, youxiduo, zhli, 高阳阳 \ No newline at end of file +xumingming, yangchuan, yaqi-zhao, yingsu00, yiweiHeOSS, youxiduo, zhli, 高阳阳 diff --git a/velox/docs/monthly-updates/january-2024.rst b/velox/docs/monthly-updates/january-2024.rst index dc917581e8e..eeae8d36721 100644 --- a/velox/docs/monthly-updates/january-2024.rst +++ b/velox/docs/monthly-updates/january-2024.rst @@ -81,4 +81,3 @@ Cody, Sergey Pershin, Sitao Lv, Taras Galkovskyi, Wei He, Yedidya Feldblum, Yuan Zhou, Yuping Fan, Zac Wen, aditi-pandit, binwei, duanmeng, hengjiang.ly, icejoywoo, lingbin, mwish, rui-mo, wypb, xiaoxmeng, xumingming, yangchuan, yingsu00, youxiduo, yuling.sh, zhli1142015, zky.zhoukeyong, zwangsheng - diff --git a/velox/docs/monthly-updates/july-2024.rst b/velox/docs/monthly-updates/july-2024.rst index 26e34b91135..4e1dc0c7828 100644 --- a/velox/docs/monthly-updates/july-2024.rst +++ b/velox/docs/monthly-updates/july-2024.rst @@ -125,4 +125,4 @@ Credits 5 xiaoxmeng - Meta 2 Ying Su - IBM 2 youxiduo - 12 Zhen Li - Microsoft \ No newline at end of file + 12 Zhen Li - Microsoft diff --git a/velox/dwio/common/tests/Lemire/FastPFor/LICENSE b/velox/dwio/common/tests/Lemire/FastPFor/LICENSE index 8405e89a0b1..37ec93a14fd 100644 --- a/velox/dwio/common/tests/Lemire/FastPFor/LICENSE +++ b/velox/dwio/common/tests/Lemire/FastPFor/LICENSE @@ -188,4 +188,4 @@ third-party archives. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/velox/dwio/dwrf/test/CommonTests.cpp b/velox/dwio/dwrf/test/CommonTests.cpp index e1c94cae1dd..03b69182c49 100644 --- a/velox/dwio/dwrf/test/CommonTests.cpp +++ b/velox/dwio/dwrf/test/CommonTests.cpp @@ -95,7 +95,7 @@ TEST_F( {proto::Stream_Kind_DICTIONARY_COUNT, StreamKind::StreamKind_DICTIONARY_COUNT}, {proto::Stream_Kind_NANO_DATA, StreamKind::StreamKind_NANO_DATA}, {proto::Stream_Kind_ROW_INDEX, StreamKind::StreamKind_ROW_INDEX}, - {proto::Stream_Kind_IN_DICTIONARY, StreamKind::StreamKind_IN_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY, StreamKind::StreamKind_STRIDE_DICTIONARY}, + {proto::Stream_Kind_IN_DICTIONARY, StreamKind::StreamKind_IN_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY, StreamKind::StreamKind_STRIDE_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY_LENGTH, StreamKind::StreamKind_STRIDE_DICTIONARY_LENGTH}, {proto::Stream_Kind_BLOOM_FILTER_UTF8, StreamKind::StreamKind_BLOOM_FILTER_UTF8}, {proto::Stream_Kind_IN_MAP, StreamKind::StreamKind_IN_MAP}, @@ -145,7 +145,7 @@ TEST_F( {proto::Stream_Kind_DICTIONARY_COUNT, StreamKind::StreamKind_DICTIONARY_COUNT}, {proto::Stream_Kind_NANO_DATA, StreamKind::StreamKind_NANO_DATA}, {proto::Stream_Kind_ROW_INDEX, StreamKind::StreamKind_ROW_INDEX}, - {proto::Stream_Kind_IN_DICTIONARY, StreamKind::StreamKind_IN_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY, StreamKind::StreamKind_STRIDE_DICTIONARY}, + {proto::Stream_Kind_IN_DICTIONARY, StreamKind::StreamKind_IN_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY, StreamKind::StreamKind_STRIDE_DICTIONARY}, {proto::Stream_Kind_STRIDE_DICTIONARY_LENGTH, StreamKind::StreamKind_STRIDE_DICTIONARY_LENGTH}, {proto::Stream_Kind_BLOOM_FILTER_UTF8, StreamKind::StreamKind_BLOOM_FILTER_UTF8}, {proto::Stream_Kind_IN_MAP, StreamKind::StreamKind_IN_MAP}, diff --git a/velox/dwio/parquet/thrift/ParquetThriftTypes.cpp b/velox/dwio/parquet/thrift/ParquetThriftTypes.cpp index 599ee76ace5..674c99300f7 100644 --- a/velox/dwio/parquet/thrift/ParquetThriftTypes.cpp +++ b/velox/dwio/parquet/thrift/ParquetThriftTypes.cpp @@ -849,21 +849,16 @@ void Statistics::printTo(std::ostream& out) const { out << "Statistics("; out << "max="; (__isset.max ? (out << to_string(max)) : (out << "")); - out << ", " - << "min="; + out << ", " << "min="; (__isset.min ? (out << to_string(min)) : (out << "")); - out << ", " - << "null_count="; + out << ", " << "null_count="; (__isset.null_count ? (out << to_string(null_count)) : (out << "")); - out << ", " - << "distinct_count="; + out << ", " << "distinct_count="; (__isset.distinct_count ? (out << to_string(distinct_count)) : (out << "")); - out << ", " - << "max_value="; + out << ", " << "max_value="; (__isset.max_value ? (out << to_string(max_value)) : (out << "")); - out << ", " - << "min_value="; + out << ", " << "min_value="; (__isset.min_value ? (out << to_string(min_value)) : (out << "")); out << ")"; } @@ -1365,8 +1360,7 @@ void DecimalType::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "DecimalType("; out << "scale=" << to_string(scale); - out << ", " - << "precision=" << to_string(precision); + out << ", " << "precision=" << to_string(precision); out << ")"; } @@ -1669,11 +1663,9 @@ void TimeUnit::printTo(std::ostream& out) const { out << "TimeUnit("; out << "MILLIS="; (__isset.MILLIS ? (out << to_string(MILLIS)) : (out << "")); - out << ", " - << "MICROS="; + out << ", " << "MICROS="; (__isset.MICROS ? (out << to_string(MICROS)) : (out << "")); - out << ", " - << "NANOS="; + out << ", " << "NANOS="; (__isset.NANOS ? (out << to_string(NANOS)) : (out << "")); out << ")"; } @@ -1784,8 +1776,7 @@ void TimestampType::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "TimestampType("; out << "isAdjustedToUTC=" << to_string(isAdjustedToUTC); - out << ", " - << "unit=" << to_string(unit); + out << ", " << "unit=" << to_string(unit); out << ")"; } @@ -1894,8 +1885,7 @@ void TimeType::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "TimeType("; out << "isAdjustedToUTC=" << to_string(isAdjustedToUTC); - out << ", " - << "unit=" << to_string(unit); + out << ", " << "unit=" << to_string(unit); out << ")"; } @@ -2004,8 +1994,7 @@ void IntType::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "IntType("; out << "bitWidth=" << to_string(bitWidth); - out << ", " - << "isSigned=" << to_string(isSigned); + out << ", " << "isSigned=" << to_string(isSigned); out << ")"; } @@ -2470,41 +2459,29 @@ void LogicalType::printTo(std::ostream& out) const { out << "LogicalType("; out << "STRING="; (__isset.STRING ? (out << to_string(STRING)) : (out << "")); - out << ", " - << "MAP="; + out << ", " << "MAP="; (__isset.MAP ? (out << to_string(MAP)) : (out << "")); - out << ", " - << "LIST="; + out << ", " << "LIST="; (__isset.LIST ? (out << to_string(LIST)) : (out << "")); - out << ", " - << "ENUM="; + out << ", " << "ENUM="; (__isset.ENUM ? (out << to_string(ENUM)) : (out << "")); - out << ", " - << "DECIMAL="; + out << ", " << "DECIMAL="; (__isset.DECIMAL ? (out << to_string(DECIMAL)) : (out << "")); - out << ", " - << "DATE="; + out << ", " << "DATE="; (__isset.DATE ? (out << to_string(DATE)) : (out << "")); - out << ", " - << "TIME="; + out << ", " << "TIME="; (__isset.TIME ? (out << to_string(TIME)) : (out << "")); - out << ", " - << "TIMESTAMP="; + out << ", " << "TIMESTAMP="; (__isset.TIMESTAMP ? (out << to_string(TIMESTAMP)) : (out << "")); - out << ", " - << "INTEGER="; + out << ", " << "INTEGER="; (__isset.INTEGER ? (out << to_string(INTEGER)) : (out << "")); - out << ", " - << "UNKNOWN="; + out << ", " << "UNKNOWN="; (__isset.UNKNOWN ? (out << to_string(UNKNOWN)) : (out << "")); - out << ", " - << "JSON="; + out << ", " << "JSON="; (__isset.JSON ? (out << to_string(JSON)) : (out << "")); - out << ", " - << "BSON="; + out << ", " << "BSON="; (__isset.BSON ? (out << to_string(BSON)) : (out << "")); - out << ", " - << "UUID="; + out << ", " << "UUID="; (__isset.UUID ? (out << to_string(UUID)) : (out << "")); out << ")"; } @@ -2801,33 +2778,24 @@ void SchemaElement::printTo(std::ostream& out) const { out << "SchemaElement("; out << "type="; (__isset.type ? (out << to_string(type)) : (out << "")); - out << ", " - << "type_length="; + out << ", " << "type_length="; (__isset.type_length ? (out << to_string(type_length)) : (out << "")); - out << ", " - << "repetition_type="; + out << ", " << "repetition_type="; (__isset.repetition_type ? (out << to_string(repetition_type)) : (out << "")); - out << ", " - << "name=" << to_string(name); - out << ", " - << "num_children="; + out << ", " << "name=" << to_string(name); + out << ", " << "num_children="; (__isset.num_children ? (out << to_string(num_children)) : (out << "")); - out << ", " - << "converted_type="; + out << ", " << "converted_type="; (__isset.converted_type ? (out << to_string(converted_type)) : (out << "")); - out << ", " - << "scale="; + out << ", " << "scale="; (__isset.scale ? (out << to_string(scale)) : (out << "")); - out << ", " - << "precision="; + out << ", " << "precision="; (__isset.precision ? (out << to_string(precision)) : (out << "")); - out << ", " - << "field_id="; + out << ", " << "field_id="; (__isset.field_id ? (out << to_string(field_id)) : (out << "")); - out << ", " - << "logicalType="; + out << ", " << "logicalType="; (__isset.logicalType ? (out << to_string(logicalType)) : (out << "")); out << ")"; } @@ -3019,14 +2987,12 @@ void DataPageHeader::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "DataPageHeader("; out << "num_values=" << to_string(num_values); - out << ", " - << "encoding=" << to_string(encoding); + out << ", " << "encoding=" << to_string(encoding); out << ", " << "definition_level_encoding=" << to_string(definition_level_encoding); out << ", " << "repetition_level_encoding=" << to_string(repetition_level_encoding); - out << ", " - << "statistics="; + out << ", " << "statistics="; (__isset.statistics ? (out << to_string(statistics)) : (out << "")); out << ")"; } @@ -3225,10 +3191,8 @@ void DictionaryPageHeader::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "DictionaryPageHeader("; out << "num_values=" << to_string(num_values); - out << ", " - << "encoding=" << to_string(encoding); - out << ", " - << "is_sorted="; + out << ", " << "encoding=" << to_string(encoding); + out << ", " << "is_sorted="; (__isset.is_sorted ? (out << to_string(is_sorted)) : (out << "")); out << ")"; } @@ -3480,24 +3444,17 @@ void DataPageHeaderV2::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "DataPageHeaderV2("; out << "num_values=" << to_string(num_values); - out << ", " - << "num_nulls=" << to_string(num_nulls); - out << ", " - << "num_rows=" << to_string(num_rows); - out << ", " - << "encoding=" << to_string(encoding); - out << ", " - << "definition_levels_byte_length=" + out << ", " << "num_nulls=" << to_string(num_nulls); + out << ", " << "num_rows=" << to_string(num_rows); + out << ", " << "encoding=" << to_string(encoding); + out << ", " << "definition_levels_byte_length=" << to_string(definition_levels_byte_length); - out << ", " - << "repetition_levels_byte_length=" + out << ", " << "repetition_levels_byte_length=" << to_string(repetition_levels_byte_length); - out << ", " - << "is_compressed="; + out << ", " << "is_compressed="; (__isset.is_compressed ? (out << to_string(is_compressed)) : (out << "")); - out << ", " - << "statistics="; + out << ", " << "statistics="; (__isset.statistics ? (out << to_string(statistics)) : (out << "")); out << ")"; } @@ -4098,12 +4055,9 @@ void BloomFilterHeader::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "BloomFilterHeader("; out << "numBytes=" << to_string(numBytes); - out << ", " - << "algorithm=" << to_string(algorithm); - out << ", " - << "hash=" << to_string(hash); - out << ", " - << "compression=" << to_string(compression); + out << ", " << "algorithm=" << to_string(algorithm); + out << ", " << "hash=" << to_string(hash); + out << ", " << "compression=" << to_string(compression); out << ")"; } @@ -4348,27 +4302,20 @@ void PageHeader::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "PageHeader("; out << "type=" << to_string(type); - out << ", " - << "uncompressed_page_size=" << to_string(uncompressed_page_size); - out << ", " - << "compressed_page_size=" << to_string(compressed_page_size); - out << ", " - << "crc="; + out << ", " << "uncompressed_page_size=" << to_string(uncompressed_page_size); + out << ", " << "compressed_page_size=" << to_string(compressed_page_size); + out << ", " << "crc="; (__isset.crc ? (out << to_string(crc)) : (out << "")); - out << ", " - << "data_page_header="; + out << ", " << "data_page_header="; (__isset.data_page_header ? (out << to_string(data_page_header)) : (out << "")); - out << ", " - << "index_page_header="; + out << ", " << "index_page_header="; (__isset.index_page_header ? (out << to_string(index_page_header)) : (out << "")); - out << ", " - << "dictionary_page_header="; + out << ", " << "dictionary_page_header="; (__isset.dictionary_page_header ? (out << to_string(dictionary_page_header)) : (out << "")); - out << ", " - << "data_page_header_v2="; + out << ", " << "data_page_header_v2="; (__isset.data_page_header_v2 ? (out << to_string(data_page_header_v2)) : (out << "")); out << ")"; @@ -4481,8 +4428,7 @@ void KeyValue::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "KeyValue("; out << "key=" << to_string(key); - out << ", " - << "value="; + out << ", " << "value="; (__isset.value ? (out << to_string(value)) : (out << "")); out << ")"; } @@ -4616,10 +4562,8 @@ void SortingColumn::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "SortingColumn("; out << "column_idx=" << to_string(column_idx); - out << ", " - << "descending=" << to_string(descending); - out << ", " - << "nulls_first=" << to_string(nulls_first); + out << ", " << "descending=" << to_string(descending); + out << ", " << "nulls_first=" << to_string(nulls_first); out << ")"; } @@ -4756,10 +4700,8 @@ void PageEncodingStats::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "PageEncodingStats("; out << "page_type=" << to_string(page_type); - out << ", " - << "encoding=" << to_string(encoding); - out << ", " - << "count=" << to_string(count); + out << ", " << "encoding=" << to_string(encoding); + out << ", " << "count=" << to_string(count); out << ")"; } @@ -5235,41 +5177,29 @@ void ColumnMetaData::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "ColumnMetaData("; out << "type=" << to_string(type); - out << ", " - << "encodings=" << to_string(encodings); - out << ", " - << "path_in_schema=" << to_string(path_in_schema); - out << ", " - << "codec=" << to_string(codec); - out << ", " - << "num_values=" << to_string(num_values); + out << ", " << "encodings=" << to_string(encodings); + out << ", " << "path_in_schema=" << to_string(path_in_schema); + out << ", " << "codec=" << to_string(codec); + out << ", " << "num_values=" << to_string(num_values); out << ", " << "total_uncompressed_size=" << to_string(total_uncompressed_size); - out << ", " - << "total_compressed_size=" << to_string(total_compressed_size); - out << ", " - << "key_value_metadata="; + out << ", " << "total_compressed_size=" << to_string(total_compressed_size); + out << ", " << "key_value_metadata="; (__isset.key_value_metadata ? (out << to_string(key_value_metadata)) : (out << "")); - out << ", " - << "data_page_offset=" << to_string(data_page_offset); - out << ", " - << "index_page_offset="; + out << ", " << "data_page_offset=" << to_string(data_page_offset); + out << ", " << "index_page_offset="; (__isset.index_page_offset ? (out << to_string(index_page_offset)) : (out << "")); - out << ", " - << "dictionary_page_offset="; + out << ", " << "dictionary_page_offset="; (__isset.dictionary_page_offset ? (out << to_string(dictionary_page_offset)) : (out << "")); - out << ", " - << "statistics="; + out << ", " << "statistics="; (__isset.statistics ? (out << to_string(statistics)) : (out << "")); - out << ", " - << "encoding_stats="; + out << ", " << "encoding_stats="; (__isset.encoding_stats ? (out << to_string(encoding_stats)) : (out << "")); - out << ", " - << "bloom_filter_offset="; + out << ", " << "bloom_filter_offset="; (__isset.bloom_filter_offset ? (out << to_string(bloom_filter_offset)) : (out << "")); out << ")"; @@ -5473,8 +5403,7 @@ void EncryptionWithColumnKey::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "EncryptionWithColumnKey("; out << "path_in_schema=" << to_string(path_in_schema); - out << ", " - << "key_metadata="; + out << ", " << "key_metadata="; (__isset.key_metadata ? (out << to_string(key_metadata)) : (out << "")); out << ")"; } @@ -5593,8 +5522,7 @@ void ColumnCryptoMetaData::printTo(std::ostream& out) const { (__isset.ENCRYPTION_WITH_FOOTER_KEY ? (out << to_string(ENCRYPTION_WITH_FOOTER_KEY)) : (out << "")); - out << ", " - << "ENCRYPTION_WITH_COLUMN_KEY="; + out << ", " << "ENCRYPTION_WITH_COLUMN_KEY="; (__isset.ENCRYPTION_WITH_COLUMN_KEY ? (out << to_string(ENCRYPTION_WITH_COLUMN_KEY)) : (out << "")); @@ -5864,33 +5792,25 @@ void ColumnChunk::printTo(std::ostream& out) const { out << "ColumnChunk("; out << "file_path="; (__isset.file_path ? (out << to_string(file_path)) : (out << "")); - out << ", " - << "file_offset=" << to_string(file_offset); - out << ", " - << "meta_data="; + out << ", " << "file_offset=" << to_string(file_offset); + out << ", " << "meta_data="; (__isset.meta_data ? (out << to_string(meta_data)) : (out << "")); - out << ", " - << "offset_index_offset="; + out << ", " << "offset_index_offset="; (__isset.offset_index_offset ? (out << to_string(offset_index_offset)) : (out << "")); - out << ", " - << "offset_index_length="; + out << ", " << "offset_index_length="; (__isset.offset_index_length ? (out << to_string(offset_index_length)) : (out << "")); - out << ", " - << "column_index_offset="; + out << ", " << "column_index_offset="; (__isset.column_index_offset ? (out << to_string(column_index_offset)) : (out << "")); - out << ", " - << "column_index_length="; + out << ", " << "column_index_length="; (__isset.column_index_length ? (out << to_string(column_index_length)) : (out << "")); - out << ", " - << "crypto_metadata="; + out << ", " << "crypto_metadata="; (__isset.crypto_metadata ? (out << to_string(crypto_metadata)) : (out << "")); - out << ", " - << "encrypted_column_metadata="; + out << ", " << "encrypted_column_metadata="; (__isset.encrypted_column_metadata ? (out << to_string(encrypted_column_metadata)) : (out << "")); @@ -6159,23 +6079,17 @@ void RowGroup::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "RowGroup("; out << "columns=" << to_string(columns); - out << ", " - << "total_byte_size=" << to_string(total_byte_size); - out << ", " - << "num_rows=" << to_string(num_rows); - out << ", " - << "sorting_columns="; + out << ", " << "total_byte_size=" << to_string(total_byte_size); + out << ", " << "num_rows=" << to_string(num_rows); + out << ", " << "sorting_columns="; (__isset.sorting_columns ? (out << to_string(sorting_columns)) : (out << "")); - out << ", " - << "file_offset="; + out << ", " << "file_offset="; (__isset.file_offset ? (out << to_string(file_offset)) : (out << "")); - out << ", " - << "total_compressed_size="; + out << ", " << "total_compressed_size="; (__isset.total_compressed_size ? (out << to_string(total_compressed_size)) : (out << "")); - out << ", " - << "ordinal="; + out << ", " << "ordinal="; (__isset.ordinal ? (out << to_string(ordinal)) : (out << "")); out << ")"; } @@ -6455,10 +6369,8 @@ void PageLocation::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "PageLocation("; out << "offset=" << to_string(offset); - out << ", " - << "compressed_page_size=" << to_string(compressed_page_size); - out << ", " - << "first_row_index=" << to_string(first_row_index); + out << ", " << "compressed_page_size=" << to_string(compressed_page_size); + out << ", " << "first_row_index=" << to_string(first_row_index); out << ")"; } @@ -6837,14 +6749,10 @@ void ColumnIndex::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "ColumnIndex("; out << "null_pages=" << to_string(null_pages); - out << ", " - << "min_values=" << to_string(min_values); - out << ", " - << "max_values=" << to_string(max_values); - out << ", " - << "boundary_order=" << to_string(boundary_order); - out << ", " - << "null_counts="; + out << ", " << "min_values=" << to_string(min_values); + out << ", " << "max_values=" << to_string(max_values); + out << ", " << "boundary_order=" << to_string(boundary_order); + out << ", " << "null_counts="; (__isset.null_counts ? (out << to_string(null_counts)) : (out << "")); out << ")"; } @@ -6977,12 +6885,10 @@ void AesGcmV1::printTo(std::ostream& out) const { out << "AesGcmV1("; out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); - out << ", " - << "aad_file_unique="; + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); - out << ", " - << "supply_aad_prefix="; + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); out << ")"; @@ -7117,12 +7023,10 @@ void AesGcmCtrV1::printTo(std::ostream& out) const { out << "AesGcmCtrV1("; out << "aad_prefix="; (__isset.aad_prefix ? (out << to_string(aad_prefix)) : (out << "")); - out << ", " - << "aad_file_unique="; + out << ", " << "aad_file_unique="; (__isset.aad_file_unique ? (out << to_string(aad_file_unique)) : (out << "")); - out << ", " - << "supply_aad_prefix="; + out << ", " << "supply_aad_prefix="; (__isset.supply_aad_prefix ? (out << to_string(supply_aad_prefix)) : (out << "")); out << ")"; @@ -7237,8 +7141,7 @@ void EncryptionAlgorithm::printTo(std::ostream& out) const { out << "EncryptionAlgorithm("; out << "AES_GCM_V1="; (__isset.AES_GCM_V1 ? (out << to_string(AES_GCM_V1)) : (out << "")); - out << ", " - << "AES_GCM_CTR_V1="; + out << ", " << "AES_GCM_CTR_V1="; (__isset.AES_GCM_CTR_V1 ? (out << to_string(AES_GCM_CTR_V1)) : (out << "")); out << ")"; @@ -7596,29 +7499,21 @@ void FileMetaData::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "FileMetaData("; out << "version=" << to_string(version); - out << ", " - << "schema=" << to_string(schema); - out << ", " - << "num_rows=" << to_string(num_rows); - out << ", " - << "row_groups=" << to_string(row_groups); - out << ", " - << "key_value_metadata="; + out << ", " << "schema=" << to_string(schema); + out << ", " << "num_rows=" << to_string(num_rows); + out << ", " << "row_groups=" << to_string(row_groups); + out << ", " << "key_value_metadata="; (__isset.key_value_metadata ? (out << to_string(key_value_metadata)) : (out << "")); - out << ", " - << "created_by="; + out << ", " << "created_by="; (__isset.created_by ? (out << to_string(created_by)) : (out << "")); - out << ", " - << "column_orders="; + out << ", " << "column_orders="; (__isset.column_orders ? (out << to_string(column_orders)) : (out << "")); - out << ", " - << "encryption_algorithm="; + out << ", " << "encryption_algorithm="; (__isset.encryption_algorithm ? (out << to_string(encryption_algorithm)) : (out << "")); - out << ", " - << "footer_signing_key_metadata="; + out << ", " << "footer_signing_key_metadata="; (__isset.footer_signing_key_metadata ? (out << to_string(footer_signing_key_metadata)) : (out << "")); @@ -7736,8 +7631,7 @@ void FileCryptoMetaData::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "FileCryptoMetaData("; out << "encryption_algorithm=" << to_string(encryption_algorithm); - out << ", " - << "key_metadata="; + out << ", " << "key_metadata="; (__isset.key_metadata ? (out << to_string(key_metadata)) : (out << "")); out << ")"; } diff --git a/velox/exec/tests/utils/TableScanTestBase.cpp b/velox/exec/tests/utils/TableScanTestBase.cpp index 67b64723310..ef571d75b6d 100644 --- a/velox/exec/tests/utils/TableScanTestBase.cpp +++ b/velox/exec/tests/utils/TableScanTestBase.cpp @@ -1,4 +1,18 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "velox/exec/tests/utils/TableScanTestBase.h" diff --git a/velox/experimental/breeze/test/generator_common.py b/velox/experimental/breeze/test/generator_common.py index 9bb2aac382d..138ca3d7eee 100644 --- a/velox/experimental/breeze/test/generator_common.py +++ b/velox/experimental/breeze/test/generator_common.py @@ -71,8 +71,8 @@ if libclang_python: sys.path.append(libclang_python) -import clang.cindex -from clang.cindex import CursorKind +import clang.cindex # noqa E402 +from clang.cindex import CursorKind # noqa E402 if libclang_path: clang.cindex.Config.set_library_path(libclang_path) diff --git a/velox/experimental/breeze/test/kernel_generator.py b/velox/experimental/breeze/test/kernel_generator.py index c6654996382..a4cedfa59c7 100755 --- a/velox/experimental/breeze/test/kernel_generator.py +++ b/velox/experimental/breeze/test/kernel_generator.py @@ -34,7 +34,7 @@ from abc import ABC, abstractmethod -AUTOGEN_HEADER = f"""/* +AUTOGEN_HEADER = """/* * This file is auto-generated from kernel_generator.py * DO NOT EDIT! */ @@ -106,7 +106,7 @@ def generate(self, tu, filename): if self.use_namespace: out.write("namespace kernels {\n\n") warp_threads = self.num_warp_threads() - if warp_threads != None: + if warp_threads is not None: out.write(f"enum {{ WARP_THREADS = {warp_threads} }};") for kernel in kernels: kernel_name = kernel["spelling"] diff --git a/velox/experimental/breeze/test/test_fixture_generator.py b/velox/experimental/breeze/test/test_fixture_generator.py index cbb6d7d19e8..3201aa6c32d 100755 --- a/velox/experimental/breeze/test/test_fixture_generator.py +++ b/velox/experimental/breeze/test/test_fixture_generator.py @@ -34,7 +34,7 @@ import subprocess -AUTOGEN_HEADER = f"""/* +AUTOGEN_HEADER = """/* * This file is auto-generated from test_fixture_generator.py * DO NOT EDIT! */ @@ -263,7 +263,7 @@ def body(self, method, fixture_type_param, template_params, function_params): {preamble} {self.launcher_fn}<{thread_count}{maybe_add_shared_mem}>( {block_count}, - &kernels::{method['spelling']}{kernel_template_args}{kernel_args} + &kernels::{method["spelling"]}{kernel_template_args}{kernel_args} ); {postamble}""" @@ -414,7 +414,7 @@ def body(self, method, fixture_type_param, template_params, function_params): lambda_params = ", ".join(lambda_params) lambda_fn = f"""\ [{size_arg}]({lambda_params}){{ - kernels::{method['spelling']}{kernel_template_args}({inner_kernel_args}); + kernels::{method["spelling"]}{kernel_template_args}({inner_kernel_args}); }}\ """ return f""" @@ -430,7 +430,7 @@ def __init__(self): self.launcher_fn = "OpenCLTestDispatch" def includes(self, fixture_name): - return f""" + return """ #include #include "test/platforms/opencl_test.h" @@ -541,7 +541,7 @@ def __init__(self): self.launcher_fn = "MetalTestDispatch" def includes(self, fixture_name): - return f""" + return """ #include #include "test/platforms/metal_test.h" diff --git a/velox/experimental/cudf/.clang-format b/velox/experimental/cudf/.clang-format index 7b028e6ff68..3cd9f5e421f 100644 --- a/velox/experimental/cudf/.clang-format +++ b/velox/experimental/cudf/.clang-format @@ -24,4 +24,4 @@ IncludeCategories: - Regex: '^<.*\..*' # other system includes (e.g. with a '.') Priority: 9 - Regex: '^<[^.]+' # STL includes (no '.') - Priority: 10 \ No newline at end of file + Priority: 10 diff --git a/velox/experimental/wave/README.md b/velox/experimental/wave/README.md index 008e48b50f4..9704aaee0ea 100644 --- a/velox/experimental/wave/README.md +++ b/velox/experimental/wave/README.md @@ -17,17 +17,16 @@ limitations under the License. # CMake: Use Base Functions > [!IMPORTANT] Please use `target_link_libraries` and `add_library` -> instead of the `velox_*` functions when adding or linking to targets +> instead of the `velox_*` functions when adding or linking to targets > within wave/ and label tests with `cuda_driver`. The `wave` GPU component links against the CUDA driver in several targets. They can be built on machines without the actual driver installed, this requires the relevant 'stub' packages to be installed (see setup scripts). -Any library that statically links against the stubs **can not** run on a -machine without an actual CUDA driver installed (like our CI). -For this reason we need to use the base functions to create standalone +Any library that statically links against the stubs **can not** run on a +machine without an actual CUDA driver installed (like our CI). +For this reason we need to use the base functions to create standalone libraries for wave to avoid linking statically against the stubs when building the monolithic library and label any tests with 'cuda_driver' to allow excluding them from ctest on machines without the driver. - diff --git a/velox/functions/prestosql/coverage/README.md b/velox/functions/prestosql/coverage/README.md index 769af4e805d..76459165c5a 100644 --- a/velox/functions/prestosql/coverage/README.md +++ b/velox/functions/prestosql/coverage/README.md @@ -15,7 +15,7 @@ to be copy-pasted into velox/docs/functions.rst file. Generates coverage map using all Presto functions. The output to be copy-pasted into velox/docs/functions/presto/coverage.rst file. The functions appear in alphabetical order. -Before generating the coverage map for all Presto functions, please ensure that the data +Before generating the coverage map for all Presto functions, please ensure that the data files at velox/functions/prestosql/coverage/data/ contain all the Presto functions. To generate a list of all Presto functions, please run ```SHOW FUNCTIONS``` in Presto. diff --git a/velox/python/arrow/arrow.pyi b/velox/python/arrow/arrow.pyi index 4302d56cbea..aec7a8b1c5f 100644 --- a/velox/python/arrow/arrow.pyi +++ b/velox/python/arrow/arrow.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +14,6 @@ # pyre-unsafe -from typing import List - from pyvelox.vector import Vector from pyarrow import Array diff --git a/velox/python/file/file.pyi b/velox/python/file/file.pyi index 0ecf36069a4..33b54c8a327 100644 --- a/velox/python/file/file.pyi +++ b/velox/python/file/file.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +16,6 @@ from pyvelox.type import Type - class File: def __init__(self, path: str, format_str: str) -> None: ... def get_schema(self) -> Type: ... diff --git a/velox/python/plan_builder/plan_builder.pyi b/velox/python/plan_builder/plan_builder.pyi index 2461ec9ece4..83cd5d1d523 100644 --- a/velox/python/plan_builder/plan_builder.pyi +++ b/velox/python/plan_builder/plan_builder.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +15,11 @@ # pyre-unsafe from enum import Enum -from typing import List, Dict, Type, Optional +from typing import Optional from pyvelox.file import File from pyvelox.type import Type - class JoinType(Enum): INNER = 1 LEFT = 2 @@ -41,11 +38,11 @@ class PlanBuilder: def table_scan( self, output_schema: Type, - aliases: Dict[str, str] = {}, - subfields: Dict[str, List[int]] = {}, + aliases: dict[str, str] = {}, + subfields: dict[str, list[int]] = {}, row_index: str = "", connector_id: str = "prism", - input_files: List[File] = [], + input_files: list[File] = [], ) -> PlanBuilder: ... def tpch_gen( self, @@ -53,7 +50,7 @@ class PlanBuilder: columns: list[str] = [], scale_factor: int = 1, num_parts: int = 1, - connector_id: str = "tpch" + connector_id: str = "tpch", ) -> PlanBuilder: ... def table_write( self, diff --git a/velox/python/runner/runner.pyi b/velox/python/runner/runner.pyi index b620d6e5ecb..d511692552b 100644 --- a/velox/python/runner/runner.pyi +++ b/velox/python/runner/runner.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +18,6 @@ from typing import Iterator, Optional from pyvelox.vector import Vector - class LocalRunner: def __init__(self, PlanNode) -> None: ... def execute(self, max_drivers: Optional[int] = None) -> Iterator[Vector]: ... diff --git a/velox/python/type/type.pyi b/velox/python/type/type.pyi index a807c3a391e..6d7f95fccd3 100644 --- a/velox/python/type/type.pyi +++ b/velox/python/type/type.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +16,6 @@ from typing import List - class Type: ... def BIGINT() -> Type: ... diff --git a/velox/python/vector/vector.pyi b/velox/python/vector/vector.pyi index 482837adf49..3504a1dae4a 100644 --- a/velox/python/vector/vector.pyi +++ b/velox/python/vector/vector.pyi @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +14,6 @@ # pyre-unsafe -from typing import List - - class Vector: def size(self) -> int: ... def print_all(self) -> str: ... diff --git a/velox/substrait/tests/data/q1_first_stage.json b/velox/substrait/tests/data/q1_first_stage.json index 1b9ba06231d..6b4c1cf7a49 100644 --- a/velox/substrait/tests/data/q1_first_stage.json +++ b/velox/substrait/tests/data/q1_first_stage.json @@ -874,4 +874,4 @@ } ], "expected_type_urls": [] -} \ No newline at end of file +} diff --git a/velox/substrait/tests/data/q6_first_stage.json b/velox/substrait/tests/data/q6_first_stage.json index b6c2f535df8..36597a36251 100644 --- a/velox/substrait/tests/data/q6_first_stage.json +++ b/velox/substrait/tests/data/q6_first_stage.json @@ -600,4 +600,4 @@ } } ] -} \ No newline at end of file +} diff --git a/website/README.md b/website/README.md index 9ebdfccf492..09465bc0eb9 100644 --- a/website/README.md +++ b/website/README.md @@ -31,4 +31,4 @@ Velox's website is automatically deployed using the files under *velox/website* is submitted, a live preview link is generated by Netlify. The link is posted in the pull request as a comment by the Netlify bot. When the pull request is merged, the changes are automatically deployed to -the website by Netlify. +the website by Netlify. diff --git a/website/blog/2023-03-15-build-experience.mdx b/website/blog/2023-03-15-build-experience.mdx index 164c893265e..d5ab05290cd 100644 --- a/website/blog/2023-03-15-build-experience.mdx +++ b/website/blog/2023-03-15-build-experience.mdx @@ -6,13 +6,13 @@ tags: [tech-blog, packaging] --- -When Velox was open sourced in August 2021, it was not nearly as easily usable and portable as it is today. In order for Velox to become the unified execution engine blurring the boundaries for data analytics and ML, we needed Velox to be easy to build and package on multiple platforms, and support a wide range of hardware architectures. If we are supporting all these platforms, we also need to ensure that Velox remains fast and regressions are caught early. +When Velox was open sourced in August 2021, it was not nearly as easily usable and portable as it is today. In order for Velox to become the unified execution engine blurring the boundaries for data analytics and ML, we needed Velox to be easy to build and package on multiple platforms, and support a wide range of hardware architectures. If we are supporting all these platforms, we also need to ensure that Velox remains fast and regressions are caught early. To improve the Velox experience for users and community developers, Velox has partnered with Voltron Data to help make Velox more accessible and user-friendly. In this blog post, we will examine the challenges we faced, the improvements that have already been made, and the ones yet to come. ## Enhancements & Improvements -Velox was a product of the mono repo and required installation of dependencies on the system via a script. Any change in the state of the host system could cause a build failure and introduce version conflicts of dependencies. Fixing these challenges was a big focus to help the Velox Community and we worked in collaboration with the Voltron Data Team. We wanted to improve the overall Velox user experience by making Velox easy to consume across a wide range of platforms to accelerate its adoption. +Velox was a product of the mono repo and required installation of dependencies on the system via a script. Any change in the state of the host system could cause a build failure and introduce version conflicts of dependencies. Fixing these challenges was a big focus to help the Velox Community and we worked in collaboration with the Voltron Data Team. We wanted to improve the overall Velox user experience by making Velox easy to consume across a wide range of platforms to accelerate its adoption. We choose hermetic builds as a solution to the aforementioned problems, as they provide a number of benefits. Hermetic builds[^1] improve reproducibility by providing isolation from the state of the host machine and produce the same result for any given commit in the Velox repository. This requires precise dependency management. diff --git a/website/blog/2024-05-31-optimize-try-more.mdx b/website/blog/2024-05-31-optimize-try-more.mdx index dc4b6edc5ce..d72f6aadb29 100644 --- a/website/blog/2024-05-31-optimize-try-more.mdx +++ b/website/blog/2024-05-31-optimize-try-more.mdx @@ -352,4 +352,4 @@ Thank you Bikramjeet VigOrri Erling, Pedro Eugenio Rocha Pedreira and Xiaoxuan Meng for brainstorming and -helping with code reviews. \ No newline at end of file +helping with code reviews. diff --git a/website/blog/2024-08-23-ci-migration.mdx b/website/blog/2024-08-23-ci-migration.mdx index 398c17279b6..f93a0d335da 100644 --- a/website/blog/2024-08-23-ci-migration.mdx +++ b/website/blog/2024-08-23-ci-migration.mdx @@ -22,7 +22,7 @@ When a pull request is submitted to Velox, the following jobs are executed: 1. Linting and Formatting workflows: 1. Header checks 2. License checks - 3. Basic Linters + 3. Basic Linters 2. Ensure Velox builds on various platforms 1. MacOS (Intel, M1) 2. Linux (Ubuntu/Centos) @@ -31,11 +31,11 @@ When a pull request is submitted to Velox, the following jobs are executed: 2. Build default Velox build 3. Build Velox with support for Parquet, Arrow and External Adapters (S3/HDFS/GCS etc.) 4. PyVelox builds -4. Run prerequisite tests +4. Run prerequisite tests 1. Unit Tests 2. Benchmarking Tests 1. [Conbench](https://velox-conbench.voltrondata.run/runs/5bd139fffa9b4e0eb020da4d63211121/) is used to store and compare results, and also alert users on regressions - 3. Various Fuzzer Tests (Expression / Aggregation/ Exchange / Join etc) + 3. Various Fuzzer Tests (Expression / Aggregation/ Exchange / Join etc) 4. Signature Check and Biased Fuzzer Tests ( Expression / Aggregation) 5. Fuzzer Tests using Presto as source of truth 5. Docker Image build jobs @@ -49,7 +49,7 @@ When a pull request is submitted to Velox, the following jobs are executed: ## Velox CI Optimization -Previous implementation of CI in CircleCI grew organically and was unoptimized, resulting in long build times, and also significantly costlier. This opportunity to migrate to Github Actions helped to take a holistic view of CI deployments and actively optimized to reduce build times and CI spend. Note however, that there has been continued investment in reducing test times to further improve Velox reliability, stability and developer experience. Some of the optimizations completed are: +Previous implementation of CI in CircleCI grew organically and was unoptimized, resulting in long build times, and also significantly costlier. This opportunity to migrate to Github Actions helped to take a holistic view of CI deployments and actively optimized to reduce build times and CI spend. Note however, that there has been continued investment in reducing test times to further improve Velox reliability, stability and developer experience. Some of the optimizations completed are: 1. **Persisting build artifacts across builds**: During every build, the object files and binaries produced are cached. In addition to this, artifacts such as scalar function signatures and aggregate function signatures are produced. These signatures are used to compare with the baseline version, by comparing against the changes in the current PR to determine if the current changes are backwards incompatible or bias the newly added changes. Using a stash to persist these artifacts helps save one build cycle. @@ -57,11 +57,11 @@ Previous implementation of CI in CircleCI grew organically and was unoptimized, ## Instrumenting Velox CI Builds -Velox CI builds were instrumented in Conbench so that it can capture various metrics about the builds: +Velox CI builds were instrumented in Conbench so that it can capture various metrics about the builds: 1. Build times at translation unit / library/ project level. 2. Binary sizes produced at TLU/ .a,.so / executable level. -3. Memory pressure -4. Measure across time how our changes affect binary sizes +3. Memory pressure +4. Measure across time how our changes affect binary sizes A nightly job is run to capture these build metrics and it is uploaded to Conbench. Velox build metrics report is available here: [Velox Build Metrics Report](https://facebookincubator.github.io/velox/bm-report/) @@ -69,10 +69,8 @@ A nightly job is run to capture these build metrics and it is uploaded to Conben ## Acknowledgements -A large part of the credit goes to Jacob Wujciak and the team at Voltron Data. We would also like to thank other collaborators in the Open Source Community and at Meta, including but not limited to: +A large part of the credit goes to Jacob Wujciak and the team at Voltron Data. We would also like to thank other collaborators in the Open Source Community and at Meta, including but not limited to: **Meta**: Sridhar Anumandla, Pedro Eugenio Rocha Pedreira, Deepak Majeti, Meta OSS Team, and others **Voltron Data**: Jacob Wujciak, Austin Dickey, Marcus Hanwell, Sri Nadukudy, and others - - diff --git a/website/blog/2025-03-25-velox-primer-part-2.mdx b/website/blog/2025-03-25-velox-primer-part-2.mdx index 48c793509cf..27407953695 100644 --- a/website/blog/2025-03-25-velox-primer-part-2.mdx +++ b/website/blog/2025-03-25-velox-primer-part-2.mdx @@ -33,9 +33,9 @@ distributed query plan with three query fragments will be created: divides its output according to a hash of *l_partkey*. 2. The second fragment reads the output from the first fragment and updates a hash table from *l_partkey* containing the number of times the particular value -of *l_partkey* has been seen (the count(*) aggregate function implementation). +of *l_partkey* has been seen (the count(*) aggregate function implementation). 3. The final fragment then reads the content of the hash tables, once the -second fragment has received all the rows from the first fragment. +second fragment has received all the rows from the first fragment.
@@ -71,7 +71,7 @@ In Prestissimo, the message that sets up a Task in a worker is called *Task Update*. A Task Update has the following information: the plan, configuration settings, and an optional list of splits. Splits are further qualified by what plan node they are intended for, and whether more splits for the recipient plan -node and split group will be coming. +node and split group will be coming. Since split generation involves enumerating files from storage (so they may take a while), Presto allows splits to be sent to workers asynchronously, such @@ -109,13 +109,13 @@ operator tree where each node consumes the output of its child operators, and returns output to the parent operator. The root node is typically a PartitionedOutputNode or a TableWriteNode. The leaf nodes are either TableScanNode, ExchangeNode or ValuesNode (used for query literals). The full -set of Velox PlanNode can be found at velox/core/PlanNode.h. +set of Velox PlanNode can be found at velox/core/PlanNode.h. The PlanNodes mostly correspond to Operators. PlanNodes are not executable as such; they are only a structure describing how to make Drivers and Operators, which do the actual execution. If the tree of nodes has a single branch, then the plan is a single pipeline. If it has nodes with more than one child -(input), then the second input of the node becomes a separate pipeline. +(input), then the second input of the node becomes a separate pipeline. `Task::start()` creates the DriverFactories, which then create the Drivers. To start execution, the Drivers are queued on a thread pool executor. The main @@ -151,7 +151,7 @@ that no more splits will be coming. In this case TableScan would be at the end. Finally, if a Split is available, TableScan interprets it. Given a TableHandle specification provided as part of the plan (list of columns and filters), the Connector (as specified in the Split) makes a DataSource. The DataSource -handles the details of IO and file and table formats. +handles the details of IO and file and table formats. The DataSource is then given the split. After this, DataSource::next() can be called repeatedly to get vectors (batches) of output from the file/section of diff --git a/website/blog/2025-05-12-velox-primer-part-3.mdx b/website/blog/2025-05-12-velox-primer-part-3.mdx index b09ea8beb34..424c6736cc9 100644 --- a/website/blog/2025-05-12-velox-primer-part-3.mdx +++ b/website/blog/2025-05-12-velox-primer-part-3.mdx @@ -19,7 +19,7 @@ stage of the query is executed, from table scan to partitioned output - or the producer side of the shuffle.   In this article, we will discuss the second query stage, or the consumer side -of the shuffle. +of the shuffle. ## Shuffle Consumer diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 016a74747ee..cffe9df5058 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -40,7 +40,7 @@ kgpai: name: Krishna Pai title: Software Engineer @ Meta url: https://github.com/kgpai - image_url: https://github.com/kgpai.png + image_url: https://github.com/kgpai.png jwujciak: name: Jacob Wujciak-Jens diff --git a/website/docs/community/01-design-philosophy.md b/website/docs/community/01-design-philosophy.md index 83902c8606b..8220f890ccc 100644 --- a/website/docs/community/01-design-philosophy.md +++ b/website/docs/community/01-design-philosophy.md @@ -8,7 +8,7 @@ title: Design Philosophy This page lists a set of directional principles and values meant to guide contributors and maintainers as they develop the Velox project. These are not meant to be hard-and-fast rules, but to inform decision making and help guide -discussions that may come up during the development of Velox. +discussions that may come up during the development of Velox. ## Velox Mission @@ -87,4 +87,3 @@ mission, a few overarching principles and values are highlighted below. * **Adaptivity.** Exposing too many configuration knobs to users increases the API complexity, and makes it more error-prone. As much as possible, we try to make the library self-adapt to find optimal execution configurations. - diff --git a/website/docs/community/02-technical-governance.md b/website/docs/community/02-technical-governance.md index b591ba723ef..a959b03c4e0 100644 --- a/website/docs/community/02-technical-governance.md +++ b/website/docs/community/02-technical-governance.md @@ -213,7 +213,7 @@ opened for discussion. Components maintainers are responsible for participating in the discussion, reviewing, providing feedback, and eventually approving the changes. If disputes are made over the change, component maintainers are responsible for settling the issue, or ultimately escalating to the PLC in case -they cannot reach consensus. +they cannot reach consensus. ### Re-Scope Project Components @@ -236,7 +236,7 @@ project. Except as described below, all code contributions to the project must be made using the Apache 2.0 License available here: [https://www.apache.org/licenses/LICENSE-2.0 -](https://www.apache.org/licenses/LICENSE-2.0) (the "Project License"). +](https://www.apache.org/licenses/LICENSE-2.0) (the "Project License"). All outbound code will be made available under the Project License. The maintainers may approve the use of an alternative open license or licenses for @@ -249,7 +249,7 @@ start?** The Velox [contributing guide](https://github.com/facebookincubator/velox/blob/main/CONTRIBUTING.md) provides guidelines on how new community members can get involved in the -project. +project. **Q: Is it possible for an external contributor to become a maintainer and be granted responsibilities over parts of the codebase?** @@ -266,7 +266,7 @@ individuals, and membership in these groups is based on merit in the community. **Q: How do I contribute code to the project?** If the change is relatively minor, a pull request on GitHub can be opened up immediately for review by the project maintainers. For larger changes, please -open an Issue to make a proposal to discuss prior. Please also see the +open an Issue to make a proposal to discuss prior. Please also see the [Velox Contributor Guide](https://github.com/facebookincubator/velox/blob/main/CONTRIBUTING.md) for contribution guidelines. diff --git a/website/docs/community/03-components-and-maintainers.md b/website/docs/community/03-components-and-maintainers.md index a4eaf1a4f3c..2983c33e8f0 100644 --- a/website/docs/community/03-components-and-maintainers.md +++ b/website/docs/community/03-components-and-maintainers.md @@ -14,7 +14,7 @@ and codebase. Maintainership status is *lagging*, not *leading*, and it is only acquired through active participation and demonstration of skills and domain-specific knowledge. All individuals listed in this page are expected to uphold -[Velox’s mission, design philosophy, and principles](./design-philosophy). +[Velox’s mission, design philosophy, and principles](./design-philosophy). ## Project Leadership Council - PLC @@ -35,7 +35,7 @@ developer documentation. Before working on a new feature or optimization, please review our [CONTRIBUTING.md](https://github.com/facebookincubator/velox/blob/main/CONTRIBUTING.md) guide and initiate a discussion on Github with the people listed as -maintainers of that component. +maintainers of that component. ### Vectors, Types, Arrow Bindings: diff --git a/website/docs/community/index.md b/website/docs/community/index.md index 79ccf952549..6d5c82ed653 100644 --- a/website/docs/community/index.md +++ b/website/docs/community/index.md @@ -4,8 +4,8 @@ slug: /community/ # Community -Velox is a project created and -[open sourced by Meta in 2023](https://engineering.fb.com/2023/03/09/open-source/velox-open-source-execution-engine/). +Velox is a project created and +[open sourced by Meta in 2023](https://engineering.fb.com/2023/03/09/open-source/velox-open-source-execution-engine/). Today, Velox is developed and maintained by a community of 200+ individuals from 20+ different organizations. This page contains more information about Velox's open source community. diff --git a/website/src/components/HomepageFeatures/index.js b/website/src/components/HomepageFeatures/index.js index a3b6ea574b0..beb1752ef72 100644 --- a/website/src/components/HomepageFeatures/index.js +++ b/website/src/components/HomepageFeatures/index.js @@ -27,7 +27,7 @@ export default function HomepageFeatures() { ))}
- +
diff --git a/website/src/components/VeloxConBanner/index.js b/website/src/components/VeloxConBanner/index.js index 76a892a924a..78dce089f33 100644 --- a/website/src/components/VeloxConBanner/index.js +++ b/website/src/components/VeloxConBanner/index.js @@ -25,4 +25,4 @@ export default function VeloxConBanner() { ); -} \ No newline at end of file +} diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 448d4262973..c97d5d25c6b 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -128,7 +128,7 @@ function KeyFeatures() { Reusability Icon

Reusability

- Features and runtime optimizations available in Velox are developed and maintained once, reducing engineering duplication and promoting reusability. + Features and runtime optimizations available in Velox are developed and maintained once, reducing engineering duplication and promoting reusability.

diff --git a/website/static/img/banner-pattern.svg b/website/static/img/banner-pattern.svg index 44c0b038939..69a146553a2 100644 --- a/website/static/img/banner-pattern.svg +++ b/website/static/img/banner-pattern.svg @@ -100,4 +100,4 @@ - \ No newline at end of file + diff --git a/website/static/img/icon-commits.svg b/website/static/img/icon-commits.svg index f027389614f..7ec9b630e44 100644 --- a/website/static/img/icon-commits.svg +++ b/website/static/img/icon-commits.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/website/static/img/icon-contributors.svg b/website/static/img/icon-contributors.svg index 1e08857ef96..68eccc8d0b6 100644 --- a/website/static/img/icon-contributors.svg +++ b/website/static/img/icon-contributors.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/website/static/img/icon-github-star.svg b/website/static/img/icon-github-star.svg index 841dd97b317..28297614d12 100644 --- a/website/static/img/icon-github-star.svg +++ b/website/static/img/icon-github-star.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/website/static/img/logo.svg b/website/static/img/logo.svg index d3e7a794ddc..8ceed2d92a5 100644 --- a/website/static/img/logo.svg +++ b/website/static/img/logo.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/website/static/img/velox-logo.svg b/website/static/img/velox-logo.svg index 67627d4cfd2..0db7eec5f1f 100644 --- a/website/static/img/velox-logo.svg +++ b/website/static/img/velox-logo.svg @@ -1 +1 @@ - \ No newline at end of file +