From c0ba18bc49d79ff3fb38e8575308de11dbd56868 Mon Sep 17 00:00:00 2001 From: Andrew Xie Date: Wed, 16 Jul 2025 12:46:15 -0700 Subject: [PATCH] ci: Use pre-commit for formatting checks Added hooks include license header checks, clang-format, gersemi (CMake formatter), shfmt, ruff (Python formatter), and multiple hooks for GitHub workflows. Checks only run on files in the `presto-native-execution` and `.github` directories. Ran pre-commit hooks on all files to reformat them. Over 100 security warnings in GitHub workflows from zizmor were addressed. Co-authored-by: Christian Zentgraf --- .github/actions/maven-owasp-scan/action.yml | 26 +- .github/labeler.yml | 2 +- .github/stale.yml | 32 +- .github/workflows/arrow-flight-tests.yml | 36 +- .github/workflows/codenotify.yml | 10 +- .github/workflows/dependency-review.yml | 12 +- .github/workflows/docs.yml | 19 +- .github/workflows/hive-tests.yml | 47 +- .github/workflows/kudu.yml | 32 +- .github/workflows/maven-checks.yml | 16 +- .github/workflows/owasp-dependency-check.yml | 24 +- .github/workflows/presto-release-prepare.yml | 46 +- .github/workflows/presto-release-publish.yml | 143 ++-- .../prestocpp-format-and-header-check.yml | 33 +- .../prestocpp-linux-adapters-build.yml | 17 +- .../prestocpp-linux-build-and-unit-test.yml | 116 +-- .github/workflows/prestocpp-linux-build.yml | 23 +- .github/workflows/prestocpp-macos-build.yml | 16 +- .../product-tests-basic-environment.yml | 32 +- .../product-tests-specific-environment.yml | 41 +- .github/workflows/release-notes-check.yml | 9 +- .github/workflows/singlestore-tests.yml | 22 +- .github/workflows/spark-integration.yml | 34 +- .github/workflows/test-other-modules.yml | 62 +- .github/workflows/tests.yml | 96 +-- .github/workflows/web-ui-checks.yml | 22 +- .github/zizmor.yml | 4 + .pre-commit-config.yaml | 116 +++ .yamlfmt.yml | 20 + .yamllint.yml | 44 ++ CONTRIBUTING.md | 8 + presto-native-execution/.clang-tidy | 96 +++ presto-native-execution/.gersemirc | 6 + presto-native-execution/CMakeLists.txt | 117 ++- presto-native-execution/Makefile | 29 - presto-native-execution/README.md | 59 +- presto-native-execution/docker-compose.yml | 17 +- presto-native-execution/etc/velox.properties | 0 .../docs/develop/orc-dump-output.rst | 2 +- .../presto_cpp/docs/index.rst | 1 - .../presto_cpp/main/CMakeLists.txt | 59 +- .../presto_cpp/main/PeriodicMemoryChecker.h | 5 +- .../presto_cpp/main/PeriodicTaskManager.cpp | 51 +- .../presto_cpp/main/PeriodicTaskManager.h | 2 +- .../presto_cpp/main/PrestoExchangeSource.cpp | 22 +- .../presto_cpp/main/PrestoExchangeSource.h | 4 +- .../presto_cpp/main/PrestoServer.h | 16 +- .../presto_cpp/main/PrestoTask.cpp | 41 +- .../main/PrestoToVeloxQueryConfig.cpp | 13 +- .../main/PrestoToVeloxQueryConfig.h | 10 +- .../presto_cpp/main/QueryContextManager.cpp | 3 +- .../presto_cpp/main/QueryContextManager.h | 19 +- .../presto_cpp/main/SessionProperties.cpp | 10 +- .../presto_cpp/main/SessionProperties.h | 17 +- .../presto_cpp/main/TaskManager.cpp | 17 +- .../presto_cpp/main/TaskResource.cpp | 32 +- .../presto_cpp/main/common/CMakeLists.txt | 12 +- .../presto_cpp/main/common/Configs.cpp | 26 +- .../presto_cpp/main/common/Configs.h | 5 +- .../presto_cpp/main/common/Counters.cpp | 15 +- .../presto_cpp/main/common/Counters.h | 4 +- .../presto_cpp/main/common/Utils.h | 2 +- .../main/common/tests/CMakeLists.txt | 11 +- .../main/common/tests/CommonTest.cpp | 5 +- .../main/common/tests/ConfigTest.cpp | 6 +- .../main/common/tests/MutableConfigs.cpp | 2 +- .../presto_cpp/main/common/tests/test_json.h | 8 +- .../presto_cpp/main/connectors/CMakeLists.txt | 27 +- .../IcebergPrestoToVeloxConnector.cpp | 2 +- .../main/connectors/PrestoToVeloxConnector.h | 6 +- .../main/connectors/Registration.cpp | 32 +- .../presto_cpp/main/connectors/Registration.h | 6 +- .../main/connectors/SystemConnector.cpp | 3 +- .../main/connectors/SystemConnector.h | 6 +- .../arrow_flight/ArrowFlightConfig.h | 5 +- .../arrow_flight/ArrowFlightConnector.cpp | 8 +- .../ArrowPrestoToVeloxConnector.h | 3 +- .../connectors/arrow_flight/CMakeLists.txt | 20 +- .../arrow_flight/auth/CMakeLists.txt | 3 +- .../ArrowFlightConnectorDataTypeTest.cpp | 111 +-- .../tests/ArrowFlightConnectorMTlsTest.cpp | 36 +- .../tests/ArrowFlightConnectorTest.cpp | 37 +- .../arrow_flight/tests/CMakeLists.txt | 33 +- .../tests/data/generate_tls_certs.sh | 0 .../utils/ArrowFlightConnectorTestBase.cpp | 5 +- .../arrow_flight/tests/utils/CMakeLists.txt | 16 +- .../functions/dynamic_registry/CMakeLists.txt | 3 +- .../main/functions/dynamic_registry/README.md | 2 +- .../dynamic_registry/examples/CMakeLists.txt | 36 +- .../examples/DynamicVarcharFunction.cpp | 2 - .../main/functions/tests/CMakeLists.txt | 6 +- .../functions/tests/FunctionMetadataTest.cpp | 5 +- .../main/functions/tests/data/Greatest.json | 4 +- .../presto_cpp/main/http/CMakeLists.txt | 6 +- .../presto_cpp/main/http/HttpClient.cpp | 18 +- .../presto_cpp/main/http/HttpServer.cpp | 4 +- .../main/http/filters/AccessLogFilter.cpp | 2 +- .../main/http/filters/CMakeLists.txt | 12 +- .../filters/HttpEndpointLatencyFilter.cpp | 29 +- .../http/filters/HttpEndpointLatencyFilter.h | 9 +- .../filters/InternalAuthenticationFilter.cpp | 6 +- .../main/http/filters/tests/CMakeLists.txt | 12 +- .../presto_cpp/main/http/tests/CMakeLists.txt | 15 +- .../presto_cpp/main/http/tests/HttpTestBase.h | 5 +- .../main/http/tests/certs/client_ca.pem | 2 +- .../main/http/tests/certs/test_cert1.pem | 2 +- .../main/http/tests/certs/test_key1.pem | 2 +- .../main/operators/BroadcastFactory.cpp | 11 +- .../presto_cpp/main/operators/CMakeLists.txt | 6 +- .../main/operators/ShuffleExchangeSource.cpp | 12 +- .../BinarySortableSerializerBenchmark.cpp | 2 - .../main/operators/benchmarks/CMakeLists.txt | 10 +- .../tests/BinarySortableSerializerTest.cpp | 66 +- .../main/operators/tests/BroadcastTest.cpp | 5 +- .../main/operators/tests/CMakeLists.txt | 14 +- .../main/operators/tests/ShuffleTest.cpp | 39 +- .../main/runtime-metrics/CMakeLists.txt | 3 +- .../PrometheusStatsReporter.cpp | 17 +- .../runtime-metrics/PrometheusStatsReporter.h | 23 +- .../main/runtime-metrics/tests/CMakeLists.txt | 12 +- .../tests/PrometheusReporterTest.cpp | 9 +- .../presto_cpp/main/tests/CMakeLists.txt | 37 +- .../presto_cpp/main/tests/ConnectorTest.cpp | 5 +- .../main/tests/JsonSignatureParserTest.cpp | 7 +- .../main/tests/LinuxMemoryCheckerTest.cpp | 71 +- .../main/tests/PeriodicMemoryCheckerTest.cpp | 29 +- .../main/tests/PrestoExchangeSourceTest.cpp | 2 +- .../presto_cpp/main/tests/PrestoTaskTest.cpp | 2 +- .../tests/PrestoToVeloxQueryConfigTest.cpp | 6 +- .../main/tests/QueryContextCacheTest.cpp | 2 +- .../main/tests/QueryContextManagerTest.cpp | 36 +- .../presto_cpp/main/tests/TaskInfoTest.cpp | 39 +- .../presto_cpp/main/tests/TaskManagerTest.cpp | 57 +- .../presto_cpp/main/tests/TaskStatusTest.cpp | 51 +- .../main/tests/TaskUpdateRequestTest.cpp | 141 ++-- .../presto_cpp/main/tests/certs/client_ca.pem | 2 +- .../main/tests/certs/test_cert1.pem | 2 +- .../presto_cpp/main/tests/certs/test_key1.pem | 2 +- .../presto_cpp/main/thrift/CMakeLists.txt | 44 +- .../thrift/presto_protocol-to-thrift-json.py | 44 +- .../thrift/presto_protocol-to-thrift-json.yml | 4 +- .../main/thrift/temp_presto_thrift.thrift | 710 ++++++++++++++++++ .../main/thrift/tests/CMakeLists.txt | 24 +- .../presto_cpp/main/thrift/thrift2json.py | 8 +- .../presto_cpp/main/types/CMakeLists.txt | 9 +- .../main/types/PrestoToVeloxExpr.cpp | 4 +- .../main/types/PrestoToVeloxQueryPlan.cpp | 17 +- .../presto_cpp/main/types/TypeParser.cpp | 3 +- .../main/types/tests/CMakeLists.txt | 36 +- .../tests/PrestoToVeloxConnectorTest.cpp | 15 +- .../main/types/tests/RowExpressionTest.cpp | 10 +- .../presto_cpp/main/types/tests/TestUtils.cpp | 28 +- .../presto_cpp/main/types/tests/TestUtils.h | 6 +- .../main/types/tests/TypeParserTest.cpp | 42 +- .../types/tests/data/PartitionedOutput.json | 2 +- .../main/types/tests/data/ValuesNode.json | 2 +- .../main/types/tests/data/ValuesPipeTest.json | 1 - .../presto_cpp/presto_protocol/CMakeLists.txt | 10 +- .../presto_cpp/presto_protocol/Makefile | 2 +- .../presto_cpp/presto_protocol/README.md | 4 +- .../connector/hive/presto_protocol_hive.cpp | 20 +- .../iceberg/presto_protocol-json-hpp.mustache | 2 +- .../iceberg/presto_protocol_iceberg.cpp | 28 +- .../iceberg/presto_protocol_iceberg.yml | 1 - .../presto_protocol/core/ConnectorProtocol.h | 87 ++- .../core/presto_protocol_core.cpp | 83 +- .../special/SystemPartitioningHandle.java | 2 +- .../presto_protocol/java-to-struct-json.py | 4 +- .../presto_protocol-protobuf.mustache | 1 - .../presto_protocol/tests/CMakeLists.txt | 9 +- .../tests/PlanFragmentTest.cpp | 4 +- .../tests/RowExpressionTest.cpp | 2 +- .../presto_protocol/tests/TupleDomainTest.cpp | 4 +- .../tests/data/ValuesNode.json | 2 +- .../presto_cpp/presto_protocol/util.py | 2 +- presto-native-execution/rhel_build.log | 1 + presto-native-execution/scripts/check.py | 287 ------- .../scripts/setup-adapters.sh | 47 +- .../scripts/setup-centos.sh | 9 +- .../scripts/setup-macos.sh | 8 +- .../scripts/setup-ubuntu.sh | 2 +- presto-native-execution/scripts/util.py | 2 +- 182 files changed, 2942 insertions(+), 1855 deletions(-) create mode 100644 .github/zizmor.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .yamlfmt.yml create mode 100644 .yamllint.yml create mode 100644 presto-native-execution/.clang-tidy create mode 100644 presto-native-execution/.gersemirc create mode 100644 presto-native-execution/etc/velox.properties create mode 100755 presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/data/generate_tls_certs.sh create mode 100644 presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift create mode 100644 presto-native-execution/rhel_build.log delete mode 100755 presto-native-execution/scripts/check.py diff --git a/.github/actions/maven-owasp-scan/action.yml b/.github/actions/maven-owasp-scan/action.yml index 9681434fad2d6..ac3f14da7bb48 100644 --- a/.github/actions/maven-owasp-scan/action.yml +++ b/.github/actions/maven-owasp-scan/action.yml @@ -1,26 +1,30 @@ -name: 'Maven OWASP Dependency Check Scan' -description: 'Runs OWASP dependency check Maven scan with consistent settings' +name: Maven OWASP Dependency Check Scan +description: Runs OWASP dependency check Maven scan with consistent settings inputs: working-directory: - description: 'Working directory for Maven command' + description: Working directory for Maven command required: false - default: '.' + default: . owasp-version: - description: 'OWASP dependency check plugin version' + description: OWASP dependency check plugin version required: false - default: '12.1.3' + default: 12.1.3 data-directory: - description: 'OWASP data directory path' + description: OWASP data directory path required: false - default: '$HOME/.owasp/dependency-check-data' + default: /tmp/.owasp/dependency-check-data + runs: - using: 'composite' + using: composite steps: - name: Run OWASP dependency check + env: + OWASP_VERSION: ${{ inputs.owasp-version }} + OWASP_DATA_DIRECTORY: ${{ inputs.data-directory }} working-directory: ${{ inputs.working-directory }} shell: bash run: | - mvn org.owasp:dependency-check-maven:${{ inputs.owasp-version }}:aggregate \ + mvn org.owasp:dependency-check-maven:$OWASP_VERSION:aggregate \ -DskipTests \ -Dformat=JSON \ -DprettyPrint=true \ @@ -32,4 +36,4 @@ runs: -DcentralAnalyzerEnabled=false \ -DnuspecAnalyzerEnabled=false \ -DnvdValidForHours=168 \ - -DdataDirectory=${{ inputs.data-directory }} \ No newline at end of file + -DdataDirectory=$OWASP_DATA_DIRECTORY diff --git a/.github/labeler.yml b/.github/labeler.yml index 2b34a25c16b91..adfd5ebb7765a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,2 +1,2 @@ docs: - - all: ['**/*.rst'] \ No newline at end of file + - all: ['**/*.rst'] diff --git a/.github/stale.yml b/.github/stale.yml index 2187bdd7550c1..93109824aba8b 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -2,21 +2,21 @@ staleLabel: stale daysUntilClose: 7 pulls: - daysUntilStale: 180 # ~6 months - markComment: > - This pull request has been automatically marked as stale because it has not had - recent activity. If you'd still like this PR merged, please comment on the task, - make sure you've addressed reviewer comments, and rebase on the latest master. - Thank you for your contributions! + daysUntilStale: 180 # ~6 months + markComment: > + This pull request has been automatically marked as stale because it has not had + recent activity. If you'd still like this PR merged, please comment on the task, + make sure you've addressed reviewer comments, and rebase on the latest master. + Thank you for your contributions! issues: - daysUntilStale: 730 # ~2 years - exemptLabels: - - bug - markComment: > - This issue has been automatically marked as stale because it has not had - any activity in the last 2 years. If you feel that this issue is important, - just comment and the stale tag will be removed; otherwise it will be - closed in 7 days. This is an attempt to ensure that our open issues remain - valuable and relevant so that we can keep track of what needs to be done and - prioritize the right things. + daysUntilStale: 730 # ~2 years + exemptLabels: + - bug + markComment: > + This issue has been automatically marked as stale because it has not had + any activity in the last 2 years. If you feel that this issue is important, + just comment and the stale tag will be removed; otherwise it will be + closed in 7 days. This is an attempt to ensure that our open issues remain + valuable and relevant so that we can keep track of what needs to be done and + prioritize the right things. diff --git a/.github/workflows/arrow-flight-tests.yml b/.github/workflows/arrow-flight-tests.yml index 22ebf9439fcb2..d9ddab9acc9f0 100644 --- a/.github/workflows/arrow-flight-tests.yml +++ b/.github/workflows/arrow-flight-tests.yml @@ -3,14 +3,17 @@ name: arrow flight tests on: pull_request: paths-ignore: - - 'presto-docs/**' + - presto-docs/** + +permissions: + contents: read env: CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -19,9 +22,9 @@ jobs: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] modules: - - ":presto-base-arrow-flight" # Only run tests for the `presto-base-arrow-flight` module + - :presto-base-arrow-flight # Only run tests for the `presto-base-arrow-flight` module timeout-minutes: 80 concurrency: @@ -33,13 +36,14 @@ jobs: - uses: actions/checkout@v4 with: show-progress: false + persist-credentials: false # Set up Java and dependencies for the build environment - uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache run: .github/bin/download_nodejs @@ -61,8 +65,12 @@ jobs: CCACHE_DIR: "${{ github.workspace }}/ccache" DEPENDENCY_DIR: "${{ github.workspace }}/adapter-deps/download" INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install" + permissions: + actions: write steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Fix git permissions # Usually actions/checkout does this but as we run in a container @@ -150,7 +158,7 @@ jobs: fail-fast: false matrix: modules: - - ":presto-base-arrow-flight" # Only run tests for the `presto-base-arrow-flight` module + - :presto-base-arrow-flight # Only run tests for the `presto-base-arrow-flight` module timeout-minutes: 80 concurrency: @@ -159,6 +167,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Fix git permissions # Usually actions/checkout does this but as we run in a container @@ -187,16 +197,16 @@ jobs: - name: Install OpenJDK8 uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache run: .github/bin/download_nodejs - name: Maven install env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }} diff --git a/.github/workflows/codenotify.yml b/.github/workflows/codenotify.yml index b2ae421053b7e..ac6c976497241 100644 --- a/.github/workflows/codenotify.yml +++ b/.github/workflows/codenotify.yml @@ -9,15 +9,19 @@ on: jobs: codenotify: runs-on: ubuntu-latest - name: "Label and notify" + name: Label and notify + permissions: + contents: read + pull-requests: write steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} show-progress: false - - uses: sourcegraph/codenotify@v0.6.4 + persist-credentials: false + - uses: sourcegraph/codenotify@54e4320f0d93f162a371d8d9dc1fb11018199746 # v0.6.4 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: actions/labeler@v4.3.0 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 38c67b8ddae6b..1d3753fc8fb75 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -1,5 +1,5 @@ -name: 'Dependency Review' -on: [pull_request] +name: Dependency Review +on: pull_request permissions: contents: read @@ -9,10 +9,12 @@ jobs: dependency-review: runs-on: ubuntu-latest steps: - - name: 'Checkout Repository' + - name: Checkout Repository uses: actions/checkout@v4 + with: + persist-credentials: false - - name: 'Dependency Review' + - name: Dependency Review uses: actions/dependency-review-action@v4 with: fail-on-severity: high @@ -20,4 +22,4 @@ jobs: comment-summary-in-pr: always show-openssf-scorecard: true warn-on-openssf-scorecard-level: 5 - retry-on-snapshot-warnings: true \ No newline at end of file + retry-on-snapshot-warnings: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc6bb5de6b955..74e148fe4105e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,25 +3,27 @@ name: docs on: pull_request: paths: - - 'presto-docs/**' + - presto-docs/** env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: test: runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: modules: - - ":presto-docs" + - :presto-docs timeout-minutes: 80 concurrency: group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }} @@ -30,15 +32,16 @@ jobs: - uses: actions/checkout@v4 with: show-progress: false + persist-credentials: false - name: Install LaTeX dependencies run: | sudo apt-get update sudo apt-get install -y texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra latexmk tex-gyre texlive-xetex fonts-freefont-otf xindy - uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 97b707f6aed6b..05eb630e5a3fe 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -1,15 +1,14 @@ name: hive tests -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -22,19 +21,21 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' hive-tests: + permissions: + contents: read strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest needs: changes timeout-minutes: 60 @@ -46,12 +47,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs @@ -68,8 +70,8 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.HIVE_AWS_ACCESSKEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.HIVE_AWS_SECRETKEY }} - S3_BUCKET: "presto-ci-test" - S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com" + S3_BUCKET: presto-ci-test + S3_BUCKET_ENDPOINT: s3.us-east-2.amazonaws.com run: | if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then presto-hive-hadoop2/bin/run_hive_s3_tests.sh @@ -85,10 +87,12 @@ jobs: fi hive-dockerized-tests: + permissions: + contents: read strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest needs: changes timeout-minutes: 20 @@ -100,12 +104,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index 7969d9cc565a5..831ce311087cd 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -1,14 +1,13 @@ name: kudu -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true RETRY: .github/bin/retry jobs: @@ -21,18 +20,20 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' kudu: + permissions: + contents: read strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest needs: changes timeout-minutes: 60 @@ -44,12 +45,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 2b5f35eefe920..79782713bb0ed 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -1,21 +1,22 @@ name: maven checks -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError RETRY: .github/bin/retry jobs: maven-checks: + permissions: + contents: read strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest timeout-minutes: 45 concurrency: @@ -30,11 +31,12 @@ jobs: - uses: actions/checkout@v4 with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache run: .github/bin/download_nodejs - name: Maven Checks diff --git a/.github/workflows/owasp-dependency-check.yml b/.github/workflows/owasp-dependency-check.yml index c2834b0a31d5c..0cc8c768d81c3 100644 --- a/.github/workflows/owasp-dependency-check.yml +++ b/.github/workflows/owasp-dependency-check.yml @@ -2,11 +2,11 @@ name: Maven OWASP Dependency Check permissions: contents: read on: - pull_request: + pull_request: {} workflow_dispatch: inputs: cvss-threshold: - description: 'CVSS score threshold for failing (7.0 = high/critical)' + description: CVSS score threshold for failing (7.0 = high/critical) required: false default: '7.0' type: string @@ -16,26 +16,28 @@ jobs: runs-on: ubuntu-latest env: CVSS_THRESHOLD: ${{ github.event.inputs.cvss-threshold || '7.0' }} - OWASP_VERSION: '12.1.3' + OWASP_VERSION: 12.1.3 steps: # Checkout PR branch first to get access to the composite action - name: Checkout PR branch uses: actions/checkout@v4 with: + persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - name: Checkout base branch uses: actions/checkout@v4 with: + persist-credentials: false ref: ${{ github.event.pull_request.base.sha }} path: base - name: Set up Java uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17 - cache: 'maven' + cache: maven - name: Get date for cache key id: get-date @@ -45,7 +47,7 @@ jobs: uses: actions/cache/restore@v4 id: cache-owasp-restore with: - path: ~/.owasp/dependency-check-data + path: /tmp/.owasp/dependency-check-data key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }} restore-keys: | owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}- @@ -56,13 +58,13 @@ jobs: with: working-directory: base owasp-version: ${{ env.OWASP_VERSION }} - data-directory: $HOME/.owasp/dependency-check-data + data-directory: /tmp/.owasp/dependency-check-data - name: Save OWASP cache after base scan if: steps.cache-owasp-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: ~/.owasp/dependency-check-data + path: /tmp/.owasp/dependency-check-data key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }}-partial - name: Run OWASP check on PR branch @@ -70,12 +72,12 @@ jobs: with: working-directory: . owasp-version: ${{ env.OWASP_VERSION }} - data-directory: $HOME/.owasp/dependency-check-data + data-directory: /tmp/.owasp/dependency-check-data - name: Compare and fail on new CVEs above threshold run: | # Extract CVEs above threshold from both branches (CVSS >= $CVSS_THRESHOLD) - threshold="${{ env.CVSS_THRESHOLD }}" + threshold=$CVSS_THRESHOLD # Validate report files exist if [ ! -f base/target/dependency-check-report.json ]; then @@ -139,7 +141,7 @@ jobs: if: always() uses: actions/cache/save@v4 with: - path: ~/.owasp/dependency-check-data + path: /tmp/.owasp/dependency-check-data key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }} - name: Upload reports diff --git a/.github/workflows/presto-release-prepare.yml b/.github/workflows/presto-release-prepare.yml index 34a411791ff2d..0b07355bcf3aa 100644 --- a/.github/workflows/presto-release-prepare.yml +++ b/.github/workflows/presto-release-prepare.yml @@ -4,12 +4,12 @@ on: workflow_dispatch: inputs: prepare_release: - description: 'Prepare release branch and tag' + description: Prepare release branch and tag type: boolean default: true required: false prepare_release_notes: - description: 'Prepare release notes pull request' + description: Prepare release notes pull request type: boolean default: true required: false @@ -36,15 +36,18 @@ jobs: token: ${{ secrets.PRESTODB_CI_TOKEN }} show-progress: false fetch-depth: 5 + persist-credentials: true - name: Ensure checkout latest code + env: + REF_NAME: ${{ github.ref_name }} run: | - git fetch origin ${{ github.ref_name }} - if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/${{ github.ref_name }})" ]; then - echo "Branch ${{ github.ref_name }} has new commits. Resetting to latest." - git reset --hard origin/${{ github.ref_name }} + git fetch origin $REF_NAME + if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/$REF_NAME)" ]; then + echo "Branch $REF_NAME has new commits. Resetting to latest." + git reset --hard origin/$REF_NAME else - echo "Branch ${{ github.ref_name }} is already up to date." + echo "Branch $REF_NAME is already up to date." fi - name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }} @@ -54,10 +57,13 @@ jobs: distribution: ${{ env.JAVA_DISTRIBUTION }} - name: Configure git + env: + GIT_CI_EMAIL: ${{ env.GIT_CI_EMAIL }} + GIT_CI_USER: ${{ env.GIT_CI_USER }} run: | git config --global --add safe.directory ${{github.workspace}} - git config --global user.email "${{ env.GIT_CI_EMAIL }}" - git config --global user.name "${{ env.GIT_CI_USER }}" + git config --global user.email "$GIT_CI_EMAIL" + git config --global user.name "$GIT_CI_USER" git config --global alias.ls 'log --pretty=format:"%cd %h %ce: %s" --date=short --no-merges' git config pull.rebase false @@ -82,10 +88,14 @@ jobs: -DdevelopmentVersion=${{ env.PRESTO_RELEASE_VERSION }} \ -DreleaseVersion=${{ env.PRESTO_RELEASE_VERSION }} grep -m 2 "" pom.xml - echo "commits on branch ${{ github.ref_name }}" + echo "commits on branch $REF_NAME" git ls -5 + env: + REF_NAME: ${{ github.ref_name }} - name: Push release tag, branch and commits + env: + REF_NAME: ${{ github.ref_name }} run: | echo "In case this job failed, please delete the tag ${{ env.PRESTO_RELEASE_VERSION }} and the branch release-${{ env.PRESTO_RELEASE_VERSION }}, and re-run the job" git checkout ${{ env.PRESTO_RELEASE_VERSION }} @@ -98,11 +108,11 @@ jobs: echo -e "\nPushed release tag to: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ env.PRESTO_RELEASE_VERSION }}" echo "Pushed release branch to: ${{ github.server_url }}/${{ github.repository }}/tree/release-${{ env.PRESTO_RELEASE_VERSION }}" - echo "Pushing to branch ${{ github.ref_name }}" - git checkout ${{ github.ref_name }} - echo "commits on branch ${{ github.ref_name }}" + echo "Pushing to branch $REF_NAME" + git checkout $REF_NAME + echo "commits on branch $REF_NAME" git ls -5 - git push origin ${{ github.ref_name }} + git push origin $REF_NAME prepare-release-notes: needs: prepare-release-branch @@ -119,6 +129,7 @@ jobs: fetch-tags: true show-progress: false fetch-depth: 0 + persist-credentials: true - name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }} uses: actions/setup-java@v4 @@ -127,10 +138,13 @@ jobs: distribution: ${{ env.JAVA_DISTRIBUTION }} - name: Configure git + env: + GIT_CI_EMAIL: ${{ env.GIT_CI_EMAIL }} + GIT_CI_USER: ${{ env.GIT_CI_USER }} run: | git config --global --add safe.directory ${{github.workspace}} - git config --global user.email "${{ env.GIT_CI_EMAIL }}" - git config --global user.name "${{ env.GIT_CI_USER }}" + git config --global user.email "$GIT_CI_EMAIL" + git config --global user.name "$GIT_CI_USER" git config pull.rebase false - name: Add git upstream diff --git a/.github/workflows/presto-release-publish.yml b/.github/workflows/presto-release-publish.yml index 78e156eeea97a..3ac315f13d229 100644 --- a/.github/workflows/presto-release-publish.yml +++ b/.github/workflows/presto-release-publish.yml @@ -4,47 +4,47 @@ on: workflow_dispatch: inputs: RELEASE_VERSION: - description: 'Release version (e.g., 0.292)' + description: Release version (e.g., 0.292) required: true publish_release_tag: - description: 'Publish release tag' + description: Publish release tag type: boolean default: true required: false release-notes-commit: - description: 'Commit SHA of release notes(required when publish_release_tag is checked)' + description: Commit SHA of release notes(required when publish_release_tag is checked) required: false publish_maven: - description: 'Publish maven artifacts' + description: Publish maven artifacts type: boolean default: true required: false publish_github_release: - description: 'Publish executable jars to github release' + description: Publish executable jars to github release type: boolean default: true required: false publish_image: - description: 'Publish presto docker image' + description: Publish presto docker image type: boolean default: true required: false publish_native_image: - description: 'Publish prestissimo docker image' + description: Publish prestissimo docker image type: boolean default: true required: false dependency_image: - description: 'prestissimo dependency image(e.g., prestodb/presto-native-dependency:latest)' + description: prestissimo dependency image(e.g., prestodb/presto-native-dependency:latest) required: false default: '' tag_image_as_latest: - description: 'Tag the published images as latest' + description: Tag the published images as latest type: boolean default: true required: false publish_docs: - description: 'Publish docs' + description: Publish docs type: boolean default: true required: false @@ -68,6 +68,8 @@ jobs: if: github.event.inputs.publish_release_tag == 'true' runs-on: ubuntu-latest environment: release + permissions: + contents: write steps: - name: Checkout repository uses: actions/checkout@v4 @@ -77,27 +79,38 @@ jobs: fetch-depth: 0 fetch-tags: true show-progress: false + persist-credentials: true - name: Configure Git + env: + GIT_CI_EMAIL: ${{ env.GIT_CI_EMAIL }} + GIT_CI_USER: ${{ env.GIT_CI_USER }} run: | - git config --global user.email "${{ env.GIT_CI_EMAIL }}" - git config --global user.name "${{ env.GIT_CI_USER }}" + git config --global user.email "$GIT_CI_EMAIL" + git config --global user.name "$GIT_CI_USER" git config pull.rebase false - name: Cherry-pick release notes if: ${{ github.event.inputs.release-notes-commit != '' }} + env: + RELEASE_NOTES_COMMIT: ${{ env.RELEASE_NOTES_COMMIT }} run: | - git cherry-pick ${{ env.RELEASE_NOTES_COMMIT }} + git cherry-pick $RELEASE_NOTES_COMMIT - name: Delete existing release tag + env: + RELEASE_TAG: ${{ env.RELEASE_TAG }} run: | - git push --delete origin ${{ env.RELEASE_TAG }} || true - git tag -d ${{ env.RELEASE_TAG }} || true + git push --delete origin $RELEASE_TAG || true + git tag -d $RELEASE_TAG || true - name: Create new release tag + env: + RELEASE_TAG: ${{ env.RELEASE_TAG }} + RELEASE_BRANCH: ${{ env.RELEASE_BRANCH }} run: | - git tag -a ${{ env.RELEASE_TAG }} -m "release ${{ env.RELEASE_TAG }}" - git push origin ${{ env.RELEASE_BRANCH }} --tags + git tag -a $RELEASE_TAG -m "release $RELEASE_TAG" + git push origin $RELEASE_BRANCH --tags publish-maven-artifacts: needs: publish-release-tag @@ -110,6 +123,8 @@ jobs: ) runs-on: ubuntu-latest environment: release + permissions: + contents: read timeout-minutes: 60 env: @@ -118,7 +133,7 @@ jobs: steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: tool-cache: false large-packages: false @@ -146,11 +161,15 @@ jobs: token: ${{ secrets.PRESTODB_CI_TOKEN }} fetch-depth: 0 fetch-tags: true + persist-credentials: false - name: Configure Git + env: + GIT_CI_EMAIL: ${{ env.GIT_CI_EMAIL }} + GIT_CI_USER: ${{ env.GIT_CI_USER }} run: | - git config --global user.email "${{ env.GIT_CI_EMAIL }}" - git config --global user.name "${{ env.GIT_CI_USER }}" + git config --global user.email "$GIT_CI_EMAIL" + git config --global user.name "$GIT_CI_USER" git config pull.rebase false - name: Create Maven Settings @@ -205,10 +224,12 @@ jobs: env: GPG_TTY: $(tty) GPG_PASSPHRASE: "${{ secrets.GPG_PASSPHRASE }}" + MAVEN_AUTO_PUBLISH: "${{ env.MAVEN_AUTO_PUBLISH }}" + RELEASE_TAG: "${{ env.RELEASE_TAG }}" run: | unset MAVEN_CONFIG ./mvnw -s ${{ github.workspace }}/settings.xml -V -B -U -e -T1C deploy \ - -Dgpg.passphrase="${{ secrets.GPG_PASSPHRASE }}" \ + -Dgpg.passphrase="$GPG_PASSPHRASE" \ -Dmaven.wagon.http.retryHandler.count=8 \ -DskipTests \ -DstagingProfileId="${{ secrets.MAVEN_STAGING_PROFILE_ID }}" \ @@ -216,8 +237,8 @@ jobs: -DkeepStagingRepositoryOnCloseRuleFailure=true \ -DautoReleaseAfterClose=true \ -DstagingProgressTimeoutMinutes=60 \ - -DdeploymentName=presto-${{ env.RELEASE_TAG }} \ - -Drelease.autoPublish=${{ env.MAVEN_AUTO_PUBLISH }} \ + -DdeploymentName=presto-$RELEASE_TAG \ + -Drelease.autoPublish=$MAVEN_AUTO_PUBLISH \ -DskipExecutableJar \ -Poss-release \ -Pdeploy-to-ossrh \ @@ -249,6 +270,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_TAG}} + persist-credentials: false - name: Download artifacts uses: actions/download-artifact@v4 @@ -257,7 +279,7 @@ jobs: path: ./ - name: Create github release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 with: tag_name: ${{ env.RELEASE_TAG }} name: Presto ${{ env.RELEASE_TAG }} @@ -283,6 +305,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_TAG}} + persist-credentials: false - name: Download artifacts uses: actions/download-artifact@v4 @@ -291,16 +314,16 @@ jobs: path: ./ - name: Login to dockerhub - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up qemu - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - name: Set up docker buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Create and use builder run: | @@ -313,7 +336,7 @@ jobs: mv ./presto-cli/target/presto-cli-*-executable.jar docker/ - name: Build docker image and publish - uses: docker/build-push-action@v6 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: docker platforms: linux/amd64,linux/arm64,linux/ppc64le @@ -339,7 +362,7 @@ jobs: timeout-minutes: 300 steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: tool-cache: false large-packages: false @@ -351,6 +374,7 @@ jobs: with: ref: ${{ env.RELEASE_TAG }} submodules: true + persist-credentials: false - name: Initialize Prestissimo submodules run: | @@ -359,21 +383,28 @@ jobs: echo "COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set dependency image tag + env: + DEPENDENCY_IMAGE: ${{ github.event.inputs.dependency_image }} + RELEASE_TAG: ${{ env.RELEASE_TAG }} run: | - if [[ -n "${{ github.event.inputs.dependency_image }}" ]]; then - echo "DEPENDENCY_IMAGE=${{ github.event.inputs.dependency_image }}" >> $GITHUB_ENV + if [[ -n "$DEPENDENCY_IMAGE" ]]; then + echo "DEPENDENCY_IMAGE=$DEPENDENCY_IMAGE" >> $GITHUB_ENV else - echo "DEPENDENCY_IMAGE=${{ github.repository_owner }}/presto-native-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}" >> $GITHUB_ENV + echo "DEPENDENCY_IMAGE=${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }}" >> $GITHUB_ENV fi - name: Build Dependency Image working-directory: presto-native-execution + env: + PUBLISH_NATIVE_IMAGE: ${{ github.event.inputs.publish_native_image }} + TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} + RELEASE_TAG: ${{ env.RELEASE_TAG }} run: | df -h if docker pull ${{ env.DEPENDENCY_IMAGE }}; then @@ -382,11 +413,11 @@ jobs: else echo "Building new depedency image" docker compose build centos-native-dependency - if [[ "${{ github.event.inputs.publish_native_image }}" == "true" ]]; then - docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }} - docker push ${{ github.repository_owner }}/presto-native-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }} - - if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then + if [[ "$PUBLISH_NATIVE_IMAGE" == "true" ]]; then + docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }} + docker push ${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }} + + if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:latest docker push ${{ github.repository_owner }}/presto-native-dependency:latest fi @@ -412,18 +443,26 @@ jobs: - name: Add release tag working-directory: presto-native-execution + env: + TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} + ORG_NAME: ${{ env.ORG_NAME }} + RELEASE_TAG: ${{ env.RELEASE_TAG }} run: | - docker tag presto/prestissimo-runtime:centos9 ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.RELEASE_TAG }} - if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then - docker tag presto/prestissimo-runtime:centos9 ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest + docker tag presto/prestissimo-runtime:centos9 $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG + if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then + docker tag presto/prestissimo-runtime:centos9 $ORG_NAME/${{ env.IMAGE_NAME }}:latest fi - name: Push to DockerHub + env: + TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} + ORG_NAME: ${{ env.ORG_NAME }} + RELEASE_TAG: ${{ env.RELEASE_TAG }} run: | - docker push ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.RELEASE_TAG }} - if [[ "${{ github.event.inputs.tag_image_as_latest }}" == "true" ]]; then - docker tag ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.RELEASE_TAG }} ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest - docker push ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest + docker push $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG + if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then + docker tag $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG $ORG_NAME/${{ env.IMAGE_NAME }}:latest + docker push $ORG_NAME/${{ env.IMAGE_NAME }}:latest fi publish-docs: @@ -431,6 +470,9 @@ jobs: if: (!failure() && !cancelled()) && github.event.inputs.publish_docs == 'true' runs-on: ubuntu-latest environment: release + permissions: + contents: write + pull-requests: write steps: - name: Checkout presto @@ -442,6 +484,7 @@ jobs: fetch-depth: 0 fetch-tags: true show-progress: false + persist-credentials: true - name: Checkout prestodb.github.io uses: actions/checkout@v4 @@ -450,18 +493,22 @@ jobs: path: prestodb.github.io ref: source token: ${{ secrets.PRESTODB_CI_TOKEN }} + persist-credentials: true - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18.x' + node-version: 18.x - name: Configure Git + env: + GIT_CI_EMAIL: ${{ env.GIT_CI_EMAIL }} + GIT_CI_USER: ${{ env.GIT_CI_USER }} run: | cd ${{ github.workspace }}/prestodb.github.io git config --global --add safe.directory ${{ github.workspace }}/prestodb.github.io - git config --global user.email "${{ env.GIT_CI_EMAIL }}" - git config --global user.name "${{ env.GIT_CI_USER }}" + git config --global user.email "$GIT_CI_EMAIL" + git config --global user.name "$GIT_CI_USER" git config --global alias.ls 'log --pretty=format:"%cd %h %ce: %s" --date=short --no-merges' git ls -5 diff --git a/.github/workflows/prestocpp-format-and-header-check.yml b/.github/workflows/prestocpp-format-and-header-check.yml index ad46395afc555..a7685c75c7ccc 100644 --- a/.github/workflows/prestocpp-format-and-header-check.yml +++ b/.github/workflows/prestocpp-format-and-header-check.yml @@ -1,11 +1,11 @@ name: prestocpp-format-and-header-check on: - workflow_dispatch: + workflow_dispatch: {} pull_request: paths: - - 'presto-native-execution/**' - - '.github/workflows/prestocpp-format-and-header-check.yml' + - presto-native-execution/** + - .github/workflows/prestocpp-format-and-header-check.yml jobs: prestocpp-format-and-header-check: @@ -13,24 +13,11 @@ jobs: concurrency: group: ${{ github.workflow }}-prestocpp-format-header-check-${{ github.event.pull_request.number }} cancel-in-progress: true - container: - image: public.ecr.aws/oss-presto/velox-dev:check + permissions: + contents: read steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Check formatting - run: | - git fetch origin master - cd presto-native-execution - make format-check - - - name: Check license headers - run: | - git fetch origin master - cd presto-native-execution - make header-check + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + persist-credentials: false + - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/prestocpp-linux-adapters-build.yml b/.github/workflows/prestocpp-linux-adapters-build.yml index b8441122a9662..a19060dd400f3 100644 --- a/.github/workflows/prestocpp-linux-adapters-build.yml +++ b/.github/workflows/prestocpp-linux-adapters-build.yml @@ -1,12 +1,11 @@ name: prestocpp-linux-adapters-build -on: - workflow_dispatch: - # Disable the automatic execution on PR because it currently will run out of disk space and - # we will address this subsequently. - # - use smaller image - in the works - # - remove the adapters downloaded files after install - JWT needs fixing because of the cmake files end up - #pull_request: +on: workflow_dispatch +# Disable the automatic execution on PR because it currently will run out of disk space and +# we will address this subsequently. +# - use smaller image - in the works +# - remove the adapters downloaded files after install - JWT needs fixing because of the cmake files end up +#pull_request: {} jobs: prestocpp-linux-adapters-build: @@ -18,8 +17,12 @@ jobs: cancel-in-progress: true env: CCACHE_DIR: "${{ github.workspace }}/ccache" + permissions: + contents: read steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Fix git permissions # Usually actions/checkout does this but as we run in a container diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index df01cbb878706..0f455b01be7e9 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -1,10 +1,10 @@ name: prestocpp-linux-build-and-unit-test on: - workflow_dispatch: + workflow_dispatch: {} schedule: - - cron: '30 0 * * *' # Runs at 00:30 UTC daily on master(default) branch - pull_request: + - cron: 30 0 * * * # Runs at 00:30 UTC daily on master(default) branch + pull_request: {} jobs: changes: @@ -16,15 +16,15 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - name: Run changes check for PRs - if: github.event_name != 'schedule' - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - name: Run changes check for PRs + if: github.event_name != 'schedule' + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' prestocpp-linux-build-for-test: runs-on: ubuntu-22.04 @@ -36,10 +36,14 @@ jobs: cancel-in-progress: true env: CCACHE_DIR: "${{ github.workspace }}/ccache" + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | @@ -133,13 +137,17 @@ jobs: container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + MAVEN_OPTS: -Xmx4G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | @@ -171,9 +179,9 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -184,7 +192,7 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) @@ -224,18 +232,22 @@ jobs: strategy: fail-fast: false matrix: - storage-format: [ "PARQUET", "DWRF" ] - enable-sidecar: [ "true", "false" ] + storage-format: [PARQUET, DWRF] + enable-sidecar: [true, false] container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + MAVEN_OPTS: -Xmx4G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | @@ -274,9 +286,9 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -287,7 +299,7 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) @@ -325,18 +337,22 @@ jobs: strategy: fail-fast: false matrix: - storage-format: [ "PARQUET", "DWRF" ] - enable-sidecar: [ "true", "false" ] + storage-format: [PARQUET, DWRF] + enable-sidecar: ["true", "false"] container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + MAVEN_OPTS: -Xmx4G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | @@ -368,9 +384,9 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -381,7 +397,7 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) @@ -419,13 +435,17 @@ jobs: container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + MAVEN_OPTS: -Xmx4G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -456,9 +476,9 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -469,7 +489,7 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-sidecar-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) @@ -504,13 +524,17 @@ jobs: container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + MAVEN_OPTS: -Xmx4G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end + permissions: + contents: read steps: - uses: actions/checkout@v4 if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -541,9 +565,9 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 17.0.15 - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: | github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' @@ -554,7 +578,7 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError run: | for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-plan-checker-router-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) diff --git a/.github/workflows/prestocpp-linux-build.yml b/.github/workflows/prestocpp-linux-build.yml index 5dce05e7cc64c..8057702dde267 100644 --- a/.github/workflows/prestocpp-linux-build.yml +++ b/.github/workflows/prestocpp-linux-build.yml @@ -1,8 +1,6 @@ name: prestocpp-linux-build -on: - workflow_dispatch: - pull_request: +on: [workflow_dispatch, pull_request] jobs: changes: @@ -14,16 +12,18 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' prestocpp-linux-build-engine: runs-on: ubuntu-22.04 + permissions: + contents: read needs: changes container: image: prestodb/presto-native-dependency:0.296-20251021214746-7eb2686 @@ -80,6 +80,8 @@ jobs: - uses: actions/checkout@v4 if: needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: needs.changes.outputs.codechange == 'true' @@ -128,7 +130,6 @@ jobs: if: needs.changes.outputs.codechange == 'true' run: | - - name: Build engine if: needs.changes.outputs.codechange == 'true' run: | diff --git a/.github/workflows/prestocpp-macos-build.yml b/.github/workflows/prestocpp-macos-build.yml index cc8989d2f5d29..7dfff75298dbf 100644 --- a/.github/workflows/prestocpp-macos-build.yml +++ b/.github/workflows/prestocpp-macos-build.yml @@ -1,8 +1,8 @@ name: prestocpp-macos-build on: - workflow_dispatch: - pull_request: + workflow_dispatch: {} + pull_request: {} jobs: changes: @@ -30,13 +30,15 @@ jobs: os: [macos-13, macos-15] runs-on: ${{ matrix.os }} needs: changes + permissions: + contents: read env: CCACHE_DIR: "${{ github.workspace }}/ccache" CMAKE_POLICY_VERSION_MINIMUM: "3.5" HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE" # The arm runners (macos-14 and later) have only 7GB RAM. BUILD_TYPE: "${{ matrix.os == 'macos-15' && 'Release' || 'Debug' }}" - INSTALL_PREFIX: "/tmp/deps-install" + INSTALL_PREFIX: /tmp/deps-install concurrency: group: ${{ github.workflow }}-prestocpp-macos-build-${{ github.event.pull_request.number }}-${{ matrix.os }} cancel-in-progress: true @@ -44,6 +46,8 @@ jobs: - uses: actions/checkout@v4 if: | needs.changes.outputs.codechange == 'true' + with: + persist-credentials: false - name: Fix git permissions if: | @@ -59,7 +63,7 @@ jobs: cd presto-native-execution make submodules - - name: "Setup MacOS" + - name: Setup MacOS if: | needs.changes.outputs.codechange == 'true' run: | @@ -111,7 +115,7 @@ jobs: needs.changes.outputs.codechange == 'true' run: PATH=${INSTALL_PREFIX}/bin:${PATH} ccache -sz - - name: "Build presto_cpp on MacOS" + - name: Build presto_cpp on MacOS if: | needs.changes.outputs.codechange == 'true' run: | @@ -122,7 +126,7 @@ jobs: if [[ "${{ matrix.os }}" == "macos-13" ]]; then # Velox sets -Wno-sign-compare but it needs to apply to Arrow. # There is also a conflict with Arrow and Velox (Glog) macros which are fixed - # in a newer version of Arrow. The issues cannot be easily fixed in Velox so + # in a newer version of Arrow. The issues cannot be easily fixed in Velox so # override the warnings that would throw errors. export CXXFLAGS="-Wno-error=sign-compare -Wno-error=macro-redefined" fi diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index 92ba0a4384ee8..216fbadc409e4 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -1,14 +1,13 @@ name: product tests (basic) -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true RETRY: .github/bin/retry jobs: @@ -21,19 +20,21 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' product-tests-basic-environment: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 60 concurrency: @@ -51,12 +52,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index c40a506f739aa..fea69491625d8 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -1,14 +1,13 @@ name: ci -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true RETRY: .github/bin/retry jobs: @@ -21,19 +20,21 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' product-tests-specific-environment1: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 60 concurrency: @@ -49,12 +50,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs @@ -96,8 +98,10 @@ jobs: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 60 concurrency: @@ -113,12 +117,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/release-notes-check.yml b/.github/workflows/release-notes-check.yml index d846f75dbdb26..57d547757e286 100644 --- a/.github/workflows/release-notes-check.yml +++ b/.github/workflows/release-notes-check.yml @@ -2,7 +2,7 @@ name: release-notes on: pull_request: - types: [ opened, edited, reopened ] + types: [opened, edited, reopened] env: RELEASE_TOOLS_VERSION: "0.11" @@ -10,6 +10,8 @@ env: jobs: check_release_note: runs-on: ubuntu-latest + permissions: + contents: read concurrency: group: ${{ github.workflow }}-check-release-note-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -18,11 +20,12 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + persist-credentials: false - uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin java-version: 8 - cache: 'maven' + cache: maven - name: Get presto-release-tools run: | ./mvnw \ diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index e65e6fba0b62f..f18d1a6e554b3 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -1,15 +1,14 @@ name: singlestore tests -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -23,7 +22,7 @@ jobs: codechange: ${{ steps.filter.outputs.codechange }} steps: # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: filters: | @@ -33,8 +32,10 @@ jobs: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 30 concurrency: @@ -45,6 +46,7 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - name: Remove unnecessary pre-installed toolchains for free disk spaces if: needs.changes.outputs.codechange == 'true' run: | @@ -60,9 +62,9 @@ jobs: - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index b87e330d17164..9f834a90c451a 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -1,15 +1,14 @@ name: spark integration -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -22,19 +21,21 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' spark-integration: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 60 concurrency: @@ -45,12 +46,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index bb0c5a31abaa8..01746c8cb1b5e 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -1,15 +1,14 @@ name: test other modules -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -22,19 +21,21 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' test-other-modules: strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] runs-on: ubuntu-latest + permissions: + contents: read needs: changes timeout-minutes: 60 concurrency: @@ -45,12 +46,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs @@ -65,30 +67,4 @@ jobs: ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:presto-docs,!:presto-server,!presto-test-coverage' - name: Maven Tests if: needs.changes.outputs.codechange == 'true' - run: | - ./mvnw test -T 1 ${MAVEN_TEST} -P skip-native-sidecar-tests -pl ' - !presto-tests, - !presto-native-tests, - !presto-accumulo, - !presto-cassandra, - !presto-hive, - !presto-kudu, - !presto-docs, - !presto-server, - !presto-main, - !presto-main-base, - !presto-mongodb, - !presto-spark-package, - !presto-spark-launcher, - !presto-spark-testing, - !presto-spark-base, - !presto-redis, - !presto-elasticsearch, - !presto-orc, - !presto-thrift-connector, - !presto-native-execution, - !presto-test-coverage, - !presto-iceberg, - !presto-singlestore, - !presto-base-arrow-flight, - !presto-plan-checker-router-plugin' + run: ./mvnw test -T 1 ${MAVEN_TEST} -P skip-native-sidecar-tests -pl '!presto-tests, !presto-native-tests, !presto-accumulo, !presto-cassandra, !presto-hive, !presto-kudu, !presto-docs, !presto-server, !presto-main, !presto-main-base, !presto-mongodb, !presto-spark-package, !presto-spark-launcher, !presto-spark-testing, !presto-spark-base, !presto-redis, !presto-elasticsearch, !presto-orc, !presto-thrift-connector, !presto-native-execution, !presto-test-coverage, !presto-iceberg, !presto-singlestore, !presto-base-arrow-flight, !presto-plan-checker-router-plugin' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f0c4f4e57cc47..028c9ba9378ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,15 +1,14 @@ name: test -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + MAVEN_OPTS: -Xmx1024M -XX:+ExitOnOutOfMemoryError + MAVEN_INSTALL_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError + MAVEN_FAST_INSTALL: -B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true + MAVEN_TEST: -B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end RETRY: .github/bin/retry jobs: @@ -22,53 +21,55 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' test: runs-on: ubuntu-latest + permissions: + contents: read needs: changes strategy: fail-fast: false matrix: - java: [ 17.0.15 ] + java: [17.0.15] modules: - - ":presto-tests -P presto-tests-execution-memory" - - ":presto-tests -P presto-tests-general" - - ":presto-tests -P ci-only-distributed-non-hash-gen" - - ":presto-tests -P ci-only-tpch-distributed-queries" - - ":presto-tests -P ci-only-local-queries" - - ":presto-tests -P ci-only-distributed-queries" - - ":presto-tests -P ci-only-aggregation-queries" - - ":presto-tests -P ci-only-plan-determinism" - - ":presto-tests -P ci-only-resource-manager" - - ":presto-accumulo" - - ":presto-cassandra -P ci-full-tests" - - ":presto-hive" - - ":presto-hive -P test-hive-materialized-queries" - - ":presto-hive -P test-hive-materialized-aggregations" - - ":presto-hive -P test-hive-recoverable-execution" - - ":presto-hive -P test-hive-pushdown-filter-queries-basic" - - ":presto-hive -P test-hive-pushdown-filter-queries-advanced" - - ":presto-hive -P test-hive-repartitioning" - - ":presto-hive -P test-hive-parquet" - - ":presto-main-base" - - ":presto-main" - - ":presto-mongodb -P ci-full-tests" - - ":presto-redis -P ci-full-tests" - - ":presto-elasticsearch" - - ":presto-orc" - - ":presto-thrift-connector" - - ":presto-spark-base -P presto-spark-tests-smoke" - - ":presto-spark-base -P presto-spark-tests-all-queries" - - ":presto-spark-base -P presto-spark-tests-spill-queries" - - ":presto-iceberg" - - ":presto-clickhouse -P ci" + - :presto-tests -P presto-tests-execution-memory + - :presto-tests -P presto-tests-general + - :presto-tests -P ci-only-distributed-non-hash-gen + - :presto-tests -P ci-only-tpch-distributed-queries + - :presto-tests -P ci-only-local-queries + - :presto-tests -P ci-only-distributed-queries + - :presto-tests -P ci-only-aggregation-queries + - :presto-tests -P ci-only-plan-determinism + - :presto-tests -P ci-only-resource-manager + - :presto-accumulo + - :presto-cassandra -P ci-full-tests + - :presto-hive + - :presto-hive -P test-hive-materialized-queries + - :presto-hive -P test-hive-materialized-aggregations + - :presto-hive -P test-hive-recoverable-execution + - :presto-hive -P test-hive-pushdown-filter-queries-basic + - :presto-hive -P test-hive-pushdown-filter-queries-advanced + - :presto-hive -P test-hive-repartitioning + - :presto-hive -P test-hive-parquet + - :presto-main-base + - :presto-main + - :presto-mongodb -P ci-full-tests + - :presto-redis -P ci-full-tests + - :presto-elasticsearch + - :presto-orc + - :presto-thrift-connector + - :presto-spark-base -P presto-spark-tests-smoke + - :presto-spark-base -P presto-spark-tests-all-queries + - :presto-spark-base -P presto-spark-tests-spill-queries + - :presto-iceberg + - :presto-clickhouse -P ci timeout-minutes: 80 concurrency: group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}-${{ matrix.java }} @@ -78,12 +79,13 @@ jobs: if: needs.changes.outputs.codechange == 'true' with: show-progress: false + persist-credentials: false - uses: actions/setup-java@v4 if: needs.changes.outputs.codechange == 'true' with: - distribution: 'temurin' + distribution: temurin java-version: ${{ matrix.java }} - cache: 'maven' + cache: maven - name: Download nodejs to maven cache if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs diff --git a/.github/workflows/web-ui-checks.yml b/.github/workflows/web-ui-checks.yml index f1ba039507f74..aae8abef8b58d 100644 --- a/.github/workflows/web-ui-checks.yml +++ b/.github/workflows/web-ui-checks.yml @@ -1,7 +1,6 @@ name: web ui checks -on: - pull_request: +on: pull_request env: # An envar that signals to tests we are executing in the CI environment @@ -18,16 +17,18 @@ jobs: outputs: codechange: ${{ steps.filter.outputs.codechange }} steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' - - 'presto-ui/**' + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' + - 'presto-ui/**' web-ui-checks: runs-on: ubuntu-latest + permissions: + contents: read needs: changes if: needs.changes.outputs.codechange == 'true' timeout-minutes: 30 @@ -38,5 +39,6 @@ jobs: - uses: actions/checkout@v4 with: show-progress: false + persist-credentials: false - name: Web UI Checks run: presto-ui/bin/check_webui.sh diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000000000..d2871abcba49f --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,4 @@ +rules: + dangerous-triggers: + ignore: + - codenotify.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..4e9d20a35fc02 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,116 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See https://pre-commit.com for more information +# General excludes, files can also be excluded on a hook level +files: ^presto-native-execution/.*|^.github/.*\.yml +exclude: "^(?:presto-native-execution/(?:velox|presto_cpp/external)/.*)|\ + ^presto-native-execution/presto_cpp/presto_protocol/.*\\.yml|\ + \\.(?:patch|header|sql)|\ + ThriftLibrary\\.cmake|\ + /data/|\ + build/(?:fb_code_builder|deps)|\ + cmake-build-debug|\ + NOTICE\\.txt|\ + scripts/git-clang-format" +default_install_hook_types: [pre-commit, pre-push] +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + + - repo: local + hooks: + - id: clang-tidy + name: clang-tidy + description: Run clang-tidy on C/C++ files + stages: + - manual + entry: clang-tidy + language: python + types_or: [c++, c] + additional_dependencies: [clang-tidy==18.1.8] + require_serial: true + + - id: license-header + name: license-header + description: Add missing license headers. + entry: presto-native-execution/scripts/license-header.py + args: [-i] + language: python + additional_dependencies: [regex] + require_serial: true + exclude: ^.github/.* + + # CMake + - repo: https://github.com/BlankSpruce/gersemi + rev: 0.21.0 + hooks: + - id: gersemi + name: CMake formatter + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v20.1.8 + hooks: + - id: clang-format + # types_or: [c++, c, cuda, metal, objective-c] + files: \.(cpp|cc|c|h|hpp|inc|cu|cuh|clcpp|mm|metal)$ + + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.11.0-1 + hooks: + - id: shfmt + # w: write changes, s: simplify, i set indent to 2 spaces + args: [-w, -s, -i, '2'] + + # Python + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.3 + hooks: + - id: ruff-check + args: [ --fix, --no-unsafe-fixes ] + - id: ruff-format + + # The following checks mostly target GitHub Actions workflows. + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.37.0 + hooks: + - id: yamllint + args: [ --format, parsable, --strict ] + exclude: .*\.clang-(tidy|format)|presto_cpp/main/thrift/.* + + - repo: https://github.com/google/yamlfmt + rev: v0.16.0 + hooks: + - id: yamlfmt + exclude: .*\.clang-(tidy|format)|presto_cpp/main/thrift/.* + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.7.0 + hooks: + - id: zizmor + + - repo: https://github.com/mpalmer/action-validator + rev: v0.7.1 + hooks: + - id: action-validator diff --git a/.yamlfmt.yml b/.yamlfmt.yml new file mode 100644 index 0000000000000..46a2713a1c677 --- /dev/null +++ b/.yamlfmt.yml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +match_type: doublestar +exclude: + - '**/.clang-format' + - '**/.clang-tidy' +formatter: + type: basic + retain_line_breaks_single: true + scan_folded_as_literal: true + indent: 2 diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000000000..866fb793d61ef --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 1 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 0 + max-spaces-inside-empty: 0 + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: disable + empty-values: + forbid-in-flow-mappings: true + forbid-in-block-sequences: true + float-values: + forbid-inf: true + forbid-nan: true + forbid-scientific-notation: true + require-numeral-before-decimal: true + indentation: disable + line-length: disable + octal-values: enable + quoted-strings: + required: only-when-needed + extra-allowed: ['.*\$\{\{.*\}\}.*'] + truthy: + allowed-values: ['true', 'false', 'on'] + level: warning diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85712802bf364..36987cca9c14b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,7 @@ Details: ## Code Style +### Java We recommend you use IntelliJ as your IDE. The code style template for the project can be found in the [codestyle](https://github.com/airlift/codestyle) repository along with our general programming and Java guidelines. In addition to those you should also adhere to the following: * **Naming** @@ -308,6 +309,13 @@ We recommend you use IntelliJ as your IDE. The code style template for the proje reinitialize them before each test in a `@BeforeMethod` method, and annotate the class with `@Test(singleThreaded = true)`. +### PrestoC++/Prestissimo (presto-native-execution) + +The project follows the [coding standards](https://github.com/facebookincubator/velox/blob/main/CONTRIBUTING.md#coding-best-practices) of the [Velox](https://github.com/facebookincubator/velox) project. + +For code formatting a pre-commit hook is used that is installed locally and can fix issues before changes are pushed to the repository. +Please install the [pre-commit](https://pre-commit.com/) tool. Once installed, run `pre-commit run` to scan and fix your staged changes manually, or optionally, install the hook in the local repository by running `pre-commit install` in the project root. +This results in the hook being automatically run on `git commit` executions. ## Commit Standards diff --git a/presto-native-execution/.clang-tidy b/presto-native-execution/.clang-tidy new file mode 100644 index 0000000000000..5b36ac93d48d2 --- /dev/null +++ b/presto-native-execution/.clang-tidy @@ -0,0 +1,96 @@ +Checks: > + *, + -abseil-*, + -android-*, + -cert-err58-cpp, + -cert-err58-cpp, + -clang-analyzer-osx-*, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-goto, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-owning-memory, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-special-member-functions, + -fuchsia-*, + -google-*, + -hicpp-avoid-c-arrays, + -hicpp-avoid-goto, + -hicpp-deprecated-headers, + -hicpp-no-array-decay, + -hicpp-special-member-functions, + -hicpp-use-equals-default, + -hicpp-vararg, + -hicpp-vararg, + -llvm-header-guard, + -llvm-include-order, + -llvmlibc-*, + -misc-no-recursion, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -misc-unused-parameters, + -modernize-avoid-c-arrays, + -modernize-deprecated-headers, + -modernize-use-nodiscard, + -modernize-use-trailing-return-type, + -mpi-*, + -objc-*, + -openmp-*, + -readability-avoid-const-params-in-decls, + -readability-convert-member-functions-to-static, + -readability-implicit-bool-conversion, + -readability-magic-numbers, + -zircon-*, + +HeaderFilterRegex: '.*' + +WarningsAsErrors: '' + +CheckOptions: + # Naming conventions as explicitly stated in CODING_STYLE.md + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.TypeAliasCase + value: CamelCase + - key: readability-identifier-naming.TypeTemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: camelBack + - key: readability-identifier-naming.VariableCase + value: camelBack + - key: readability-identifier-naming.ParameterCase + value: camelBack + - key: readability-identifier-naming.PrivateMemberCase + value: camelBack + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.ProtectedMemberCase + value: camelBack + - key: readability-identifier-naming.ProtectedMemberSuffix + value: _ + - key: readability-identifier-naming.MacroDefinitionCase + value: UPPER_CASE + - key: readability-identifier-naming.NamespaceCase + value: lower_case + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantPrefix + value: k + + # Use nullptr instead of NULL or 0 + - key: modernize-use-nullptr.NullMacros + value: 'NULL' + + # Prefer enum class over enum + - key: modernize-use-using.IgnoreUsingStdAllocator + value: 1 diff --git a/presto-native-execution/.gersemirc b/presto-native-execution/.gersemirc new file mode 100644 index 0000000000000..380ff2031d1c3 --- /dev/null +++ b/presto-native-execution/.gersemirc @@ -0,0 +1,6 @@ +# vim: set filetype=yaml : + +line_length: 100 +indent: 2 +definitions: +- presto_cpp/main/thrift/ThriftLibrary.cmake diff --git a/presto-native-execution/CMakeLists.txt b/presto-native-execution/CMakeLists.txt index 1e7fce6c7717b..d7becdc6188a0 100644 --- a/presto-native-execution/CMakeLists.txt +++ b/presto-native-execution/CMakeLists.txt @@ -22,7 +22,8 @@ execute_process( bash -c "( source ${CMAKE_CURRENT_SOURCE_DIR}/velox/scripts/setup-helper-functions.sh && echo -n $(get_cxx_flags $ENV{CPU_TARGET}))" OUTPUT_VARIABLE SCRIPT_CXX_FLAGS - RESULT_VARIABLE COMMAND_STATUS) + RESULT_VARIABLE COMMAND_STATUS +) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) @@ -30,8 +31,7 @@ message("Appending CMAKE_CXX_FLAGS with ${SCRIPT_CXX_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}") # Known warnings that are benign can be disabled. -set(DISABLED_WARNINGS - "-Wno-nullability-completeness -Wno-deprecated-declarations") +set(DISABLED_WARNINGS "-Wno-nullability-completeness -Wno-deprecated-declarations") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0.0") @@ -42,8 +42,7 @@ endif() # Important warnings that must be explicitly enabled. set(ENABLE_WARNINGS "-Wreorder") -set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} ${DISABLED_WARNINGS} ${ENABLE_WARNINGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${DISABLED_WARNINGS} ${ENABLE_WARNINGS}") # Add all Presto options below. @@ -83,36 +82,27 @@ option(PRESTO_ENABLE_SPATIAL "Enable spatial support" ON) # and turn on int128. add_compile_definitions(FOLLY_HAVE_INT128_T=1 FOLLY_CFG_NO_COROUTINES) -set(VELOX_ENABLE_S3 - ${PRESTO_ENABLE_S3} - CACHE BOOL "Build S3 support") +set(VELOX_ENABLE_S3 ${PRESTO_ENABLE_S3} CACHE BOOL "Build S3 support") -set(VELOX_ENABLE_HDFS - ${PRESTO_ENABLE_HDFS} - CACHE BOOL "Build HDFS support") +set(VELOX_ENABLE_HDFS ${PRESTO_ENABLE_HDFS} CACHE BOOL "Build HDFS support") -set(VELOX_ENABLE_GCS - ${PRESTO_ENABLE_GCS} - CACHE BOOL "Build GCS support") +set(VELOX_ENABLE_GCS ${PRESTO_ENABLE_GCS} CACHE BOOL "Build GCS support") -set(VELOX_ENABLE_ABFS - ${PRESTO_ENABLE_ABFS} - CACHE BOOL "Build ABFS support") +set(VELOX_ENABLE_ABFS ${PRESTO_ENABLE_ABFS} CACHE BOOL "Build ABFS support") -set(VELOX_ENABLE_PARQUET - ${PRESTO_ENABLE_PARQUET} - CACHE BOOL "Enable Parquet support") +set(VELOX_ENABLE_PARQUET ${PRESTO_ENABLE_PARQUET} CACHE BOOL "Enable Parquet support") -set(VELOX_ENABLE_REMOTE_FUNCTIONS - ${PRESTO_ENABLE_REMOTE_FUNCTIONS} - CACHE BOOL "Enable remote function support in Velox") +set( + VELOX_ENABLE_REMOTE_FUNCTIONS + ${PRESTO_ENABLE_REMOTE_FUNCTIONS} + CACHE BOOL + "Enable remote function support in Velox" +) if(PRESTO_ENABLE_REMOTE_FUNCTIONS) add_compile_definitions(PRESTO_ENABLE_REMOTE_FUNCTIONS) endif() -set(VELOX_ENABLE_CUDF - ${PRESTO_ENABLE_CUDF} - CACHE BOOL "Enable cuDF support") +set(VELOX_ENABLE_CUDF ${PRESTO_ENABLE_CUDF} CACHE BOOL "Enable cuDF support") if(PRESTO_ENABLE_CUDF) add_compile_definitions(PRESTO_ENABLE_CUDF) enable_language(CUDA) @@ -120,25 +110,20 @@ if(PRESTO_ENABLE_CUDF) cmake_policy(SET CMP0104 NEW) endif() -set(VELOX_ENABLE_GEO - ${PRESTO_ENABLE_SPATIAL} - CACHE BOOL "Enable Velox Geometry (aka spatial) support") +set( + VELOX_ENABLE_GEO + ${PRESTO_ENABLE_SPATIAL} + CACHE BOOL + "Enable Velox Geometry (aka spatial) support" +) -set(VELOX_BUILD_TESTING - OFF - CACHE BOOL "Enable Velox tests") +set(VELOX_BUILD_TESTING OFF CACHE BOOL "Enable Velox tests") -set(VELOX_ENABLE_SPARK_FUNCTIONS - OFF - CACHE BOOL "Enable Velox Spark functions") +set(VELOX_ENABLE_SPARK_FUNCTIONS OFF CACHE BOOL "Enable Velox Spark functions") -set(VELOX_ENABLE_EXAMPLES - OFF - CACHE BOOL "Enable Velox examples") +set(VELOX_ENABLE_EXAMPLES OFF CACHE BOOL "Enable Velox examples") -set(VELOX_BUILD_TEST_UTILS - ${PRESTO_ENABLE_TESTING} - CACHE BOOL "Enable Velox test utils") +set(VELOX_BUILD_TEST_UTILS ${PRESTO_ENABLE_TESTING} CACHE BOOL "Enable Velox test utils") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -151,15 +136,8 @@ set(Boost_USE_MULTITHREADED TRUE) find_package( Boost 1.66.0 - REQUIRED - program_options - context - filesystem - regex - thread - system - date_time - atomic) + REQUIRED program_options context filesystem regex thread system date_time atomic +) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) find_package(gflags COMPONENTS shared) @@ -179,16 +157,18 @@ find_package(ZLIB) find_library(SNAPPY snappy) find_package(folly CONFIG REQUIRED) -set(FOLLY_WITH_DEPENDENCIES - ${FOLLY_LIBRARIES} - ${DOUBLE_CONVERSION} - Boost::context - dl - ${EVENT} - ${SNAPPY} - ${LZ4} - ${ZSTD} - ${ZLIB_LIBRARIES}) +set( + FOLLY_WITH_DEPENDENCIES + ${FOLLY_LIBRARIES} + ${DOUBLE_CONVERSION} + Boost::context + dl + ${EVENT} + ${SNAPPY} + ${LZ4} + ${ZSTD} + ${ZLIB_LIBRARIES} +) find_package(BZip2 MODULE) if(BZIP2_FOUND) @@ -212,8 +192,14 @@ find_package(wangle CONFIG) find_package(FBThrift) include_directories(SYSTEM ${FBTHRIFT_INCLUDE_DIR}) -set(PROXYGEN_LIBRARIES ${PROXYGEN_HTTP_SERVER} ${PROXYGEN} ${WANGLE} ${FIZZ} - ${MVFST_EXCEPTION}) +set( + PROXYGEN_LIBRARIES + ${PROXYGEN_HTTP_SERVER} + ${PROXYGEN} + ${WANGLE} + ${FIZZ} + ${MVFST_EXCEPTION} +) find_path(PROXYGEN_DIR NAMES include/proxygen) set(PROXYGEN_INCLUDE_DIR "${PROXYGEN_DIR}/include/") @@ -228,9 +214,7 @@ include_directories(${CMAKE_BINARY_DIR}) set(VELOX_GTEST_INCUDE_DIR "velox/third_party/googletest/googletest/include") # Do not use the Mono library because it causes link errors. -set(VELOX_MONO_LIBRARY - OFF - CACHE BOOL "Build Velox mono library") +set(VELOX_MONO_LIBRARY OFF CACHE BOOL "Build Velox mono library") add_subdirectory(velox) @@ -248,8 +232,7 @@ if(PRESTO_ENABLE_JWT) endif() if("${MAX_LINK_JOBS}") - set_property(GLOBAL APPEND PROPERTY JOB_POOLS - "presto_link_job_pool=${MAX_LINK_JOBS}") + set_property(GLOBAL APPEND PROPERTY JOB_POOLS "presto_link_job_pool=${MAX_LINK_JOBS}") else() set_property(GLOBAL APPEND PROPERTY JOB_POOLS "presto_link_job_pool=8") endif() diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile index 27a811c11d462..8b82be02f75ed 100644 --- a/presto-native-execution/Makefile +++ b/presto-native-execution/Makefile @@ -127,35 +127,6 @@ presto_protocol: #: Build the presto_protocol serde library TypeSignature: #: Build the Presto TypeSignature parser cd presto_cpp/main/types; $(MAKE) TypeSignature -format-fix: #: Fix formatting issues in the presto-native-execution directory -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format master --fix -else - scripts/check.py format master --fix -endif - -format-check: #: Check for formatting issues in the presto-native-execution directory - clang-format --version -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py format master -else - scripts/check.py format master -endif - -header-fix: #: Fix license header issues in the presto-native-execution directory -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header master --fix -else - scripts/check.py header master --fix -endif - -header-check: #: Check for license header issues in the presto-native-execution directory -ifneq ("$(wildcard ${PYTHON_VENV}/pyvenv.cfg)","") - source ${PYTHON_VENV}/bin/activate; scripts/check.py header master -else - scripts/check.py header master -endif - help: #: Show the help messages @cat $(firstword $(MAKEFILE_LIST)) | \ awk '/^[-a-z]+:/' | \ diff --git a/presto-native-execution/README.md b/presto-native-execution/README.md index a27c657a2c11e..5956a5d5efd15 100644 --- a/presto-native-execution/README.md +++ b/presto-native-execution/README.md @@ -279,43 +279,36 @@ Run CLion: ### Code formatting, headers, and clang-tidy -Makefile targets exist for showing, fixing and checking formatting, license -headers and clang-tidy warnings. These targets are shortcuts for calling -`presto/presto-native-execution/scripts/check.py` . - -GitHub Actions run `make format-check`, `make header-check` and -`make tidy-check` as part of our continuous integration. Pull requests should -pass linux-build, format-check, header-check and other jobs without errors -before being accepted. - -Formatting issues found on the changed lines in the current commit can be -displayed using `make format-show`. These issues can be fixed by using `make -format-fix`. This will apply formatting changes to changed lines in the -current commit. - -Header issues found on the changed files in the current commit can be displayed -using `make header-show`. These issues can be fixed by using `make -header-fix`. This will apply license header updates to the files in the current -commit. - -Similar commands `make tidy-show`, `make-tidy-fix`, `make tidy-check` exist for -running clang-tidy, but these checks are currently advisory only. - -An entire directory tree of files can be formatted and have license headers added -using the `tree` variant of the format.sh commands: -``` -presto/presto-native-execution/scripts/check.py format tree -presto/presto-native-execution/scripts/check.py format tree --fix +Code formatting, license headers, and other checks are handled by pre-commit. + +The [pre-commit](https://pre-commit.com/) configuration in `.pre-commit-config.yaml` +provides Git hooks that run automatically before commits and pushes to check and fix +formatting and license headers. + +GitHub Actions run pre-commit checks as part of our continuous integration. Using +pre-commit hooks locally ensures pull requests pass these checks before they have +the chance to fail. When pre-commit automatically fixes issues on commit, it is a good +idea to manually check the modified files to ensure pre-commit did not make unintended +changes. -presto/presto-native-execution/scripts/check.py header tree -presto/presto-native-execution/scripts/check.py header tree --fix +To install the pre-commit hooks, first ensure your Python version is 3.9 or higher. +Then run: + +``` +pip install pre-commit +pre-commit install --allow-missing-config ``` -All the available formatting commands can be displayed by using -`presto/presto-native-execution/scripts/check.py help`. +The option `--allow-missing-config` will allow commits and pushes to succeed locally +if the config is missing (e.g. you are working on an older branch). + +In addition to the Git hooks, pre-commit can be run manually on changed files using +`pre-commit run` or on all files using `pre-commit run -a`. To run a +specific hook, use `pre-commit run [hook-id]` and refer to a specific hook `id` in +`.pre-commit-config.yaml`. -There is currently no mechanism to *opt out* files or directories from the -checks. When we need one it can be added. +The `clang-tidy` hook is not run locally or in CI by default, but +can be run manually for optional checks using `pre-commit run --hook-stage manual clang-tidy`. ## Create Pull Request * Submit PRs as usual following [Presto repository guidelines](https://github.com/prestodb/presto/wiki/Review-and-Commit-guidelines). diff --git a/presto-native-execution/docker-compose.yml b/presto-native-execution/docker-compose.yml index b051db521461f..f1f39ddc8e16d 100644 --- a/presto-native-execution/docker-compose.yml +++ b/presto-native-execution/docker-compose.yml @@ -26,15 +26,13 @@ services: image: presto/prestissimo-runtime:ubuntu-22.04 build: args: - # A few files in Velox require significant memory to compile and link. - # Build requires 18GB of memory for 2 threads. + # A few files in Velox require significant memory to compile and link. + # Build requires 18GB of memory for 2 threads. - NUM_THREADS=2 # default value for NUM_THREADS. - DEPENDENCY_IMAGE=presto/prestissimo-dependency:ubuntu-22.04 - BASE_IMAGE=ubuntu:22.04 - OSNAME=ubuntu - - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF - -DPRESTO_ENABLE_PARQUET=ON - -DPRESTO_ENABLE_S3=ON + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON context: . dockerfile: scripts/dockerfiles/prestissimo-runtime.dockerfile @@ -54,12 +52,9 @@ services: image: presto/prestissimo-runtime:centos9 build: args: - # A few files in Velox require significant memory to compile and link. - # Build requires 18GB of memory for 2 threads. + # A few files in Velox require significant memory to compile and link. + # Build requires 18GB of memory for 2 threads. - NUM_THREADS=2 # default value for NUM_THREADS - - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF - -DPRESTO_ENABLE_PARQUET=ON - -DPRESTO_ENABLE_S3=ON - -DPRESTO_ENABLE_CUDF=${GPU:-OFF} + - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON -DPRESTO_ENABLE_CUDF=${GPU:-OFF} context: . dockerfile: scripts/dockerfiles/prestissimo-runtime.dockerfile diff --git a/presto-native-execution/etc/velox.properties b/presto-native-execution/etc/velox.properties new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/presto-native-execution/presto_cpp/docs/develop/orc-dump-output.rst b/presto-native-execution/presto_cpp/docs/develop/orc-dump-output.rst index 8f09b7a3ecd11..a79ab6d7b171a 100644 --- a/presto-native-execution/presto_cpp/docs/develop/orc-dump-output.rst +++ b/presto-native-execution/presto_cpp/docs/develop/orc-dump-output.rst @@ -84,4 +84,4 @@ Sample output of an orcfiledump tool orc.writer.version -> 1 presto_query_id -> 20210814_094649_15363_c5483 orc.writer.name -> presto - presto_version -> 0.259.1 \ No newline at end of file + presto_version -> 0.259.1 diff --git a/presto-native-execution/presto_cpp/docs/index.rst b/presto-native-execution/presto_cpp/docs/index.rst index 588b88d5eb374..6b60868f0600d 100644 --- a/presto-native-execution/presto_cpp/docs/index.rst +++ b/presto-native-execution/presto_cpp/docs/index.rst @@ -6,4 +6,3 @@ Presto Native Execution Documentation :maxdepth: 2 develop - diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index b5e7879847238..2239894914bb0 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -39,10 +39,17 @@ add_library( TaskManager.cpp TaskResource.cpp PeriodicHeartbeatManager.cpp - PeriodicServiceInventoryManager.cpp) + PeriodicServiceInventoryManager.cpp +) -add_dependencies(presto_server_lib presto_operators presto_protocol - presto_types presto_thrift-cpp2 presto_thrift_extra) +add_dependencies( + presto_server_lib + presto_operators + presto_protocol + presto_types + presto_thrift-cpp2 + presto_thrift_extra +) target_include_directories(presto_server_lib PRIVATE ${presto_thrift_INCLUDES}) @@ -96,7 +103,8 @@ target_link_libraries( ${FOLLY_WITH_DEPENDENCIES} ${GLOG} ${GFLAGS_LIBRARIES} - pthread) + pthread +) if(PRESTO_ENABLE_CUDF) target_link_libraries(presto_server_lib velox_cudf_exec) @@ -106,28 +114,28 @@ endif() # likely due to a conflict between Arrow Thrift from velox_hive_connector and # FBThrift libraries. The build issue is fixed by linking velox_hive_connector # dependencies first followed by FBThrift. -target_link_libraries(presto_server_lib presto_thrift-cpp2 presto_thrift_extra - ${THRIFT_LIBRARY}) +target_link_libraries(presto_server_lib presto_thrift-cpp2 presto_thrift_extra ${THRIFT_LIBRARY}) if(PRESTO_ENABLE_REMOTE_FUNCTIONS) - add_library(presto_server_remote_function JsonSignatureParser.cpp - RemoteFunctionRegisterer.cpp) - - target_link_libraries(presto_server_remote_function velox_expression - velox_functions_remote ${FOLLY_WITH_DEPENDENCIES}) + add_library(presto_server_remote_function JsonSignatureParser.cpp RemoteFunctionRegisterer.cpp) + + target_link_libraries( + presto_server_remote_function + velox_expression + velox_functions_remote + ${FOLLY_WITH_DEPENDENCIES} + ) target_link_libraries(presto_server_lib presto_server_remote_function) endif() -set_property(TARGET presto_server_lib PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_server_lib PROPERTY JOB_POOL_LINK presto_link_job_pool) add_executable(presto_server PrestoMain.cpp) # The below additional flags are necessary for resolving dependencies for # loading dynamic libraries. if(APPLE) - target_link_options(presto_server BEFORE PUBLIC - "-Wl,-undefined,dynamic_lookup") + target_link_options(presto_server BEFORE PUBLIC "-Wl,-undefined,dynamic_lookup") else() target_link_options(presto_server BEFORE PUBLIC "-Wl,-export-dynamic") endif() @@ -137,12 +145,13 @@ endif() # However, we also would need to add its dependencies (tpch_gen etc). TODO # change the target in Velox to a library target then we can move this to the # presto_server_lib. -target_link_libraries(presto_server presto_server_lib velox_tpch_connector - velox_tpcds_connector) +target_link_libraries(presto_server presto_server_lib velox_tpch_connector velox_tpcds_connector) # Clang requires explicit linking with libatomic. -if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" - AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15) +if( + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" + AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15 +) target_link_libraries(presto_server atomic) endif() @@ -158,9 +167,7 @@ if(PRESTO_STATS_REPORTER_TYPE) add_subdirectory(runtime-metrics) target_link_libraries(presto_server presto_prometheus_reporter) else() - message( - FATAL_ERROR - "${PRESTO_STATS_REPORTER_TYPE} is not a valid stats reporter name") + message(FATAL_ERROR "${PRESTO_STATS_REPORTER_TYPE} is not a valid stats reporter name") endif() endif() @@ -168,14 +175,10 @@ if(PRESTO_MEMORY_CHECKER_TYPE) add_compile_definitions(PRESTO_MEMORY_CHECKER_TYPE) # Check if the current platform is Linux and the memory checker type is # LINUX_MEMORY_CHECKER. - if(UNIX - AND NOT APPLE - AND (PRESTO_MEMORY_CHECKER_TYPE STREQUAL "LINUX_MEMORY_CHECKER")) + if(UNIX AND NOT APPLE AND (PRESTO_MEMORY_CHECKER_TYPE STREQUAL "LINUX_MEMORY_CHECKER")) add_library(presto_linux_memory_checker OBJECT LinuxMemoryChecker.cpp) target_link_libraries(presto_server presto_linux_memory_checker) else() - message( - FATAL_ERROR - "${PRESTO_MEMORY_CHECKER_TYPE} is not a valid memory checker name") + message(FATAL_ERROR "${PRESTO_MEMORY_CHECKER_TYPE} is not a valid memory checker name") endif() endif() diff --git a/presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h b/presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h index 48b8ea5ab9236..2f7d61286ac0a 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h +++ b/presto-native-execution/presto_cpp/main/PeriodicMemoryChecker.h @@ -77,8 +77,9 @@ class PeriodicMemoryChecker { virtual void stop(); /// Returns the last known cached 'current' system memory usage in bytes. If - /// 'fetchFresh' is true, retrieves and returns the current system memory usage. - /// The returned value is used to compare with 'Config::systemMemLimitBytes'. + /// 'fetchFresh' is true, retrieves and returns the current system memory + /// usage. The returned value is used to compare with + /// 'Config::systemMemLimitBytes'. int64_t systemUsedMemoryBytes(bool fetchFresh = false); protected: diff --git a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp index 8537f173fe564..f1a18146c327d 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp @@ -148,27 +148,34 @@ class HiveConnectorStatsReporter { explicit HiveConnectorStatsReporter( std::shared_ptr connector) : connector_(std::move(connector)), - numElementsMetricName_(fmt::format( - kCounterHiveFileHandleCacheNumElementsFormat, - connector_->connectorId())), - pinnedSizeMetricName_(fmt::format( - kCounterHiveFileHandleCachePinnedSizeFormat, - connector_->connectorId())), - curSizeMetricName_(fmt::format( - kCounterHiveFileHandleCacheCurSizeFormat, - connector_->connectorId())), - numAccumulativeHitsMetricName_(fmt::format( - kCounterHiveFileHandleCacheNumAccumulativeHitsFormat, - connector_->connectorId())), - numAccumulativeLookupsMetricName_(fmt::format( - kCounterHiveFileHandleCacheNumAccumulativeLookupsFormat, - connector_->connectorId())), - numHitsMetricName_(fmt::format( - kCounterHiveFileHandleCacheNumHitsFormat, - connector_->connectorId())), - numLookupsMetricName_(fmt::format( - kCounterHiveFileHandleCacheNumLookupsFormat, - connector_->connectorId())) { + numElementsMetricName_( + fmt::format( + kCounterHiveFileHandleCacheNumElementsFormat, + connector_->connectorId())), + pinnedSizeMetricName_( + fmt::format( + kCounterHiveFileHandleCachePinnedSizeFormat, + connector_->connectorId())), + curSizeMetricName_( + fmt::format( + kCounterHiveFileHandleCacheCurSizeFormat, + connector_->connectorId())), + numAccumulativeHitsMetricName_( + fmt::format( + kCounterHiveFileHandleCacheNumAccumulativeHitsFormat, + connector_->connectorId())), + numAccumulativeLookupsMetricName_( + fmt::format( + kCounterHiveFileHandleCacheNumAccumulativeLookupsFormat, + connector_->connectorId())), + numHitsMetricName_( + fmt::format( + kCounterHiveFileHandleCacheNumHitsFormat, + connector_->connectorId())), + numLookupsMetricName_( + fmt::format( + kCounterHiveFileHandleCacheNumLookupsFormat, + connector_->connectorId())) { DEFINE_METRIC(numElementsMetricName_, velox::StatType::AVG); DEFINE_METRIC(pinnedSizeMetricName_, velox::StatType::AVG); DEFINE_METRIC(curSizeMetricName_, velox::StatType::AVG); @@ -427,7 +434,7 @@ void PeriodicTaskManager::addConnectorStatsTask() { } void PeriodicTaskManager::updateOperatingSystemStats() { - struct rusage usage {}; + struct rusage usage{}; memset(&usage, 0, sizeof(usage)); getrusage(RUSAGE_SELF, &usage); diff --git a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.h b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.h index 74ffb2fe96a6c..0f1874504021a 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicTaskManager.h +++ b/presto-native-execution/presto_cpp/main/PeriodicTaskManager.h @@ -13,8 +13,8 @@ */ #pragma once -#include #include +#include #include "velox/common/memory/Memory.h" namespace folly { diff --git a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp index 0f77121e9bbc6..77c526631e2df 100644 --- a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp @@ -159,10 +159,10 @@ folly::SemiFuture PrestoExchangeSource::request( } failedAttempts_ = 0; - dataRequestRetryState_ = - RetryState(std::chrono::duration_cast( - SystemConfig::instance()->exchangeMaxErrorDuration()) - .count()); + dataRequestRetryState_ = RetryState( + std::chrono::duration_cast( + SystemConfig::instance()->exchangeMaxErrorDuration()) + .count()); doRequest(dataRequestRetryState_.nextDelayMs(), maxBytes, maxWait); return future; @@ -256,14 +256,14 @@ void PrestoExchangeSource::handleDataResponse( } void PrestoExchangeSource::processDataResponse( - std::unique_ptr response, bool isGetDataSizeRequest) { + std::unique_ptr response, + bool isGetDataSizeRequest) { if (isGetDataSizeRequest) { RECORD_HISTOGRAM_METRIC_VALUE( kCounterExchangeGetDataSizeDuration, dataRequestRetryState_.durationMs()); RECORD_HISTOGRAM_METRIC_VALUE( - kCounterExchangeGetDataSizeNumTries, - dataRequestRetryState_.numTries()); + kCounterExchangeGetDataSizeNumTries, dataRequestRetryState_.numTries()); } else { RECORD_HISTOGRAM_METRIC_VALUE( kCounterExchangeRequestDuration, dataRequestRetryState_.durationMs()); @@ -490,10 +490,10 @@ void PrestoExchangeSource::abortResults() { return; } - abortRetryState_ = - RetryState(std::chrono::duration_cast( - SystemConfig::instance()->exchangeMaxErrorDuration()) - .count()); + abortRetryState_ = RetryState( + std::chrono::duration_cast( + SystemConfig::instance()->exchangeMaxErrorDuration()) + .count()); VLOG(1) << "Sending abort results " << basePath_; doAbortResults(abortRetryState_.nextDelayMs()); } diff --git a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h index 2399cc7eeb5e8..b98959097d1ab 100644 --- a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h +++ b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h @@ -200,7 +200,9 @@ class PrestoExchangeSource : public velox::exec::ExchangeSource { // response without an end marker. Sends delete-results if received an end // marker. The sequence of operations is: add data or end marker to the // queue; complete the future, send ack or delete-results. - void processDataResponse(std::unique_ptr response, bool isGetDataSizeRequest); + void processDataResponse( + std::unique_ptr response, + bool isGetDataSizeRequest); // If 'retry' is true, then retry the http request failure until reaches the // retry limit, otherwise just set exchange source error without retry. As diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 8e658c89a157e..391c5306f9987 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -120,9 +120,9 @@ class PrestoServer { /// Hook for derived PrestoServer implementations to add/stop additional /// periodic tasks. - virtual void addAdditionalPeriodicTasks(){}; + virtual void addAdditionalPeriodicTasks() {}; - virtual void stopAdditionalPeriodicTasks(){}; + virtual void stopAdditionalPeriodicTasks() {}; virtual void initializeCoordinatorDiscoverer(); @@ -133,7 +133,7 @@ class PrestoServer { virtual std::shared_ptr getExprSetListener(); virtual std::shared_ptr - getSplitListenerFactory(); + getSplitListenerFactory(); virtual std::vector registerVeloxConnectors( const fs::path& configDirectoryPath); @@ -256,13 +256,13 @@ class PrestoServer { // Executor for HTTP request processing after dispatching std::unique_ptr httpSrvCpuExecutor_; - // Executor for query engine driver executions. The underlying thread pool - // executor is a folly::CPUThreadPoolExecutor. The executor is stored as - // abstract type to provide flexibility of thread pool monitoring. The - // underlying folly::CPUThreadPoolExecutor can be obtained through + // Executor for query engine driver executions. The underlying thread pool + // executor is a folly::CPUThreadPoolExecutor. The executor is stored as + // abstract type to provide flexibility of thread pool monitoring. The + // underlying folly::CPUThreadPoolExecutor can be obtained through // 'driverCpuExecutor()' method. std::unique_ptr driverExecutor_; - // Raw pointer pointing to the underlying folly::CPUThreadPoolExecutor of + // Raw pointer pointing to the underlying folly::CPUThreadPoolExecutor of // 'driverExecutor_'. folly::CPUThreadPoolExecutor* driverCpuExecutor_; diff --git a/presto-native-execution/presto_cpp/main/PrestoTask.cpp b/presto-native-execution/presto_cpp/main/PrestoTask.cpp index 3621a763bdbdb..447625e533cd6 100644 --- a/presto-native-execution/presto_cpp/main/PrestoTask.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoTask.cpp @@ -351,7 +351,8 @@ void updatePipelineStats( protocol::PipelineStats& prestoPipelineStats) { prestoPipelineStats.inputPipeline = veloxPipelineStats.inputPipeline; prestoPipelineStats.outputPipeline = veloxPipelineStats.outputPipeline; - prestoPipelineStats.firstStartTimeInMillis = prestoTaskStats.createTimeInMillis; + prestoPipelineStats.firstStartTimeInMillis = + prestoTaskStats.createTimeInMillis; prestoPipelineStats.lastStartTimeInMillis = prestoTaskStats.endTimeInMillis; prestoPipelineStats.lastEndTimeInMillis = prestoTaskStats.endTimeInMillis; @@ -442,12 +443,18 @@ void updatePipelineStats( prestoOp.blockedWall = protocol::Duration( veloxOp.blockedWallNanos, protocol::TimeUnit::NANOSECONDS); - prestoOp.userMemoryReservationInBytes = veloxOp.memoryStats.userMemoryReservation; - prestoOp.revocableMemoryReservationInBytes = veloxOp.memoryStats.revocableMemoryReservation; - prestoOp.systemMemoryReservationInBytes = veloxOp.memoryStats.systemMemoryReservation; - prestoOp.peakUserMemoryReservationInBytes = veloxOp.memoryStats.peakUserMemoryReservation; - prestoOp.peakSystemMemoryReservationInBytes = veloxOp.memoryStats.peakSystemMemoryReservation; - prestoOp.peakTotalMemoryReservationInBytes = veloxOp.memoryStats.peakTotalMemoryReservation; + prestoOp.userMemoryReservationInBytes = + veloxOp.memoryStats.userMemoryReservation; + prestoOp.revocableMemoryReservationInBytes = + veloxOp.memoryStats.revocableMemoryReservation; + prestoOp.systemMemoryReservationInBytes = + veloxOp.memoryStats.systemMemoryReservation; + prestoOp.peakUserMemoryReservationInBytes = + veloxOp.memoryStats.peakUserMemoryReservation; + prestoOp.peakSystemMemoryReservationInBytes = + veloxOp.memoryStats.peakSystemMemoryReservation; + prestoOp.peakTotalMemoryReservationInBytes = + veloxOp.memoryStats.peakTotalMemoryReservation; prestoOp.spilledDataSizeInBytes = veloxOp.spilledBytes; @@ -769,8 +776,12 @@ void PrestoTask::updateTimeInfoLocked( taskRuntimeStats["endTime"].addValue(veloxTaskStats.endTimeMs); } taskRuntimeStats.insert({"nativeProcessCpuTime", fromNanos(processCpuTime_)}); - // Represents the time between receiving first taskUpdate and task creation time - taskRuntimeStats.insert({"taskCreationTime", fromNanos((createFinishTimeMs - firstTimeReceiveTaskUpdateMs) * 1'000'000)}); + // Represents the time between receiving first taskUpdate and task creation + // time. + taskRuntimeStats.insert( + {"taskCreationTime", + fromNanos( + (createFinishTimeMs - firstTimeReceiveTaskUpdateMs) * 1'000'000)}); } void PrestoTask::updateMemoryInfoLocked( @@ -828,14 +839,14 @@ void PrestoTask::updateExecutionInfoLocked( // the coordinator and worker may not be upgraded at the same time. // // To ensure safe rollout: - // - We are introducing new fields (e.g., `totalNewDrivers`) instead of modifying or - // removing existing ones. + // - We are introducing new fields (e.g., `totalNewDrivers`) instead of + // modifying or removing existing ones. // - The worker is updated first to populate both old and new fields. - // - The coordinator continues to use the old fields until it is updated to handle - // the new ones. + // - The coordinator continues to use the old fields until it is updated to + // handle the new ones. // - // Once both coordinator and worker support the new fields, we can safely remove - // the legacy fields in a follow-up cleanup PR. + // Once both coordinator and worker support the new fields, we can safely + // remove the legacy fields in a follow-up cleanup PR. prestoTaskStats.totalDrivers = veloxTaskStats.numTotalSplits; prestoTaskStats.queuedDrivers = veloxTaskStats.numQueuedSplits; diff --git a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp index 67650ae6983c9..445b68bb84c7f 100644 --- a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp @@ -65,7 +65,7 @@ void updateFromSessionConfigs( velox::common::stringToCompressionKind(compression); queryConfigs[velox::core::QueryConfig::kShuffleCompressionKind] = velox::common::compressionKindToString(compressionKind); - } else if(!sessionProperties->hasVeloxConfig(it.first)) { + } else if (!sessionProperties->hasVeloxConfig(it.first)) { sessionProperties->updateSessionPropertyValue(it.first, it.second); } else { queryConfigs[sessionProperties->toVeloxConfig(it.first)] = it.second; @@ -73,7 +73,8 @@ void updateFromSessionConfigs( } if (session.startTime) { - queryConfigs[velox::core::QueryConfig::kSessionStartTime] = std::to_string(session.startTime); + queryConfigs[velox::core::QueryConfig::kSessionStartTime] = + std::to_string(session.startTime); } if (session.source) { @@ -202,17 +203,15 @@ std::unordered_map toVeloxConfigs( } velox::core::QueryConfig toVeloxConfigs( - const protocol::SessionRepresentation& session, - const std::map& extraCredentials) { + const protocol::SessionRepresentation& session, + const std::map& extraCredentials) { // Start with the session-based configuration auto configs = toVeloxConfigs(session); // If there are any extra credentials, add them all to the config if (!extraCredentials.empty()) { // Create new config map with all extra credentials added - configs.insert( - extraCredentials.begin(), - extraCredentials.end()); + configs.insert(extraCredentials.begin(), extraCredentials.end()); } return velox::core::QueryConfig(configs); } diff --git a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.h b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.h index 67cd637e85694..8e6b4cbf9261e 100644 --- a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.h +++ b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.h @@ -25,20 +25,20 @@ class QueryConfig; namespace facebook::presto { -/// Translates Presto configs to Velox 'QueryConfig' config map. Presto query +/// Translates Presto configs to Velox 'QueryConfig' config map. Presto query /// session properties take precedence over Presto system config properties. std::unordered_map toVeloxConfigs( const protocol::SessionRepresentation& session); -/// Translates Presto configs to Velox 'QueryConfig' config map. It is the +/// Translates Presto configs to Velox 'QueryConfig' config map. It is the /// temporary overload that builds a QueryConfig from session properties and -/// extraCredentials, including all extraCredentials so they can be consumed by +/// extraCredentials, including all extraCredentials so they can be consumed by /// UDFs and connectors. /// This implementation is a temporary solution until a more unified /// configuration mechanism (TokenProvider) is available. velox::core::QueryConfig toVeloxConfigs( - const protocol::SessionRepresentation& session, - const std::map& extraCredentials); + const protocol::SessionRepresentation& session, + const std::map& extraCredentials); std::unordered_map> toConnectorConfigs(const protocol::TaskUpdateRequest& taskUpdateRequest); diff --git a/presto-native-execution/presto_cpp/main/QueryContextManager.cpp b/presto-native-execution/presto_cpp/main/QueryContextManager.cpp index 538e7df861a8e..f31868cfca41d 100644 --- a/presto-native-execution/presto_cpp/main/QueryContextManager.cpp +++ b/presto-native-execution/presto_cpp/main/QueryContextManager.cpp @@ -45,7 +45,8 @@ QueryContextManager::findOrCreateQueryCtx( const protocol::TaskUpdateRequest& taskUpdateRequest) { return findOrCreateQueryCtx( taskId, - toVeloxConfigs(taskUpdateRequest.session, taskUpdateRequest.extraCredentials), + toVeloxConfigs( + taskUpdateRequest.session, taskUpdateRequest.extraCredentials), toConnectorConfigs(taskUpdateRequest)); } diff --git a/presto-native-execution/presto_cpp/main/QueryContextManager.h b/presto-native-execution/presto_cpp/main/QueryContextManager.h index 99a2a70263948..61ce3b78c203b 100644 --- a/presto-native-execution/presto_cpp/main/QueryContextManager.h +++ b/presto-native-execution/presto_cpp/main/QueryContextManager.h @@ -136,9 +136,10 @@ class QueryContextManager { void setQueryHasStartedTasks(const protocol::TaskId& taskId); /// Calls the given functor for every present query context. - void visitAllContexts(const std::function& visitor) const; + void visitAllContexts( + const std::function< + void(const protocol::QueryId&, const velox::core::QueryCtx*)>& + visitor) const; /// Test method to clear the query context cache. void testingClearCache(); @@ -149,11 +150,13 @@ class QueryContextManager { private: virtual std::shared_ptr createAndCacheQueryCtx( - QueryContextCache& cache, - const protocol::QueryId& queryId, - velox::core::QueryConfig&& queryConfig, - std::unordered_map>&& connectorConfigs, - std::shared_ptr&& pool); + QueryContextCache& cache, + const protocol::QueryId& queryId, + velox::core::QueryConfig&& queryConfig, + std::unordered_map< + std::string, + std::shared_ptr>&& connectorConfigs, + std::shared_ptr&& pool); std::shared_ptr findOrCreateQueryCtx( const protocol::TaskId& taskId, diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index 52160c1e2db91..5a8d9a0edd438 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -38,12 +38,7 @@ void SessionProperties::addSessionProperty( const std::optional veloxConfig, const std::string& defaultValue) { sessionProperties_[name] = std::make_shared( - name, - description, - type->toString(), - isHidden, - veloxConfig, - defaultValue); + name, description, type->toString(), isHidden, veloxConfig, defaultValue); } // List of native session properties is kept as the source of truth here. @@ -588,7 +583,8 @@ SessionProperties::SessionProperties() { const std::string SessionProperties::toVeloxConfig( const std::string& name) const { auto it = sessionProperties_.find(name); - if (it != sessionProperties_.end() && it->second->getVeloxConfig().has_value()) { + if (it != sessionProperties_.end() && + it->second->getVeloxConfig().has_value()) { return it->second->getVeloxConfig().value(); } return name; diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index 6bd9ebc1662cd..4db928cdd641d 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -343,7 +343,7 @@ class SessionProperties { /// a single output for each input batch. static constexpr const char* kUnnestSplitOutput = "native_unnest_split_output"; - + /// Preferred size of batches in bytes to be returned by operators from /// Operator::getOutput. It is used when an estimate of average row size is /// known. Otherwise kPreferredOutputBatchRows is used. @@ -363,7 +363,8 @@ class SessionProperties { /// output rows. static constexpr const char* kMaxOutputBatchRows = "max_output_batch_rows"; - /// Enable (reader) row size tracker as a fallback to file level row size estimates. + /// Enable (reader) row size tracker as a fallback to file level row size + /// estimates. static constexpr const char* kRowSizeTrackingEnabled = "row_size_tracking_enabled"; @@ -375,15 +376,17 @@ class SessionProperties { inline bool hasVeloxConfig(const std::string& key) { auto sessionProperty = sessionProperties_.find(key); - if(sessionProperty == sessionProperties_.end()) { - // In this case a queryConfig is being created so we should return - // true since it will also have a veloxConfig. - return true; + if (sessionProperty == sessionProperties_.end()) { + // In this case a queryConfig is being created so we should return + // true since it will also have a veloxConfig. + return true; } return sessionProperty->second->getVeloxConfig().has_value(); } - inline void updateSessionPropertyValue(const std::string& key, const std::string& value) { + inline void updateSessionPropertyValue( + const std::string& key, + const std::string& value) { auto sessionProperty = sessionProperties_.find(key); VELOX_CHECK(sessionProperty != sessionProperties_.end()); sessionProperty->second->updateValue(value); diff --git a/presto-native-execution/presto_cpp/main/TaskManager.cpp b/presto-native-execution/presto_cpp/main/TaskManager.cpp index ab0fb8809c816..2d849295873ff 100644 --- a/presto-native-execution/presto_cpp/main/TaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/TaskManager.cpp @@ -355,9 +355,10 @@ TaskManager::TaskManager( folly::Executor* driverExecutor, folly::Executor* httpSrvCpuExecutor, folly::Executor* spillerExecutor) - : queryContextManager_(std::make_unique( - driverExecutor, - spillerExecutor)), + : queryContextManager_( + std::make_unique( + driverExecutor, + spillerExecutor)), bufferManager_(velox::exec::OutputBufferManager::getInstanceRef()), httpSrvCpuExecutor_(httpSrvCpuExecutor) { VELOX_CHECK_NOT_NULL(bufferManager_, "invalid OutputBufferManager"); @@ -1028,8 +1029,9 @@ folly::Future> TaskManager::getTaskInfo( auto prestoTask = findOrCreateTask(taskId); if (!currentState || !maxWait) { // Return current TaskInfo without waiting. - promise.setValue(std::make_unique( - prestoTask->updateInfo(summarize))); + promise.setValue( + std::make_unique( + prestoTask->updateInfo(summarize))); prestoTask->updateCoordinatorHeartbeat(); return std::move(future).via(httpSrvCpuExecutor_); } @@ -1072,8 +1074,9 @@ folly::Future> TaskManager::getTaskInfo( prestoTask->task->stateChangeFuture(maxWaitMicros) .via(httpSrvCpuExecutor_) .thenValue([promiseHolder, prestoTask, summarize](auto&& /*done*/) { - promiseHolder->promise.setValue(std::make_unique( - prestoTask->updateInfo(summarize))); + promiseHolder->promise.setValue( + std::make_unique( + prestoTask->updateInfo(summarize))); }) .thenError( folly::tag_t{}, diff --git a/presto-native-execution/presto_cpp/main/TaskResource.cpp b/presto-native-execution/presto_cpp/main/TaskResource.cpp index ec08cf19bcf02..3fc4a2e42b4ff 100644 --- a/presto-native-execution/presto_cpp/main/TaskResource.cpp +++ b/presto-native-execution/presto_cpp/main/TaskResource.cpp @@ -216,10 +216,12 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTaskImpl( bool summarize = message->hasQueryParam("summarize"); const auto& headers = message->getHeaders(); - const auto& acceptHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); + const auto& acceptHeader = + headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); const auto sendThrift = acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; - const auto& contentHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_CONTENT_TYPE); + const auto& contentHeader = + headers.getSingleOrEmpty(proxygen::HTTP_HEADER_CONTENT_TYPE); const auto receiveThrift = contentHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; @@ -242,7 +244,11 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTaskImpl( std::unique_ptr taskInfo; try { taskInfo = createOrUpdateFunc( - taskId, requestBody, summarize, startProcessCpuTimeNs, receiveThrift); + taskId, + requestBody, + summarize, + startProcessCpuTimeNs, + receiveThrift); } catch (const velox::VeloxException& e) { // Creating an empty task, putting errors inside so that next // status fetch from coordinator will catch the error and well @@ -355,7 +361,8 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTask( bool receiveThrift) { protocol::TaskUpdateRequest updateRequest; if (receiveThrift) { - auto thriftTaskUpdateRequest = std::make_shared(); + auto thriftTaskUpdateRequest = + std::make_shared(); thriftRead(requestBody, thriftTaskUpdateRequest); fromThrift(*thriftTaskUpdateRequest, updateRequest); } else { @@ -364,7 +371,10 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTask( velox::core::PlanFragment planFragment; std::shared_ptr queryCtx; if (updateRequest.fragment) { - protocol::PlanFragment prestoPlan = json::parse(receiveThrift ? *updateRequest.fragment : velox::encoding::Base64::decode(*updateRequest.fragment)); + protocol::PlanFragment prestoPlan = json::parse( + receiveThrift + ? *updateRequest.fragment + : velox::encoding::Base64::decode(*updateRequest.fragment)); queryCtx = taskManager_.getQueryContextManager()->findOrCreateQueryCtx( @@ -397,7 +407,8 @@ proxygen::RequestHandler* TaskResource::deleteTask( } bool summarize = message->hasQueryParam("summarize"); const auto& headers = message->getHeaders(); - const auto& acceptHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); + const auto& acceptHeader = + headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); const auto sendThrift = acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; @@ -415,7 +426,8 @@ proxygen::RequestHandler* TaskResource::deleteTask( return std::move(taskInfo); }) .via(folly::EventBaseManager::get()->getEventBase()) - .thenValue([taskId, downstream, handlerState, sendThrift](auto&& taskInfo) { + .thenValue([taskId, downstream, handlerState, sendThrift]( + auto&& taskInfo) { if (!handlerState->requestExpired()) { if (taskInfo == nullptr) { sendTaskNotFound(downstream, taskId); @@ -550,7 +562,8 @@ proxygen::RequestHandler* TaskResource::getTaskStatus( auto maxWait = getMaxWait(message); const auto& headers = message->getHeaders(); - const auto& acceptHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); + const auto& acceptHeader = + headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); const auto sendThrift = acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; @@ -621,7 +634,8 @@ proxygen::RequestHandler* TaskResource::getTaskInfo( bool summarize = message->hasQueryParam("summarize"); const auto& headers = message->getHeaders(); - const auto& acceptHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); + const auto& acceptHeader = + headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); const auto sendThrift = acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; diff --git a/presto-native-execution/presto_cpp/main/common/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/CMakeLists.txt index 582bdb3b0fd51..0f98aeb721bfd 100644 --- a/presto-native-execution/presto_cpp/main/common/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/CMakeLists.txt @@ -14,11 +14,15 @@ add_library(presto_exception Exception.cpp) add_library(presto_common Counters.cpp Utils.cpp ConfigReader.cpp Configs.cpp) target_link_libraries(presto_exception velox_exception) -set_property(TARGET presto_exception PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_exception PROPERTY JOB_POOL_LINK presto_link_job_pool) -target_link_libraries(presto_common velox_common_config velox_core - velox_exception velox_presto_serializer) +target_link_libraries( + presto_common + velox_common_config + velox_core + velox_exception + velox_presto_serializer +) set_property(TARGET presto_common PROPERTY JOB_POOL_LINK presto_link_job_pool) if(PRESTO_ENABLE_TESTING) diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index 052ffd84c4ff0..78dc8bc652131 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -47,14 +47,11 @@ uint32_t hardwareConcurrency() { return numLogicalCores; } -#define STR_PROP(_key_, _val_) \ - { std::string(_key_), std::string(_val_) } +#define STR_PROP(_key_, _val_) {std::string(_key_), std::string(_val_)} #define NUM_PROP(_key_, _val_) \ - { std::string(_key_), folly::to(_val_) } -#define BOOL_PROP(_key_, _val_) \ - { std::string(_key_), bool2String(_val_) } -#define NONE_PROP(_key_) \ - { std::string(_key_), folly::none } + {std::string(_key_), folly::to(_val_)} +#define BOOL_PROP(_key_, _val_) {std::string(_key_), bool2String(_val_)} +#define NONE_PROP(_key_) {std::string(_key_), folly::none} } // namespace void ConfigBase::initialize(const std::string& filePath, bool optionalConfig) { @@ -691,8 +688,9 @@ std::string SystemConfig::sharedArbitratorFastExponentialGrowthCapacityLimit() kSharedArbitratorFastExponentialGrowthCapacityLimitDefault = "512MB"; return optionalProperty( kSharedArbitratorFastExponentialGrowthCapacityLimit) - .value_or(std::string( - kSharedArbitratorFastExponentialGrowthCapacityLimitDefault)); + .value_or( + std::string( + kSharedArbitratorFastExponentialGrowthCapacityLimitDefault)); } std::string SystemConfig::sharedArbitratorSlowCapacityGrowPct() const { @@ -742,8 +740,9 @@ std::string SystemConfig::sharedArbitratorMemoryReclaimThreadsHwMultiplier() kSharedArbitratorMemoryReclaimThreadsHwMultiplierDefault = "0.5"; return optionalProperty( kSharedArbitratorMemoryReclaimThreadsHwMultiplier) - .value_or(std::string( - kSharedArbitratorMemoryReclaimThreadsHwMultiplierDefault)); + .value_or( + std::string( + kSharedArbitratorMemoryReclaimThreadsHwMultiplierDefault)); } std::string SystemConfig::sharedArbitratorGlobalArbitrationMemoryReclaimPct() @@ -752,8 +751,9 @@ std::string SystemConfig::sharedArbitratorGlobalArbitrationMemoryReclaimPct() kSharedArbitratorGlobalArbitrationMemoryReclaimPctDefault = "10"; return optionalProperty( kSharedArbitratorGlobalArbitrationMemoryReclaimPct) - .value_or(std::string( - kSharedArbitratorGlobalArbitrationMemoryReclaimPctDefault)); + .value_or( + std::string( + kSharedArbitratorGlobalArbitrationMemoryReclaimPctDefault)); } std::string SystemConfig::sharedArbitratorGlobalArbitrationAbortTimeRatio() diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index b22c8ee282b04..b966d26540772 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -140,8 +140,9 @@ class ConfigBase { protected: ConfigBase() - : config_(std::make_unique( - std::unordered_map())){}; + : config_( + std::make_unique( + std::unordered_map())) {}; // Check if all properties are registered. void checkRegisteredProperties( diff --git a/presto-native-execution/presto_cpp/main/common/Counters.cpp b/presto-native-execution/presto_cpp/main/common/Counters.cpp index 048f7201029f0..2d9b903971440 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.cpp +++ b/presto-native-execution/presto_cpp/main/common/Counters.cpp @@ -147,14 +147,7 @@ void registerPrestoMetrics() { // Tracks exchange get-data-size request duration in range of [0, 300s] with // 300 buckets and reports P50, P90, P99, and P100. DEFINE_HISTOGRAM_METRIC( - kCounterExchangeGetDataSizeDuration, - 1'000, - 0, - 300'000, - 50, - 90, - 99, - 100); + kCounterExchangeGetDataSizeDuration, 1'000, 0, 300'000, 50, 90, 99, 100); // Tracks exchange get-data-size request num of tries in range of [0, 20] with // 20 buckets and reports P50, P90, P99, and P100. DEFINE_HISTOGRAM_METRIC( @@ -182,8 +175,10 @@ void registerPrestoMetrics() { 99, 100); - DEFINE_METRIC(kCounterExchangeIoEvbViolation, facebook::velox::StatType::COUNT); - DEFINE_METRIC(kCounterHttpServerIoEvbViolation, facebook::velox::StatType::COUNT); + DEFINE_METRIC( + kCounterExchangeIoEvbViolation, facebook::velox::StatType::COUNT); + DEFINE_METRIC( + kCounterHttpServerIoEvbViolation, facebook::velox::StatType::COUNT); // NOTE: Metrics type exporting for thread pool executor counters are in // PeriodicTaskManager because they have dynamic names and report configs. The diff --git a/presto-native-execution/presto_cpp/main/common/Counters.h b/presto-native-execution/presto_cpp/main/common/Counters.h index 4dbccac18813a..28e9fc3cc0999 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.h +++ b/presto-native-execution/presto_cpp/main/common/Counters.h @@ -205,9 +205,9 @@ constexpr std::string_view kCounterThreadPoolMaxIdleTimeNsFormat{ /// ================== EVB Counters ==================== constexpr folly::StringPiece kCounterExchangeIoEvbViolation{ - "presto_cpp.exchange_io_evb_violation_count"}; + "presto_cpp.exchange_io_evb_violation_count"}; constexpr folly::StringPiece kCounterHttpServerIoEvbViolation{ - "presto_cpp.http_server_io_evb_violation_count"}; + "presto_cpp.http_server_io_evb_violation_count"}; /// ================== Memory Pushback Counters ================= diff --git a/presto-native-execution/presto_cpp/main/common/Utils.h b/presto-native-execution/presto_cpp/main/common/Utils.h index 3b9560d54e370..63462436aafa8 100644 --- a/presto-native-execution/presto_cpp/main/common/Utils.h +++ b/presto-native-execution/presto_cpp/main/common/Utils.h @@ -12,9 +12,9 @@ * limitations under the License. */ #pragma once +#include #include #include -#include namespace facebook::presto::util { diff --git a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt index fb83b2f843062..0c96e5270744b 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt @@ -11,10 +11,7 @@ # limitations under the License. add_library(presto_mutable_configs MutableConfigs.cpp) -target_link_libraries( - presto_mutable_configs - presto_common - velox_file) +target_link_libraries(presto_mutable_configs presto_common velox_file) add_executable(presto_common_test CommonTest.cpp ConfigTest.cpp) @@ -34,7 +31,7 @@ target_link_libraries( velox_window ${RE2} GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) -set_property(TARGET presto_common_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_common_test PROPERTY JOB_POOL_LINK presto_link_job_pool) diff --git a/presto-native-execution/presto_cpp/main/common/tests/CommonTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/CommonTest.cpp index 0c435376663fa..f35b13890c70b 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/CommonTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/CommonTest.cpp @@ -137,8 +137,9 @@ TEST(VeloxToPrestoExceptionTranslatorTest, allErrorCodeTranslations) { for (const auto& [errorSource, errorCodeMap] : translateMap) { for (const auto& [errorCode, expectedErrorCode] : errorCodeMap) { - SCOPED_TRACE(fmt::format( - "errorSource: {}, errorCode: {}", errorSource, errorCode)); + SCOPED_TRACE( + fmt::format( + "errorSource: {}, errorCode: {}", errorSource, errorCode)); // Determine the exception type based on error source if (errorSource == velox::error_source::kErrorSourceRuntime) { diff --git a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp index 10c1030bf6e02..ce07dc3cfa545 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp @@ -213,9 +213,9 @@ TEST_F(ConfigTest, optionalNodeConfigs) { ASSERT_EQ( config.nodeInternalAddress([]() { return "0.0.0.0"; }), "127.0.0.1"); - init(config, {{std::string(NodeConfig::kNodePrometheusExecutorThreads), "4"}}); - ASSERT_EQ( - config.prometheusExecutorThreads(), 4); + init( + config, {{std::string(NodeConfig::kNodePrometheusExecutorThreads), "4"}}); + ASSERT_EQ(config.prometheusExecutorThreads(), 4); } TEST_F(ConfigTest, optionalSystemConfigsWithDefault) { diff --git a/presto-native-execution/presto_cpp/main/common/tests/MutableConfigs.cpp b/presto-native-execution/presto_cpp/main/common/tests/MutableConfigs.cpp index 1232cc0e4c5cf..cc729e58186ef 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/MutableConfigs.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/MutableConfigs.cpp @@ -11,8 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/common/tests/MutableConfigs.h" +#include "presto_cpp/main/common/Configs.h" #include "velox/common/file/File.h" #include "velox/common/file/FileSystems.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" diff --git a/presto-native-execution/presto_cpp/main/common/tests/test_json.h b/presto-native-execution/presto_cpp/main/common/tests/test_json.h index 169a48534a2be..e2aaafbb6a53e 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/test_json.h +++ b/presto-native-execution/presto_cpp/main/common/tests/test_json.h @@ -13,10 +13,10 @@ */ #pragma once +#include +#include #include #include -#include -#include #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" @@ -52,7 +52,9 @@ inline std::string slurp(const std::string& path) { return buf.str(); } -inline std::string getDataPath(const std::string& dirUnderFbcode, const std::string& fileName) { +inline std::string getDataPath( + const std::string& dirUnderFbcode, + const std::string& fileName) { std::string currentPath = fs::current_path().c_str(); if (boost::algorithm::ends_with(currentPath, "fbcode")) { return currentPath + dirUnderFbcode + fileName; diff --git a/presto-native-execution/presto_cpp/main/connectors/CMakeLists.txt b/presto-native-execution/presto_cpp/main/connectors/CMakeLists.txt index 694c13fbb00f6..328ff8bf38933 100644 --- a/presto-native-execution/presto_cpp/main/connectors/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/connectors/CMakeLists.txt @@ -9,18 +9,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -add_library(presto_connectors - IcebergPrestoToVeloxConnector.cpp - PrestoToVeloxConnectorUtils.cpp - HivePrestoToVeloxConnector.cpp - Registration.cpp - PrestoToVeloxConnector.cpp - SystemConnector.cpp) +add_library( + presto_connectors + IcebergPrestoToVeloxConnector.cpp + PrestoToVeloxConnectorUtils.cpp + HivePrestoToVeloxConnector.cpp + Registration.cpp + PrestoToVeloxConnector.cpp + SystemConnector.cpp +) if(PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR) add_subdirectory(arrow_flight) - target_compile_definitions(presto_connectors - PUBLIC PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR) + target_compile_definitions(presto_connectors PUBLIC PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR) target_link_libraries(presto_connectors presto_flight_connector) endif() @@ -28,5 +29,9 @@ if(PRESTO_ENABLE_CUDF) target_link_libraries(presto_connectors velox_cudf_hive_connector cudf::cudf) endif() -target_link_libraries(presto_connectors presto_velox_expr_conversion - velox_type_fbhive velox_tpcds_connector) +target_link_libraries( + presto_connectors + presto_velox_expr_conversion + velox_type_fbhive + velox_tpcds_connector +) diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp index f5083ab45f2ab..ef116c94c7785 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp @@ -13,8 +13,8 @@ */ #include "presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h" -#include "presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h" #include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" +#include "presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h" #include "presto_cpp/presto_protocol/connector/iceberg/IcebergConnectorProtocol.h" #include "velox/connectors/hive/iceberg/IcebergDataSink.h" diff --git a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnector.h b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnector.h index 2c8fafa333951..a3fce5054dfce 100644 --- a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnector.h @@ -59,8 +59,7 @@ class PrestoToVeloxConnector { const protocol::TableHandle& tableHandle, const VeloxExprConverter& exprConverter, const TypeParser& typeParser, - velox::connector::ColumnHandleMap& assignments) - const = 0; + velox::connector::ColumnHandleMap& assignments) const = 0; [[nodiscard]] virtual std::unique_ptr< velox::connector::ConnectorInsertTableHandle> @@ -134,8 +133,7 @@ class TpchPrestoToVeloxConnector final : public PrestoToVeloxConnector { const protocol::TableHandle& tableHandle, const VeloxExprConverter& exprConverter, const TypeParser& typeParser, - velox::connector::ColumnHandleMap& assignments) - const final; + velox::connector::ColumnHandleMap& assignments) const final; std::unique_ptr createConnectorProtocol() const final; diff --git a/presto-native-execution/presto_cpp/main/connectors/Registration.cpp b/presto-native-execution/presto_cpp/main/connectors/Registration.cpp index c8b3dab1429b6..52cfd06f2a15d 100644 --- a/presto-native-execution/presto_cpp/main/connectors/Registration.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/Registration.cpp @@ -50,16 +50,19 @@ std::vector listConnectorFactories() { void registerConnectors() { registerConnectorFactories(); - registerPrestoToVeloxConnector(std::make_unique( - velox::connector::hive::HiveConnectorFactory::kHiveConnectorName)); + registerPrestoToVeloxConnector( + std::make_unique( + velox::connector::hive::HiveConnectorFactory::kHiveConnectorName)); registerPrestoToVeloxConnector( std::make_unique(kHiveHadoop2ConnectorName)); registerPrestoToVeloxConnector( std::make_unique(kIcebergConnectorName)); - registerPrestoToVeloxConnector(std::make_unique( - velox::connector::tpch::TpchConnectorFactory::kTpchConnectorName)); - registerPrestoToVeloxConnector(std::make_unique( - velox::connector::tpcds::TpcdsConnectorFactory::kTpcdsConnectorName)); + registerPrestoToVeloxConnector( + std::make_unique( + velox::connector::tpch::TpchConnectorFactory::kTpchConnectorName)); + registerPrestoToVeloxConnector( + std::make_unique( + velox::connector::tpcds::TpcdsConnectorFactory::kTpcdsConnectorName)); // Presto server uses system catalog or system schema in other catalogs // in different places in the code. All these resolve to the SystemConnector. @@ -74,8 +77,9 @@ void registerConnectors() { std::make_unique("$system@system")); #ifdef PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR - registerPrestoToVeloxConnector(std::make_unique( - ArrowFlightConnectorFactory::kArrowFlightConnectorName)); + registerPrestoToVeloxConnector( + std::make_unique( + ArrowFlightConnectorFactory::kArrowFlightConnectorName)); #endif } @@ -85,7 +89,8 @@ void registerConnectorFactories() { // Register Hive connector factory facebook::presto::registerConnectorFactory( - std::make_shared()); + std::make_shared< + facebook::velox::connector::hive::HiveConnectorFactory>()); // Register Hive Hadoop2 connector factory facebook::presto::registerConnectorFactory( @@ -93,7 +98,8 @@ void registerConnectorFactories() { kHiveHadoop2ConnectorName)); #ifdef PRESTO_ENABLE_CUDF facebook::presto::unregisterConnectorFactory( - facebook::velox::connector::hive::HiveConnectorFactory::kHiveConnectorName); + facebook::velox::connector::hive::HiveConnectorFactory:: + kHiveConnectorName); facebook::presto::unregisterConnectorFactory(kHiveHadoop2ConnectorName); // Register cuDF Hive connector factory @@ -104,7 +110,8 @@ void registerConnectorFactories() { // Register cudf Hive connector factory facebook::presto::registerConnectorFactory( std::make_shared(kHiveHadoop2ConnectorName)); + CudfHiveConnectorFactory>( + kHiveHadoop2ConnectorName)); #endif // Register TPC-DS connector factory @@ -114,7 +121,8 @@ void registerConnectorFactories() { // Register TPCH connector factory facebook::presto::registerConnectorFactory( - std::make_shared()); + std::make_shared< + facebook::velox::connector::tpch::TpchConnectorFactory>()); // Register Iceberg connector factory (using Hive implementation) facebook::presto::registerConnectorFactory( diff --git a/presto-native-execution/presto_cpp/main/connectors/Registration.h b/presto-native-execution/presto_cpp/main/connectors/Registration.h index 75420958b0f4a..cbff3e83c2bbf 100644 --- a/presto-native-execution/presto_cpp/main/connectors/Registration.h +++ b/presto-native-execution/presto_cpp/main/connectors/Registration.h @@ -27,7 +27,7 @@ class ConnectorFactory; } // namespace facebook::velox::connector namespace facebook::presto { -using facebook::velox::connector::ConnectorFactory; +using facebook::velox::connector::ConnectorFactory; namespace detail { inline std::unordered_map>& @@ -45,8 +45,8 @@ connectorFactories() { inline bool registerConnectorFactory( std::shared_ptr factory) { const bool ok = detail::connectorFactories() - .insert({factory->connectorName(), factory}) - .second; + .insert({factory->connectorName(), factory}) + .second; VELOX_CHECK( ok, "ConnectorFactory with name '{}' is already registered", diff --git a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp index d75dc5435c60b..1d2ecb6ed181e 100644 --- a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp @@ -111,7 +111,8 @@ SystemDataSource::SystemDataSource( "ColumnHandle is missing for output column '{}'", outputName); - auto handle = std::dynamic_pointer_cast(it->second); + auto handle = + std::dynamic_pointer_cast(it->second); VELOX_CHECK_NOT_NULL( handle, "ColumnHandle must be an instance of SystemColumnHandle " diff --git a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h index 5e71ca277f0a7..536d8d2ab6838 100644 --- a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h @@ -163,7 +163,8 @@ class SystemConnector : public velox::connector::Connector { std::unique_ptr createDataSink( velox::RowTypePtr /*inputType*/, - velox::connector::ConnectorInsertTableHandlePtr /*connectorInsertTableHandle*/, + velox::connector:: + ConnectorInsertTableHandlePtr /*connectorInsertTableHandle*/, velox::connector::ConnectorQueryCtx* /*connectorQueryCtx*/, velox::connector::CommitStrategy /*commitStrategy*/) override final { VELOX_NYI("SystemConnector does not support data sink."); @@ -191,8 +192,7 @@ class SystemPrestoToVeloxConnector final : public PrestoToVeloxConnector { const protocol::TableHandle& tableHandle, const VeloxExprConverter& exprConverter, const TypeParser& typeParser, - velox::connector::ColumnHandleMap& assignments) - const final; + velox::connector::ColumnHandleMap& assignments) const final; std::unique_ptr createConnectorProtocol() const final; diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConfig.h b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConfig.h index 4839733baaab0..3ccfe06038d5c 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConfig.h +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConfig.h @@ -41,8 +41,7 @@ class ArrowFlightConfig { static constexpr const char* kClientSslCertificate = "arrow-flight.client-ssl-certificate"; - static constexpr const char* kClientSslKey = - "arrow-flight.client-ssl-key"; + static constexpr const char* kClientSslKey = "arrow-flight.client-ssl-key"; std::string authenticatorName() const; @@ -57,7 +56,7 @@ class ArrowFlightConfig { std::optional serverSslCertificate() const; std::optional clientSslCertificate() const; - + std::optional clientSslKey() const; private: diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp index f39dbcb8215ee..69083e8035fb9 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp @@ -75,7 +75,9 @@ ArrowFlightConnector::initClientOpts( if (clientCertPath.has_value()) { std::ifstream certFile(clientCertPath.value()); VELOX_CHECK( - certFile.is_open(), "Could not open client certificate at {}", clientCertPath.value()); + certFile.is_open(), + "Could not open client certificate at {}", + clientCertPath.value()); clientOpts->cert_chain.assign( (std::istreambuf_iterator(certFile)), (std::istreambuf_iterator())); @@ -85,7 +87,9 @@ ArrowFlightConnector::initClientOpts( if (clientKeyPath.has_value()) { std::ifstream keyFile(clientKeyPath.value()); VELOX_CHECK( - keyFile.is_open(), "Could not open client key at {}", clientKeyPath.value()); + keyFile.is_open(), + "Could not open client key at {}", + clientKeyPath.value()); clientOpts->private_key.assign( (std::istreambuf_iterator(keyFile)), (std::istreambuf_iterator())); diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowPrestoToVeloxConnector.h b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowPrestoToVeloxConnector.h index 5ac419d012b9c..5729c18372068 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowPrestoToVeloxConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowPrestoToVeloxConnector.h @@ -35,8 +35,7 @@ class ArrowPrestoToVeloxConnector final : public PrestoToVeloxConnector { const protocol::TableHandle& tableHandle, const VeloxExprConverter& exprConverter, const TypeParser& typeParser, - velox::connector::ColumnHandleMap& assignments) - const final; + velox::connector::ColumnHandleMap& assignments) const final; std::unique_ptr createConnectorProtocol() const final; diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/CMakeLists.txt b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/CMakeLists.txt index ed3e60f6be707..b4dc9a1829df1 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/CMakeLists.txt @@ -18,16 +18,22 @@ add_library(presto_flight_connector_utils INTERFACE Macros.h) target_link_libraries(presto_flight_connector_utils INTERFACE velox_exception) add_library( - presto_flight_connector OBJECT - ArrowFlightConnector.cpp ArrowPrestoToVeloxConnector.cpp - ArrowFlightConfig.cpp) + presto_flight_connector + OBJECT + ArrowFlightConnector.cpp + ArrowPrestoToVeloxConnector.cpp + ArrowFlightConfig.cpp +) -target_compile_definitions(presto_flight_connector - PUBLIC PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR) +target_compile_definitions(presto_flight_connector PUBLIC PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR) target_link_libraries( - presto_flight_connector velox_connector ArrowFlight::arrow_flight_shared - presto_flight_connector_utils presto_flight_connector_auth) + presto_flight_connector + velox_connector + ArrowFlight::arrow_flight_shared + presto_flight_connector_utils + presto_flight_connector_auth +) if(PRESTO_ENABLE_TESTING) add_subdirectory(tests) diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/auth/CMakeLists.txt b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/auth/CMakeLists.txt index 1e7eba3154a0e..5e9564361ae47 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/auth/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/auth/CMakeLists.txt @@ -11,5 +11,4 @@ # limitations under the License. add_library(presto_flight_connector_auth Authenticator.cpp) -target_link_libraries(presto_flight_connector_auth - presto_flight_connector_utils velox_exception) +target_link_libraries(presto_flight_connector_auth presto_flight_connector_utils velox_exception) diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorDataTypeTest.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorDataTypeTest.cpp index 257497caf224d..f1a489600b825 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorDataTypeTest.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorDataTypeTest.cpp @@ -77,14 +77,16 @@ TEST_F(ArrowFlightConnectorDataTypeTest, integerTypes) { auto bigintVec = makeFlatVector(bigData); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"tinyint_col", "smallint_col", "integer_col", "bigint_col"}, - {velox::TINYINT(), - velox::SMALLINT(), - velox::INTEGER(), - velox::BIGINT()})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW( + {"tinyint_col", "smallint_col", "integer_col", "bigint_col"}, + {velox::TINYINT(), + velox::SMALLINT(), + velox::INTEGER(), + velox::BIGINT()})) + .planNode(); AssertQueryBuilder(plan) .splits(makeSplits({"sample-data"})) @@ -115,10 +117,12 @@ TEST_F(ArrowFlightConnectorDataTypeTest, realType) { auto doubleVec = makeFlatVector(doubleData); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"real_col", "double_col"}, {velox::REAL(), velox::DOUBLE()})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW( + {"real_col", "double_col"}, {velox::REAL(), velox::DOUBLE()})) + .planNode(); AssertQueryBuilder(plan) .splits(makeSplits({"sample-data"})) @@ -243,9 +247,12 @@ TEST_F(ArrowFlightConnectorDataTypeTest, timestampType) { core::PlanNodePtr plan; plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"timestampsec_col", "timestampmilli_col", "timestampmicro_col"}, - {velox::TIMESTAMP(), velox::TIMESTAMP(), velox::TIMESTAMP()})) + .flightTableScan( + velox::ROW( + {"timestampsec_col", + "timestampmilli_col", + "timestampmicro_col"}, + {velox::TIMESTAMP(), velox::TIMESTAMP(), velox::TIMESTAMP()})) .planNode(); AssertQueryBuilder(plan) @@ -301,11 +308,13 @@ TEST_F(ArrowFlightConnectorDataTypeTest, decimalType) { auto decimalVecBigInt = makeFlatVector(decimalValuesBigInt); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"decimal_col_bigint"}, - {velox::DECIMAL(18, 2)})) // precision can't be 0 and < scale - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW( + {"decimal_col_bigint"}, + {velox::DECIMAL(18, 2)})) // precision can't be 0 and < scale + .planNode(); // Execute the query and assert the results AssertQueryBuilder(plan) @@ -338,10 +347,11 @@ TEST_F(ArrowFlightConnectorDataTypeTest, arrayType) { updateTable("sample-data", makeArrowTable({"int_array_col"}, {listArray})); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"int_array_col"}, {velox::ARRAY(velox::INTEGER())})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW({"int_array_col"}, {velox::ARRAY(velox::INTEGER())})) + .planNode(); auto expectedData = makeNullableArrayVector(data); AssertQueryBuilder(plan) @@ -396,10 +406,12 @@ TEST_F(ArrowFlightConnectorDataTypeTest, mapType) { updateTable("sample-data", makeArrowTable({"map_col"}, {mapArray})); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"map_col"}, {velox::MAP(velox::INTEGER(), velox::BIGINT())})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW( + {"map_col"}, {velox::MAP(velox::INTEGER(), velox::BIGINT())})) + .planNode(); auto expectedData = makeNullableMapVector(data); AssertQueryBuilder(plan) @@ -422,13 +434,15 @@ TEST_F(ArrowFlightConnectorDataTypeTest, rowType) { updateTable("sample-data", makeArrowTable({"row_col"}, {structArray})); core::PlanNodePtr plan; - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"row_col"}, - {velox::ROW( - {"int_col", "varchar_col", "double_col"}, - {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW( + {"row_col"}, + {velox::ROW( + {"int_col", "varchar_col", "double_col"}, + {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})})) + .planNode(); auto expectedData = makeRowVector( {makeFlatVector(intData), @@ -482,19 +496,20 @@ TEST_F(ArrowFlightConnectorDataTypeTest, allTypes) { core::PlanNodePtr plan; plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"daydate_col", - "timestamp_col", - "varchar_col", - "real_col", - "int_col", - "bool_col"}, - {velox::DATE(), - velox::TIMESTAMP(), - velox::VARCHAR(), - velox::DOUBLE(), - velox::INTEGER(), - velox::BOOLEAN()})) + .flightTableScan( + velox::ROW( + {"daydate_col", + "timestamp_col", + "varchar_col", + "real_col", + "int_col", + "bool_col"}, + {velox::DATE(), + velox::TIMESTAMP(), + velox::VARCHAR(), + velox::DOUBLE(), + velox::INTEGER(), + velox::BOOLEAN()})) .planNode(); AssertQueryBuilder(plan) diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorMTlsTest.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorMTlsTest.cpp index 8225172c8b526..9f17a2f1552cd 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorMTlsTest.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorMTlsTest.cpp @@ -96,16 +96,19 @@ class ArrowFlightConnectorMtlsTest : public ArrowFlightConnectorMtlsTestBase { {ArrowFlightConfig::kDefaultServerSslEnabled, "true"}, {ArrowFlightConfig::kServerVerify, "true"}, {ArrowFlightConfig::kServerSslCertificate, - "./data/certs/ca.crt"}, - {ArrowFlightConfig::kClientSslCertificate, "./data/certs/client.crt"}, - {ArrowFlightConfig::kClientSslKey, "./data/certs/client.key"}})) {} + "./data/certs/ca.crt"}, + {ArrowFlightConfig::kClientSslCertificate, + "./data/certs/client.crt"}, + {ArrowFlightConfig::kClientSslKey, + "./data/certs/client.key"}})) {} }; TEST_F(ArrowFlightConnectorMtlsTest, successfulMtlsConnection) { executeSuccessfulQuery(); } -class ArrowFlightMtlsNoClientCertTest : public ArrowFlightConnectorMtlsTestBase { +class ArrowFlightMtlsNoClientCertTest + : public ArrowFlightConnectorMtlsTestBase { protected: ArrowFlightMtlsNoClientCertTest() : ArrowFlightConnectorMtlsTestBase( @@ -113,7 +116,8 @@ class ArrowFlightMtlsNoClientCertTest : public ArrowFlightConnectorMtlsTestBase std::unordered_map{ {ArrowFlightConfig::kDefaultServerSslEnabled, "true"}, {ArrowFlightConfig::kServerVerify, "true"}, - {ArrowFlightConfig::kServerSslCertificate, "./data/certs/ca.crt"}})) {} + {ArrowFlightConfig::kServerSslCertificate, + "./data/certs/ca.crt"}})) {} }; TEST_F(ArrowFlightMtlsNoClientCertTest, mtlsFailsWithoutClientCert) { @@ -121,7 +125,8 @@ TEST_F(ArrowFlightMtlsNoClientCertTest, mtlsFailsWithoutClientCert) { VELOX_ASSERT_THROW(queryFunction(), "failed to connect"); } -class ArrowFlightConnectorImplicitSslTest : public ArrowFlightConnectorMtlsTestBase { +class ArrowFlightConnectorImplicitSslTest + : public ArrowFlightConnectorMtlsTestBase { protected: ArrowFlightConnectorImplicitSslTest() : ArrowFlightConnectorMtlsTestBase( @@ -129,16 +134,19 @@ class ArrowFlightConnectorImplicitSslTest : public ArrowFlightConnectorMtlsTestB std::unordered_map{ {ArrowFlightConfig::kServerVerify, "true"}, {ArrowFlightConfig::kServerSslCertificate, - "./data/certs/ca.crt"}, - {ArrowFlightConfig::kClientSslCertificate, "./data/certs/client.crt"}, - {ArrowFlightConfig::kClientSslKey, "./data/certs/client.key"}})) {} + "./data/certs/ca.crt"}, + {ArrowFlightConfig::kClientSslCertificate, + "./data/certs/client.crt"}, + {ArrowFlightConfig::kClientSslKey, + "./data/certs/client.key"}})) {} }; TEST_F(ArrowFlightConnectorImplicitSslTest, successfulImplicitSslConnection) { executeSuccessfulQuery(); } -class ArrowFlightImplicitSslNoClientCertTest : public ArrowFlightConnectorMtlsTestBase { +class ArrowFlightImplicitSslNoClientCertTest + : public ArrowFlightConnectorMtlsTestBase { protected: ArrowFlightImplicitSslNoClientCertTest() : ArrowFlightConnectorMtlsTestBase( @@ -146,11 +154,13 @@ class ArrowFlightImplicitSslNoClientCertTest : public ArrowFlightConnectorMtlsTe std::unordered_map{ {ArrowFlightConfig::kDefaultServerSslEnabled, "true"}, {ArrowFlightConfig::kServerVerify, "true"}, - {ArrowFlightConfig::kServerSslCertificate, "./data/certs/ca.crt"} - })) {} + {ArrowFlightConfig::kServerSslCertificate, + "./data/certs/ca.crt"}})) {} }; -TEST_F(ArrowFlightImplicitSslNoClientCertTest, mtlsFailsWithoutClientCertOnImplicitSsl) { +TEST_F( + ArrowFlightImplicitSslNoClientCertTest, + mtlsFailsWithoutClientCertOnImplicitSsl) { auto queryFunction = createQueryFunction(); VELOX_ASSERT_THROW(queryFunction(), "failed to connect"); } diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorTest.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorTest.cpp index acb41c5087ea1..a8f9cb36a10a8 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/ArrowFlightConnectorTest.cpp @@ -89,10 +89,11 @@ TEST_F(ArrowFlightConnectorTest, dataSource) { core::PlanNodePtr plan; // direct test - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"id", "value"}, {velox::BIGINT(), velox::INTEGER()})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW({"id", "value"}, {velox::BIGINT(), velox::INTEGER()})) + .planNode(); AssertQueryBuilder(plan) .splits(makeSplits({"sample-data"})) @@ -118,8 +119,9 @@ TEST_F(ArrowFlightConnectorTest, dataSource) { // invalid columnHandle test plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"ducks", "value"}, {velox::BIGINT(), velox::INTEGER()})) + .flightTableScan( + velox::ROW( + {"ducks", "value"}, {velox::BIGINT(), velox::INTEGER()})) .planNode(); VELOX_ASSERT_THROW( @@ -156,9 +158,10 @@ TEST_F(ArrowFlightConnectorTest, multipleBatches) { core::PlanNodePtr plan; plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"int_col", "varchar_col", "double_col"}, - {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})) + .flightTableScan( + velox::ROW( + {"int_col", "varchar_col", "double_col"}, + {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})) .planNode(); auto intVec = makeFlatVector(intData); @@ -204,9 +207,10 @@ TEST_F(ArrowFlightConnectorTest, multipleSplits) { core::PlanNodePtr plan; plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"int_col", "varchar_col", "double_col"}, - {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})) + .flightTableScan( + velox::ROW( + {"int_col", "varchar_col", "double_col"}, + {velox::INTEGER(), velox::VARCHAR(), velox::DOUBLE()})) .planNode(); std::vector intDataAll(intData); @@ -259,10 +263,11 @@ TEST_F(ArrowFlightConnectorTestDefaultServer, dataSource) { core::PlanNodePtr plan; // direct test - plan = ArrowFlightPlanBuilder() - .flightTableScan(velox::ROW( - {"id", "value"}, {velox::BIGINT(), velox::INTEGER()})) - .planNode(); + plan = + ArrowFlightPlanBuilder() + .flightTableScan( + velox::ROW({"id", "value"}, {velox::BIGINT(), velox::INTEGER()})) + .planNode(); AssertQueryBuilder(plan) .splits(makeSplits({"sample-data"})) diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/CMakeLists.txt index f346d1212a1ad..264aa6438a455 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/CMakeLists.txt @@ -11,27 +11,35 @@ # limitations under the License. add_subdirectory(utils) -add_executable(presto_flight_connector_infra_test - TestingArrowFlightServerTest.cpp) +add_executable(presto_flight_connector_infra_test TestingArrowFlightServerTest.cpp) add_test(presto_flight_connector_infra_test presto_flight_connector_infra_test) target_link_libraries( - presto_flight_connector_infra_test presto_protocol - presto_flight_connector_test_lib GTest::gtest GTest::gtest_main ${GLOG}) + presto_flight_connector_infra_test + presto_protocol + presto_flight_connector_test_lib + GTest::gtest + GTest::gtest_main + ${GLOG} +) add_executable( presto_flight_connector_test - ArrowFlightConnectorTest.cpp ArrowFlightConnectorAuthTest.cpp + ArrowFlightConnectorTest.cpp + ArrowFlightConnectorAuthTest.cpp ArrowFlightConnectorMTlsTest.cpp - ArrowFlightConnectorTlsTest.cpp ArrowFlightConnectorDataTypeTest.cpp - ArrowFlightConfigTest.cpp) + ArrowFlightConnectorTlsTest.cpp + ArrowFlightConnectorDataTypeTest.cpp + ArrowFlightConfigTest.cpp +) add_custom_target( - copy_flight_test_data ALL - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different - "${CMAKE_CURRENT_SOURCE_DIR}/data" - "${CMAKE_CURRENT_BINARY_DIR}/data" + copy_flight_test_data + ALL + COMMAND + ${CMAKE_COMMAND} -E copy_directory_if_different "${CMAKE_CURRENT_SOURCE_DIR}/data" + "${CMAKE_CURRENT_BINARY_DIR}/data" COMMENT "Copying test data files..." ) @@ -45,4 +53,5 @@ target_link_libraries( gtest gtest_main presto_flight_connector_test_lib - presto_protocol) + presto_protocol +) diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/data/generate_tls_certs.sh b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/data/generate_tls_certs.sh new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp index 2b692880f073d..2dbbaca394bcf 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp @@ -78,8 +78,9 @@ ArrowFlightConnectorTestBase::makeSplits( AFC_ASSIGN_OR_RAISE( auto flightEndpointStr, flightEndpoint.SerializeToString()); auto flightEndpointBytes = folly::base64Encode(flightEndpointStr); - splits.push_back(std::make_shared( - kFlightConnectorId, flightEndpointBytes)); + splits.push_back( + std::make_shared( + kFlightConnectorId, flightEndpointBytes)); } return splits; } diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/CMakeLists.txt b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/CMakeLists.txt index b6d2337a2d301..00881474b5604 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/CMakeLists.txt @@ -11,9 +11,17 @@ # limitations under the License. add_library( presto_flight_connector_test_lib - TestingArrowFlightServer.cpp ArrowFlightConnectorTestBase.cpp Utils.cpp - ArrowFlightPlanBuilder.cpp) + TestingArrowFlightServer.cpp + ArrowFlightConnectorTestBase.cpp + Utils.cpp + ArrowFlightPlanBuilder.cpp +) target_link_libraries( - presto_flight_connector_test_lib arrow presto_flight_connector - velox_exception presto_flight_connector_utils velox_exec_test_lib) + presto_flight_connector_test_lib + arrow + presto_flight_connector + velox_exception + presto_flight_connector_utils + velox_exec_test_lib +) diff --git a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt index cdab11b599e14..bb13a5e2eeb17 100644 --- a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt @@ -14,5 +14,4 @@ if(${PRESTO_ENABLE_EXAMPLES}) add_subdirectory(examples) endif() -add_library(presto_dynamic_function_registrar INTERFACE - DynamicFunctionRegistrar.h) +add_library(presto_dynamic_function_registrar INTERFACE DynamicFunctionRegistrar.h) diff --git a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md index 8163737d76879..8b50a754c716b 100644 --- a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md +++ b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md @@ -1 +1 @@ -See [Function Plugin](https://prestodb.io/docs/current/presto-cpp/plugin.html) on how to use the Dynamic Library Loader. \ No newline at end of file +See [Function Plugin](https://prestodb.io/docs/current/presto-cpp/plugin.html) on how to use the Dynamic Library Loader. diff --git a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt index e10ff2d75d264..e1bf73c2c02bd 100644 --- a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt @@ -11,19 +11,19 @@ # limitations under the License. add_library(presto_example_function_dynamic SHARED DynamicFunction.cpp) -add_library(presto_example_varchar_function_dynamic SHARED - DynamicVarcharFunction.cpp) -add_library(presto_example_non_default_function_dynamic SHARED - DynamicNonDefaultFunction.cpp) +add_library(presto_example_varchar_function_dynamic SHARED DynamicVarcharFunction.cpp) +add_library(presto_example_non_default_function_dynamic SHARED DynamicNonDefaultFunction.cpp) -set(CMAKE_DYLIB_TEST_LINK_LIBRARIES presto_dynamic_function_registrar fmt::fmt - gflags::gflags xsimd) -target_link_libraries(presto_example_function_dynamic - PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES}) -target_link_libraries(presto_example_varchar_function_dynamic - PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES}) -target_link_libraries(presto_example_non_default_function_dynamic - PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES}) +set(CMAKE_DYLIB_TEST_LINK_LIBRARIES presto_dynamic_function_registrar fmt::fmt gflags::gflags xsimd) +target_link_libraries(presto_example_function_dynamic PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES}) +target_link_libraries( + presto_example_varchar_function_dynamic + PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES} +) +target_link_libraries( + presto_example_non_default_function_dynamic + PRIVATE ${CMAKE_DYLIB_TEST_LINK_LIBRARIES} +) if(APPLE) set(COMMON_LIBRARY_LINK_OPTIONS "-Wl,-undefined,dynamic_lookup") @@ -31,9 +31,9 @@ else() set(COMMON_LIBRARY_LINK_OPTIONS "-Wl,--exclude-libs,ALL") endif() -target_link_options(presto_example_function_dynamic PRIVATE - ${COMMON_LIBRARY_LINK_OPTIONS}) -target_link_options(presto_example_varchar_function_dynamic PRIVATE - ${COMMON_LIBRARY_LINK_OPTIONS}) -target_link_options(presto_example_non_default_function_dynamic PRIVATE - ${COMMON_LIBRARY_LINK_OPTIONS}) +target_link_options(presto_example_function_dynamic PRIVATE ${COMMON_LIBRARY_LINK_OPTIONS}) +target_link_options(presto_example_varchar_function_dynamic PRIVATE ${COMMON_LIBRARY_LINK_OPTIONS}) +target_link_options( + presto_example_non_default_function_dynamic + PRIVATE ${COMMON_LIBRARY_LINK_OPTIONS} +) diff --git a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp index 8ae53d6406194..730b84694c2b6 100644 --- a/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp +++ b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp @@ -1,6 +1,4 @@ /* - * Copyright (c) Facebook, Inc. and its affiliates. - * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt index cbe76a3312c86..03e94007ff1a1 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt @@ -15,7 +15,8 @@ add_executable(presto_function_metadata_test FunctionMetadataTest.cpp) add_test( NAME presto_function_metadata_test COMMAND presto_function_metadata_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) target_link_libraries( presto_function_metadata_test @@ -26,4 +27,5 @@ target_link_libraries( velox_functions_prestosql velox_window GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) diff --git a/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp b/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp index d1319d456fdb6..226133c9e437c 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp +++ b/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp @@ -46,7 +46,10 @@ class FunctionMetadataTest : public ::testing::Test { size_t expectedSize) { json metadataList = functionMetadata_.at(name); EXPECT_EQ(metadataList.size(), expectedSize); - std::string expectedStr = slurp(test::utils::getDataPath("/github/presto-trunk/presto-native-execution/presto_cpp/main/functions/tests/data/", expectedFile)); + std::string expectedStr = slurp( + test::utils::getDataPath( + "/github/presto-trunk/presto-native-execution/presto_cpp/main/functions/tests/data/", + expectedFile)); auto expected = json::parse(expectedStr); auto comparator = [](const json& a, const json& b) { return (a["outputType"] < b["outputType"]); diff --git a/presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json index b19a32a35712c..8952199ae5059 100644 --- a/presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json +++ b/presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json @@ -178,7 +178,7 @@ "schema":"default", "typeVariableConstraints":[], "variableArity":true - }, + }, { "docString":"presto.default.greatest", "functionKind":"SCALAR", @@ -195,4 +195,4 @@ "variableArity":true } ] -} \ No newline at end of file +} diff --git a/presto-native-execution/presto_cpp/main/http/CMakeLists.txt b/presto-native-execution/presto_cpp/main/http/CMakeLists.txt index c60ddefac4393..0caf4fc0733a8 100644 --- a/presto-native-execution/presto_cpp/main/http/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/http/CMakeLists.txt @@ -13,8 +13,7 @@ add_library(presto_http HttpClient.cpp HttpServer.cpp) if(PRESTO_ENABLE_JWT) add_compile_definitions(JWT_DISABLE_PICOJSON) - target_include_directories( - presto_http PRIVATE ${CMAKE_SOURCE_DIR}/presto_cpp/external/json) + target_include_directories(presto_http PRIVATE ${CMAKE_SOURCE_DIR}/presto_cpp/external/json) endif() add_subdirectory(filters) @@ -33,7 +32,8 @@ target_link_libraries( ${GLOG} ${GFLAGS_LIBRARIES} ${FMT} - ${RE2}) + ${RE2} +) set_property(TARGET presto_http PROPERTY JOB_POOL_LINK presto_link_job_pool) diff --git a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp index e80c9b7909381..60c60b29742f7 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp +++ b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp @@ -165,8 +165,9 @@ HttpResponse::nextAllocationSize(uint64_t dataLength) const { minAllocSize, std::min( maxResponseAllocBytes_, - velox::bits::nextPowerOfTwo(velox::bits::roundUp( - dataLength + bodyChainBytes_, minResponseAllocBytes_)))); + velox::bits::nextPowerOfTwo( + velox::bits::roundUp( + dataLength + bodyChainBytes_, minResponseAllocBytes_)))); } std::string HttpResponse::dumpBodyChain() const { @@ -557,8 +558,10 @@ void RequestBuilder::addJwtIfConfigured() { auto secretHash = std::vector(SHA256_DIGEST_LENGTH); folly::ssl::OpenSSLHash::sha256( folly::range(secretHash), - folly::ByteRange(folly::StringPiece( - SystemConfig::instance()->internalCommunicationSharedSecret()))); + folly::ByteRange( + folly::StringPiece( + SystemConfig::instance() + ->internalCommunicationSharedSecret()))); const auto time = std::chrono::system_clock::now(); const auto token = @@ -570,9 +573,10 @@ void RequestBuilder::addJwtIfConfigured() { std::chrono::seconds{ SystemConfig::instance() ->internalCommunicationJwtExpirationSeconds()}) - .sign(jwt::algorithm::hs256{std::string( - reinterpret_cast(secretHash.data()), - secretHash.size())}); + .sign( + jwt::algorithm::hs256{std::string( + reinterpret_cast(secretHash.data()), + secretHash.size())}); header(kPrestoInternalBearer, token); } #endif // PRESTO_ENABLE_JWT diff --git a/presto-native-execution/presto_cpp/main/http/HttpServer.cpp b/presto-native-execution/presto_cpp/main/http/HttpServer.cpp index 45a788122790a..4c3bba438ce47 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpServer.cpp +++ b/presto-native-execution/presto_cpp/main/http/HttpServer.cpp @@ -132,8 +132,8 @@ proxygen::HTTPServer::IPConfig HttpsConfig::ipConfig() const { sslCfg.sslCiphers = supportedCiphers_; if (!clientCaFile_.empty()) { sslCfg.clientCAFiles = {clientCaFile_}; - sslCfg.clientVerification = - folly::SSLContext::VerifyClientCertificate::ALWAYS; + sslCfg.clientVerification = + folly::SSLContext::VerifyClientCertificate::ALWAYS; } if (http2Enabled_) { diff --git a/presto-native-execution/presto_cpp/main/http/filters/AccessLogFilter.cpp b/presto-native-execution/presto_cpp/main/http/filters/AccessLogFilter.cpp index 1ac4372ae33a4..5ad2173e33de4 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/AccessLogFilter.cpp +++ b/presto-native-execution/presto_cpp/main/http/filters/AccessLogFilter.cpp @@ -12,8 +12,8 @@ * limitations under the License. */ -#include #include "presto_cpp/main/http/filters/AccessLogFilter.h" +#include namespace facebook::presto::http::filters { diff --git a/presto-native-execution/presto_cpp/main/http/filters/CMakeLists.txt b/presto-native-execution/presto_cpp/main/http/filters/CMakeLists.txt index d0f97d0321c47..832dd0b540499 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/http/filters/CMakeLists.txt @@ -10,12 +10,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_library(http_filters AccessLogFilter.cpp HttpEndpointLatencyFilter.cpp - InternalAuthenticationFilter.cpp StatsFilter.cpp) +add_library( + http_filters + AccessLogFilter.cpp + HttpEndpointLatencyFilter.cpp + InternalAuthenticationFilter.cpp + StatsFilter.cpp +) if(PRESTO_ENABLE_JWT) - target_include_directories( - http_filters PRIVATE ${CMAKE_SOURCE_DIR}/presto_cpp/external/json) + target_include_directories(http_filters PRIVATE ${CMAKE_SOURCE_DIR}/presto_cpp/external/json) endif() target_link_libraries(http_filters presto_common ${PROXYGEN_LIBRARIES}) diff --git a/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.cpp b/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.cpp index 050b14fc6dbcd..85c5ac3c96779 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.cpp +++ b/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.cpp @@ -28,20 +28,21 @@ HttpEndpointLatencyFilter::HttpEndpointLatencyFilter( void HttpEndpointLatencyFilter::updateLatency( const std::string& endpoint, uint64_t latencyUs) { - metricMap().withWLock([&](std::unordered_map& - map) { - auto itr = map.find(endpoint); - if (itr != map.end()) { - auto& metrics = itr->second; - metrics.maxLatencyUs = std::max(metrics.maxLatencyUs, latencyUs); - metrics.avgLatencyUs = - (metrics.avgLatencyUs * metrics.count + latencyUs) / - (metrics.count + 1); - ++metrics.count; - } else { - map.emplace(endpoint, EndPointMetrics{endpoint, latencyUs, latencyUs, 1}); - } - }); + metricMap().withWLock( + [&](std::unordered_map& map) { + auto itr = map.find(endpoint); + if (itr != map.end()) { + auto& metrics = itr->second; + metrics.maxLatencyUs = std::max(metrics.maxLatencyUs, latencyUs); + metrics.avgLatencyUs = + (metrics.avgLatencyUs * metrics.count + latencyUs) / + (metrics.count + 1); + ++metrics.count; + } else { + map.emplace( + endpoint, EndPointMetrics{endpoint, latencyUs, latencyUs, 1}); + } + }); } // static diff --git a/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.h b/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.h index 612bd9f027cda..427f3bee92993 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.h +++ b/presto-native-execution/presto_cpp/main/http/filters/HttpEndpointLatencyFilter.h @@ -92,10 +92,11 @@ class HttpEndpointLatencyFilterFactory : public proxygen::RequestHandlerFactory { public: explicit HttpEndpointLatencyFilterFactory(http::HttpServer* httpServer) - : endpoints_(std::make_shared>>>( - httpServer->endpoints())) {} + : endpoints_( + std::make_shared>>>( + httpServer->endpoints())) {} void onServerStart(folly::EventBase* /*evb*/) noexcept override {} diff --git a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp index 860bcd8d78976..bea48739e0af1 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp +++ b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp @@ -134,8 +134,10 @@ void InternalAuthenticationFilter::processAndVerifyJwt( auto secretHash = std::vector(SHA256_DIGEST_LENGTH); folly::ssl::OpenSSLHash::sha256( folly::range(secretHash), - folly::ByteRange(folly::StringPiece( - SystemConfig::instance()->internalCommunicationSharedSecret()))); + folly::ByteRange( + folly::StringPiece( + SystemConfig::instance() + ->internalCommunicationSharedSecret()))); // Decode and verify the JWT. auto decodedJwt = jwt::decode(token); diff --git a/presto-native-execution/presto_cpp/main/http/filters/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/http/filters/tests/CMakeLists.txt index 386d48f4aa982..de06c01d63e00 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/http/filters/tests/CMakeLists.txt @@ -15,7 +15,13 @@ add_executable(presto_http_filter_test HttpFilterTest.cpp) add_test( NAME presto_http_filter_test COMMAND presto_http_filter_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) -target_link_libraries(presto_http_filter_test http_filters presto_http - GTest::gtest GTest::gtest_main) +target_link_libraries( + presto_http_filter_test + http_filters + presto_http + GTest::gtest + GTest::gtest_main +) diff --git a/presto-native-execution/presto_cpp/main/http/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/http/tests/CMakeLists.txt index 3e58e1753b7e2..241cf46ca1a49 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/http/tests/CMakeLists.txt @@ -14,13 +14,12 @@ add_executable(presto_http_test HttpTest.cpp) add_test( NAME presto_http_test COMMAND presto_http_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) -target_link_libraries(presto_http_test presto_http GTest::gtest - GTest::gtest_main) +target_link_libraries(presto_http_test presto_http GTest::gtest GTest::gtest_main) -set_property(TARGET presto_http_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_http_test PROPERTY JOB_POOL_LINK presto_link_job_pool) if(PRESTO_ENABLE_JWT) add_executable(presto_http_jwt_test HttpJwtTest.cpp) @@ -28,8 +27,8 @@ if(PRESTO_ENABLE_JWT) add_test( NAME presto_http_jwt_test COMMAND presto_http_jwt_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) - target_link_libraries(presto_http_jwt_test presto_http GTest::gtest - GTest::gtest_main) + target_link_libraries(presto_http_jwt_test presto_http GTest::gtest GTest::gtest_main) endif() diff --git a/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h b/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h index 25ef2a7cb7f17..f67a2ff0451c5 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h +++ b/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h @@ -165,8 +165,9 @@ void echo( if (body.empty()) { proxygen::ResponseBuilder(downstream) .status(facebook::presto::http::kHttpOk, "") - .body(folly::IOBuf::wrapBuffer( - message->getURL().c_str(), message->getURL().size())) + .body( + folly::IOBuf::wrapBuffer( + message->getURL().c_str(), message->getURL().size())) .sendWithEOM(); return; } diff --git a/presto-native-execution/presto_cpp/main/http/tests/certs/client_ca.pem b/presto-native-execution/presto_cpp/main/http/tests/certs/client_ca.pem index 5b982399a1c0a..87c78fee2dd56 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/certs/client_ca.pem +++ b/presto-native-execution/presto_cpp/main/http/tests/certs/client_ca.pem @@ -43,4 +43,4 @@ oJVHuDzW6KxQh4tV8Fg5hhbwJ7Y4AP99AY/WXYZKqjSu7lv9LO4ie4zBEgJol6Bs u4shAoGBAKbPrKUobbhsihDZsS8hjDQ9g0eMagxOJq57WqTEzAqih0k1YQQhjhHL fiANKrBbAFFOxdn1JfJNBdBXSEeVdJ+AMy7wj0K6dWGNP3A4FKqs5g5zb+grdPEJ l8qRFT+gODcZT3iSokEUuPnG0y07sM7CUS5rc94NqV/nE1dgQenA ------END RSA PRIVATE KEY----- \ No newline at end of file +-----END RSA PRIVATE KEY----- diff --git a/presto-native-execution/presto_cpp/main/http/tests/certs/test_cert1.pem b/presto-native-execution/presto_cpp/main/http/tests/certs/test_cert1.pem index ee5551d60cd60..db38566a56633 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/certs/test_cert1.pem +++ b/presto-native-execution/presto_cpp/main/http/tests/certs/test_cert1.pem @@ -16,4 +16,4 @@ LDv5K7msbk4RvrWlM2rVVx41e03c+VxdUF6sBVgYANbE0uQ6ULYVWIPRz8lItWKl a19JzTyU8NQeOfXDlcxPg7KgdRGyTcbGlzOFr30ZQCK3OoL351XhRr/jlHQdScVA JCusLYHJHpBBFs4ZT7sbkBSk7W80oUpCleklLjPTAPwAfWVYNBCsVWba++mvmmWI QYrc4pXNR3oEDUZz02Zo28rvXJgXj3WHg1TBOf6y42KnHH9i1ejre7fomg== ------END CERTIFICATE----- \ No newline at end of file +-----END CERTIFICATE----- diff --git a/presto-native-execution/presto_cpp/main/http/tests/certs/test_key1.pem b/presto-native-execution/presto_cpp/main/http/tests/certs/test_key1.pem index 5819b04258c32..18d9c01cb262a 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/certs/test_key1.pem +++ b/presto-native-execution/presto_cpp/main/http/tests/certs/test_key1.pem @@ -24,4 +24,4 @@ oJVHuDzW6KxQh4tV8Fg5hhbwJ7Y4AP99AY/WXYZKqjSu7lv9LO4ie4zBEgJol6Bs u4shAoGBAKbPrKUobbhsihDZsS8hjDQ9g0eMagxOJq57WqTEzAqih0k1YQQhjhHL fiANKrBbAFFOxdn1JfJNBdBXSEeVdJ+AMy7wj0K6dWGNP3A4FKqs5g5zb+grdPEJ l8qRFT+gODcZT3iSokEUuPnG0y07sM7CUS5rc94NqV/nE1dgQenA ------END RSA PRIVATE KEY----- \ No newline at end of file +-----END RSA PRIVATE KEY----- diff --git a/presto-native-execution/presto_cpp/main/operators/BroadcastFactory.cpp b/presto-native-execution/presto_cpp/main/operators/BroadcastFactory.cpp index 15f97d2a1997a..08da15cf55660 100644 --- a/presto-native-execution/presto_cpp/main/operators/BroadcastFactory.cpp +++ b/presto-native-execution/presto_cpp/main/operators/BroadcastFactory.cpp @@ -112,11 +112,12 @@ void BroadcastFileWriter::updateWriteStats( uint64_t /* fileWriteTimeNs */) { writtenBytes_ += writtenBytes; if (FOLLY_UNLIKELY(writtenBytes_ > maxBroadcastBytes_)) { - PRESTO_BROADCAST_LIMIT_EXCEEDED(fmt::format( - "Storage broadcast join exceeded per task broadcast limit " - "writtenBytes_ {} vs maxBroadcastBytes_ {}", - succinctBytes(writtenBytes_), - succinctBytes(maxBroadcastBytes_))); + PRESTO_BROADCAST_LIMIT_EXCEEDED( + fmt::format( + "Storage broadcast join exceeded per task broadcast limit " + "writtenBytes_ {} vs maxBroadcastBytes_ {}", + succinctBytes(writtenBytes_), + succinctBytes(maxBroadcastBytes_))); } } diff --git a/presto-native-execution/presto_cpp/main/operators/CMakeLists.txt b/presto-native-execution/presto_cpp/main/operators/CMakeLists.txt index 6f1be6b01bca4..b14e7ff0cd3e3 100644 --- a/presto-native-execution/presto_cpp/main/operators/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/operators/CMakeLists.txt @@ -21,7 +21,8 @@ add_library( BroadcastWrite.cpp BroadcastFactory.cpp BroadcastExchangeSource.cpp - BinarySortableSerializer.cpp) + BinarySortableSerializer.cpp +) target_link_libraries( presto_operators @@ -31,7 +32,8 @@ target_link_libraries( velox_exec velox_presto_serializer velox_vector - velox_row_fast) + velox_row_fast +) if(PRESTO_ENABLE_TESTING) add_subdirectory(tests) diff --git a/presto-native-execution/presto_cpp/main/operators/ShuffleExchangeSource.cpp b/presto-native-execution/presto_cpp/main/operators/ShuffleExchangeSource.cpp index e3246f875f729..f11b5c0c646d5 100644 --- a/presto-native-execution/presto_cpp/main/operators/ShuffleExchangeSource.cpp +++ b/presto-native-execution/presto_cpp/main/operators/ShuffleExchangeSource.cpp @@ -46,11 +46,11 @@ ShuffleExchangeSource::request( } else { for (auto& batch : batches) { totalBytes = batch->data->size(); - VELOX_CHECK_LE(totalBytes, std::numeric_limits::max()); - ++numBatches_; - queue_->enqueueLocked( - std::make_unique(std::move(batch)), - promises); + VELOX_CHECK_LE(totalBytes, std::numeric_limits::max()); + ++numBatches_; + queue_->enqueueLocked( + std::make_unique(std::move(batch)), + promises); } } } @@ -62,7 +62,7 @@ ShuffleExchangeSource::request( }) .deferError( [](folly::exception_wrapper e) mutable - -> ShuffleExchangeSource::Response { + -> ShuffleExchangeSource::Response { VELOX_FAIL("ShuffleReader::{} failed: {}", "next", e.what()); }); }; diff --git a/presto-native-execution/presto_cpp/main/operators/benchmarks/BinarySortableSerializerBenchmark.cpp b/presto-native-execution/presto_cpp/main/operators/benchmarks/BinarySortableSerializerBenchmark.cpp index 1362c0c35dba7..b8f789246af1b 100644 --- a/presto-native-execution/presto_cpp/main/operators/benchmarks/BinarySortableSerializerBenchmark.cpp +++ b/presto-native-execution/presto_cpp/main/operators/benchmarks/BinarySortableSerializerBenchmark.cpp @@ -1,6 +1,4 @@ /* - * Copyright (c) Facebook, Inc. and its affiliates. - * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at diff --git a/presto-native-execution/presto_cpp/main/operators/benchmarks/CMakeLists.txt b/presto-native-execution/presto_cpp/main/operators/benchmarks/CMakeLists.txt index 4179fec55c123..ec8ba69c34051 100644 --- a/presto-native-execution/presto_cpp/main/operators/benchmarks/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/operators/benchmarks/CMakeLists.txt @@ -12,12 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(binary_sortable_serializer_benchmark - BinarySortableSerializerBenchmark.cpp) +add_executable(binary_sortable_serializer_benchmark BinarySortableSerializerBenchmark.cpp) target_link_libraries( binary_sortable_serializer_benchmark - PRIVATE - presto_operators - velox_vector_fuzzer - Folly::folly - Folly::follybenchmark) + PRIVATE presto_operators velox_vector_fuzzer Folly::folly Folly::follybenchmark +) diff --git a/presto-native-execution/presto_cpp/main/operators/tests/BinarySortableSerializerTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/BinarySortableSerializerTest.cpp index 77481b697befb..8982af1e1c875 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/BinarySortableSerializerTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/BinarySortableSerializerTest.cpp @@ -87,8 +87,9 @@ class BinarySortableSerializerTest : public ::testing::Test, std::vector> fields; - fields.push_back(std::make_shared( - velox::CppToType::create(), /*name=*/"c0")); + fields.push_back( + std::make_shared( + velox::CppToType::create(), /*name=*/"c0")); return compareRowVector(rowVector, fields, {ordering}); } @@ -103,8 +104,9 @@ class BinarySortableSerializerTest : public ::testing::Test, std::vector> fields; - fields.push_back(std::make_shared( - velox::CppToType::create(), /*name=*/"c0")); + fields.push_back( + std::make_shared( + velox::CppToType::create(), /*name=*/"c0")); return compareRowVector(rowVector, fields, {ordering}); } @@ -128,8 +130,9 @@ class BinarySortableSerializerTest : public ::testing::Test, } std::vector> fields; - fields.push_back(std::make_shared( - rowField->type(), /*name=*/"c0")); + fields.push_back( + std::make_shared( + rowField->type(), /*name=*/"c0")); return compareRowVector(rowVector, fields, {ordering}); } @@ -192,7 +195,8 @@ class BinarySortableSerializerFuzzerTest : public ::testing::Test, const auto rowVector = makeData(rowType); const auto fields = getFields(rowType); - for (velox::vector_size_t offset = 0; offset + batchSize < rowVector->size(); + for (velox::vector_size_t offset = 0; + offset + batchSize < rowVector->size(); offset += batchSize) { checkBatchedSizeCalculation( rowVector, offset, batchSize, fields, testOrdering); @@ -405,8 +409,9 @@ TEST_F(BinarySortableSerializerTest, LongTypeAllFields) { std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -428,8 +433,9 @@ TEST_F(BinarySortableSerializerTest, LongTypeAllFieldsDescending) { std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -453,8 +459,9 @@ TEST_F(BinarySortableSerializerTest, LongTypeAllFieldsWithNulls) { std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -478,8 +485,9 @@ TEST_F(BinarySortableSerializerTest, DefaultNullsOrdering) { std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -505,8 +513,9 @@ TEST_F( std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -532,8 +541,9 @@ TEST_F( std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -553,10 +563,12 @@ TEST_F(BinarySortableSerializerTest, LongTypeSubsetOfFields) { auto c2 = vectorMaker_.flatVectorNullable({2, 0}); std::vector> fields; - fields.push_back(std::make_shared( - velox::BIGINT(), /*name=*/"c2")); - fields.push_back(std::make_shared( - velox::BIGINT(), /*name=*/"c0")); + fields.push_back( + std::make_shared( + velox::BIGINT(), /*name=*/"c2")); + fields.push_back( + std::make_shared( + velox::BIGINT(), /*name=*/"c0")); auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -656,8 +668,9 @@ TEST_F(BinarySortableSerializerTest, ConstantVector) { std::vector> fields; fields.reserve(3); for (int32_t i = 0; i < 3; ++i) { - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", i))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", i))); } auto rowVector = vectorMaker_.rowVector({c0, c1, c2}); @@ -1045,7 +1058,8 @@ TEST_F(BinarySortableSerializerTest, ArrayTypeSingleFieldTests) { // null < [] EXPECT_TRUE( singleArrayFieldCompare( - {std::nullopt, {std::vector>{}}}, velox::core::kAscNullsFirst) < 0); + {std::nullopt, {std::vector>{}}}, + velox::core::kAscNullsFirst) < 0); } TEST_F(BinarySortableSerializerTest, RowTypeSingleFieldTests) { diff --git a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp index 2c5da4ffd06c7..ee8426e406d20 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp @@ -172,8 +172,9 @@ class BroadcastTest : public exec::test::OperatorTestBase, for (const auto& name : newLayout.value()) { columns.push_back(vector->childAt(rowType->getChildIdx(name))); } - reordered.push_back(std::make_shared( - pool(), newRowType, nullptr /*nulls*/, vector->size(), columns)); + reordered.push_back( + std::make_shared( + pool(), newRowType, nullptr /*nulls*/, vector->size(), columns)); } return reordered; } diff --git a/presto-native-execution/presto_cpp/main/operators/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/operators/tests/CMakeLists.txt index 5a082aa4c59a1..7a66574a655fd 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/operators/tests/CMakeLists.txt @@ -15,8 +15,12 @@ target_link_libraries(presto_operators_plan_builder velox_core) add_executable( presto_operators_test - PlanNodeSerdeTest.cpp ShuffleTest.cpp BroadcastTest.cpp - BinarySortableSerializerTest.cpp PlanNodeBuilderTest.cpp) + PlanNodeSerdeTest.cpp + ShuffleTest.cpp + BroadcastTest.cpp + BinarySortableSerializerTest.cpp + PlanNodeBuilderTest.cpp +) add_test(presto_operators_test presto_operators_test) @@ -39,7 +43,7 @@ target_link_libraries( velox_exec GTest::gmock GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) -set_property(TARGET presto_operators_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_operators_test PROPERTY JOB_POOL_LINK presto_link_job_pool) diff --git a/presto-native-execution/presto_cpp/main/operators/tests/ShuffleTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/ShuffleTest.cpp index b7ac7c266a480..b04b940ec6c3e 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/ShuffleTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/ShuffleTest.cpp @@ -723,12 +723,13 @@ class ShuffleTest : public exec::test::OperatorTestBase { for (int it = 0; it < numIterations; it++) { auto seed = folly::Random::rand32(); - SCOPED_TRACE(fmt::format( - "Iteration {}, numPartitions {}, replicateNullsAndAny {}, seed {}", - it, - numPartitions, - replicateNullsAndAny, - seed)); + SCOPED_TRACE( + fmt::format( + "Iteration {}, numPartitions {}, replicateNullsAndAny {}, seed {}", + it, + numPartitions, + replicateNullsAndAny, + seed)); VectorFuzzer fuzzer(opts, pool_.get(), seed); std::vector inputVectors; @@ -973,8 +974,9 @@ TEST_F(ShuffleTest, endToEndWithSortedShuffle) { auto ordering = {velox::core::SortOrder(velox::core::kAscNullsFirst)}; std::vector> fields; - fields.push_back(std::make_shared( - velox::BIGINT(), fmt::format("c{}", 1))); + fields.push_back( + std::make_shared( + velox::BIGINT(), fmt::format("c{}", 1))); // Make sure all previously registered exchange factory are gone. velox::exec::ExchangeSource::factories().clear(); @@ -1021,8 +1023,9 @@ TEST_F(ShuffleTest, endToEndWithSortedShuffleRowLimit) { auto ordering = {velox::core::SortOrder(velox::core::kAscNullsFirst)}; std::vector> fields; - fields.push_back(std::make_shared( - velox::VARCHAR(), fmt::format("c{}", 1))); + fields.push_back( + std::make_shared( + velox::VARCHAR(), fmt::format("c{}", 1))); // Make sure all previously registered exchange factory are gone. velox::exec::ExchangeSource::factories().clear(); @@ -1382,11 +1385,13 @@ class DummyShuffleInterfaceFactory : public ShuffleInterfaceFactory { TEST_F(ShuffleTest, shuffleInterfaceRegistration) { const std::string kShuffleName = "dummy-shuffle"; - EXPECT_TRUE(ShuffleInterfaceFactory::registerFactory( - kShuffleName, std::make_unique())); + EXPECT_TRUE( + ShuffleInterfaceFactory::registerFactory( + kShuffleName, std::make_unique())); EXPECT_NO_THROW(ShuffleInterfaceFactory::factory(kShuffleName)); - EXPECT_FALSE(ShuffleInterfaceFactory::registerFactory( - kShuffleName, std::make_unique())); + EXPECT_FALSE( + ShuffleInterfaceFactory::registerFactory( + kShuffleName, std::make_unique())); } TEST_F(ShuffleTest, shuffleReadRuntimeStats) { @@ -1471,16 +1476,14 @@ TEST_F(ShuffleTest, shuffleReadRuntimeStats) { ASSERT_GT(decodeTimeStat.sum, 0); ASSERT_EQ(velox::RuntimeCounter::Unit::kNanos, decodeTimeStat.unit); - ASSERT_EQ( - runtimeStats.count("shuffleNumBatchesPerRead"), 1); + ASSERT_EQ(runtimeStats.count("shuffleNumBatchesPerRead"), 1); const auto& batchesPerReadStat = runtimeStats.at("shuffleNumBatchesPerRead"); ASSERT_EQ(velox::RuntimeCounter::Unit::kNone, batchesPerReadStat.unit); ASSERT_GT(batchesPerReadStat.count, 0); ASSERT_GT(batchesPerReadStat.sum, 0); - ASSERT_EQ( - runtimeStats.count("shuffleNumBatches"), 1); + ASSERT_EQ(runtimeStats.count("shuffleNumBatches"), 1); const auto& numBatchesStat = runtimeStats.at("shuffleNumBatches"); ASSERT_GT(numBatchesStat.count, 0); ASSERT_GT(numBatchesStat.sum, 0); diff --git a/presto-native-execution/presto_cpp/main/runtime-metrics/CMakeLists.txt b/presto-native-execution/presto_cpp/main/runtime-metrics/CMakeLists.txt index b41b7ab78d8d1..f2c2512c4053d 100644 --- a/presto-native-execution/presto_cpp/main/runtime-metrics/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/runtime-metrics/CMakeLists.txt @@ -17,8 +17,7 @@ find_package(prometheus-cpp CONFIG REQUIRED) # Prepare a static library to link with prometheus_reporter_test add_library(presto_prometheus_reporter OBJECT PrometheusStatsReporter.cpp) -target_link_libraries(presto_prometheus_reporter presto_common - prometheus-cpp::core) +target_link_libraries(presto_prometheus_reporter presto_common prometheus-cpp::core) if(PRESTO_ENABLE_TESTING) add_subdirectory(tests) diff --git a/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.cpp b/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.cpp index ac4464aab00d0..16ab6a660c8bf 100644 --- a/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.cpp +++ b/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.cpp @@ -47,10 +47,10 @@ struct PrometheusStatsReporter::PrometheusImpl { }; PrometheusStatsReporter::PrometheusStatsReporter( - const std::map& labels, int numThreads) + const std::map& labels, + int numThreads) : executor_(std::make_shared(numThreads)), - impl_(std::make_shared(labels)) { -} + impl_(std::make_shared(labels)) {} void PrometheusStatsReporter::registerMetricExportType( const char* key, @@ -133,8 +133,9 @@ void PrometheusStatsReporter::registerHistogramMetricExportType( .Register(*impl_->registry); ::prometheus::Summary::Quantiles quantiles; for (auto pct : pcts) { - quantiles.push_back(::prometheus::detail::CKMSQuantiles::Quantile( - pct / static_cast(100), 0)); + quantiles.push_back( + ::prometheus::detail::CKMSQuantiles::Quantile( + pct / static_cast(100), 0)); } auto& summaryMetric = summaryFamily.Add({impl_->labels}, quantiles); registeredMetricsMap_.insert( @@ -176,14 +177,16 @@ void PrometheusStatsReporter::addMetricValue(const char* key, size_t value) break; } case velox::StatType::SUM: { - auto* gauge = reinterpret_cast<::prometheus::Gauge*>(statsInfo.metricPtr); + auto* gauge = + reinterpret_cast<::prometheus::Gauge*>(statsInfo.metricPtr); gauge->Increment(static_cast(value)); break; } case velox::StatType::AVG: case velox::StatType::RATE: { // Overrides the existing state. - auto* gauge = reinterpret_cast<::prometheus::Gauge*>(statsInfo.metricPtr); + auto* gauge = + reinterpret_cast<::prometheus::Gauge*>(statsInfo.metricPtr); gauge->Set(static_cast(value)); break; } diff --git a/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.h b/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.h index c5764d12581f6..8e1cd4513a354 100644 --- a/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.h +++ b/presto-native-execution/presto_cpp/main/runtime-metrics/PrometheusStatsReporter.h @@ -12,14 +12,13 @@ * limitations under the License. */ -#include #include +#include #include "presto_cpp/main/common/Configs.h" #include "velox/common/base/Exceptions.h" #include "velox/common/base/GTestMacros.h" #include "velox/common/base/StatsReporter.h" - namespace facebook::presto::prometheus { struct StatsInfo { @@ -41,9 +40,9 @@ class PrometheusStatsReporter : public facebook::velox::BaseStatsReporter { class PrometheusImpl; public: - explicit PrometheusStatsReporter( - const std::map& labels, int numThreads); + const std::map& labels, + int numThreads); void registerMetricExportType(const char* key, velox::StatType) const override; @@ -76,13 +75,13 @@ class PrometheusStatsReporter : public facebook::velox::BaseStatsReporter { const std::vector& /* statTypes */, const std::vector& /* pcts */, const std::vector& /* slidingWindowsSeconds */) const override {}; - + void registerDynamicQuantileMetricExportType( const char* /* keyPattern */, const std::vector& /* statTypes */, const std::vector& /* pcts */, const std::vector& /* slidingWindowsSeconds */) const override {}; - + void registerDynamicQuantileMetricExportType( folly::StringPiece /* keyPattern */, const std::vector& /* statTypes */, @@ -110,13 +109,13 @@ class PrometheusStatsReporter : public facebook::velox::BaseStatsReporter { const override {}; void addQuantileMetricValue(folly::StringPiece /* key */, size_t /* value */) - const override{}; - + const override {}; + void addDynamicQuantileMetricValue( const std::string& /* key */, folly::Range /* subkeys */, size_t /* value */) const override {}; - + virtual void addDynamicQuantileMetricValue( const char* /* key */, folly::Range /* subkeys */, @@ -142,7 +141,8 @@ class PrometheusStatsReporter : public facebook::velox::BaseStatsReporter { const std::string worker = !hostName ? "" : hostName; std::map labels{ {"cluster", cluster}, {"worker", worker}}; - return std::make_unique(labels, nodeConfig->prometheusExecutorThreads()); + return std::make_unique( + labels, nodeConfig->prometheusExecutorThreads()); } private: @@ -150,7 +150,8 @@ class PrometheusStatsReporter : public facebook::velox::BaseStatsReporter { std::shared_ptr impl_; // A map of labels assigned to each metric which helps in filtering at client // end. - mutable folly::ConcurrentHashMap registeredMetricsMap_; + mutable folly::ConcurrentHashMap + registeredMetricsMap_; VELOX_FRIEND_TEST(PrometheusReporterTest, testCountAndGauge); VELOX_FRIEND_TEST(PrometheusReporterTest, testHistogramSummary); VELOX_FRIEND_TEST(PrometheusReporterTest, testConcurrentReporting); diff --git a/presto-native-execution/presto_cpp/main/runtime-metrics/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/runtime-metrics/tests/CMakeLists.txt index f99c6419fb0fa..be7c1bca980e4 100644 --- a/presto-native-execution/presto_cpp/main/runtime-metrics/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/runtime-metrics/tests/CMakeLists.txt @@ -14,6 +14,12 @@ add_executable(prometheus_reporter_test PrometheusReporterTest.cpp) add_test( NAME prometheus_reporter_test COMMAND prometheus_reporter_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(prometheus_reporter_test velox_exec_test_lib - presto_prometheus_reporter GTest::gtest GTest::gtest_main) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) +target_link_libraries( + prometheus_reporter_test + velox_exec_test_lib + presto_prometheus_reporter + GTest::gtest + GTest::gtest_main +) diff --git a/presto-native-execution/presto_cpp/main/runtime-metrics/tests/PrometheusReporterTest.cpp b/presto-native-execution/presto_cpp/main/runtime-metrics/tests/PrometheusReporterTest.cpp index bca47b348c29d..9667d3725b19f 100644 --- a/presto-native-execution/presto_cpp/main/runtime-metrics/tests/PrometheusReporterTest.cpp +++ b/presto-native-execution/presto_cpp/main/runtime-metrics/tests/PrometheusReporterTest.cpp @@ -21,7 +21,8 @@ class PrometheusReporterTest : public testing::Test { public: void SetUp() override { reporter = std::make_shared(testLabels, 1); - multiThreadedReporter = std::make_shared(testLabels, 2); + multiThreadedReporter = + std::make_shared(testLabels, 2); } void verifySerializedResult( @@ -51,10 +52,12 @@ TEST_F(PrometheusReporterTest, testConcurrentReporting) { "test.key3", facebook::velox::StatType::SUM); EXPECT_EQ( facebook::velox::StatType::COUNT, - multiThreadedReporter->registeredMetricsMap_.find("test.key1")->second.statType); + multiThreadedReporter->registeredMetricsMap_.find("test.key1") + ->second.statType); EXPECT_EQ( facebook::velox::StatType::SUM, - multiThreadedReporter->registeredMetricsMap_.find("test.key3")->second.statType); + multiThreadedReporter->registeredMetricsMap_.find("test.key3") + ->second.statType); std::vector testData = {10, 12, 14}; for (auto i : testData) { diff --git a/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt index 83069c4f581cc..459cbc9e623b9 100644 --- a/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt @@ -26,17 +26,21 @@ add_executable( QueryContextManagerTest.cpp TaskInfoTest.cpp TaskStatusTest.cpp - TaskUpdateRequestTest.cpp) + TaskUpdateRequestTest.cpp +) -if(DEFINED PRESTO_MEMORY_CHECKER_TYPE AND PRESTO_MEMORY_CHECKER_TYPE STREQUAL - "LINUX_MEMORY_CHECKER") +if( + DEFINED PRESTO_MEMORY_CHECKER_TYPE + AND PRESTO_MEMORY_CHECKER_TYPE STREQUAL "LINUX_MEMORY_CHECKER" +) target_sources(presto_server_test PRIVATE LinuxMemoryCheckerTest.cpp) endif() add_test( NAME presto_server_test COMMAND presto_server_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) target_link_libraries( presto_server_test @@ -56,25 +60,31 @@ target_link_libraries( ${RE2} GTest::gmock GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) # Clang requires explicit linking with libatomic. -if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" - AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15) +if( + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" + AND "${CMAKE_CXX_COMPILER_VERSION}" VERSION_GREATER_EQUAL 15 +) target_link_libraries(presto_server_test atomic) endif() -set_property(TARGET presto_server_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_server_test PROPERTY JOB_POOL_LINK presto_link_job_pool) if(PRESTO_ENABLE_REMOTE_FUNCTIONS) - add_executable(presto_server_remote_function_test - JsonSignatureParserTest.cpp RemoteFunctionRegistererTest.cpp) + add_executable( + presto_server_remote_function_test + JsonSignatureParserTest.cpp + RemoteFunctionRegistererTest.cpp + ) add_test( NAME presto_server_remote_function_test COMMAND presto_server_remote_function_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) target_link_libraries( presto_server_remote_function_test @@ -83,5 +93,6 @@ if(PRESTO_ENABLE_REMOTE_FUNCTIONS) velox_temp_path GTest::gmock GTest::gtest - GTest::gtest_main) + GTest::gtest_main + ) endif() diff --git a/presto-native-execution/presto_cpp/main/tests/ConnectorTest.cpp b/presto-native-execution/presto_cpp/main/tests/ConnectorTest.cpp index 9fe02f4313e31..973ca68fa8961 100644 --- a/presto-native-execution/presto_cpp/main/tests/ConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/ConnectorTest.cpp @@ -12,13 +12,13 @@ * limitations under the License. */ -#include "presto_cpp/main/connectors/Registration.h" +#include "velox/connectors/Connector.h" #include #include +#include "presto_cpp/main/connectors/Registration.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/config/Config.h" -#include "velox/connectors/Connector.h" using namespace facebook::presto; @@ -198,4 +198,3 @@ TEST_F(ConnectorTest, connectorFactorySingleton) { } } // namespace - diff --git a/presto-native-execution/presto_cpp/main/tests/JsonSignatureParserTest.cpp b/presto-native-execution/presto_cpp/main/tests/JsonSignatureParserTest.cpp index 2f7779309c3af..616c14cf091fa 100644 --- a/presto-native-execution/presto_cpp/main/tests/JsonSignatureParserTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/JsonSignatureParserTest.cpp @@ -64,9 +64,10 @@ TEST_F(JsonSignatureParserTest, broken) { "[{\"outputType\": \"varchar\", \"paramTypes\": [123]}]}}"), "Function type name should be a string."); - EXPECT_NO_THROW(JsonSignatureParser parser( - "{\"udfSignatureMap\": {\"func\": " - "[{\"outputType\": \"varchar\", \"paramTypes\": [\"varchar\"]}]}}")); + EXPECT_NO_THROW( + JsonSignatureParser parser( + "{\"udfSignatureMap\": {\"func\": " + "[{\"outputType\": \"varchar\", \"paramTypes\": [\"varchar\"]}]}}")); } TEST_F(JsonSignatureParserTest, simpleTypes) { diff --git a/presto-native-execution/presto_cpp/main/tests/LinuxMemoryCheckerTest.cpp b/presto-native-execution/presto_cpp/main/tests/LinuxMemoryCheckerTest.cpp index bd50fb50416c5..098e471094ce8 100644 --- a/presto-native-execution/presto_cpp/main/tests/LinuxMemoryCheckerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/LinuxMemoryCheckerTest.cpp @@ -119,22 +119,27 @@ TEST_F(LinuxMemoryCheckerTest, basic) { // Default config. ASSERT_NO_THROW(LinuxMemoryChecker(PeriodicMemoryChecker::Config{})); - ASSERT_NO_THROW(LinuxMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "/path/to/dir", "prefix", 5, 512})); + ASSERT_NO_THROW(LinuxMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "/path/to/dir", "prefix", 5, 512})); VELOX_ASSERT_THROW( - LinuxMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 0, 32, true, 5, "/path/to/dir", "prefix", 5, 512}), + LinuxMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 0, 32, true, 5, "/path/to/dir", "prefix", 5, 512}), "(0 vs. 0)"); VELOX_ASSERT_THROW( - LinuxMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "", "prefix", 5, 512}), + LinuxMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "", "prefix", 5, 512}), "heapDumpLogDir cannot be empty when heap dump is enabled."); VELOX_ASSERT_THROW( - LinuxMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "/path/to/dir", "", 5, 512}), + LinuxMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "/path/to/dir", "", 5, 512}), "heapDumpFilePrefix cannot be empty when heap dump is enabled."); - LinuxMemoryChecker memChecker(PeriodicMemoryChecker::Config{ - 1'000, false, 0, 0, false, 5, "/path/to/dir", "prefix", 5, 512}); + LinuxMemoryChecker memChecker( + PeriodicMemoryChecker::Config{ + 1'000, false, 0, 0, false, 5, "/path/to/dir", "prefix", 5, 512}); ASSERT_NO_THROW(memChecker.start()); VELOX_ASSERT_THROW(memChecker.start(), "start() called more than once"); @@ -154,17 +159,18 @@ TEST_F(LinuxMemoryCheckerTest, sysMemLimitBytesCheck) { // the available memory of deployment. // systemMemLimitBytes = 130,000,000,000 bytes. // available memory of deployment = 131,000,000,000 bytes. - LinuxMemoryChecker memChecker(PeriodicMemoryChecker::Config{ - 1'000, - true, - 130000000000, - 32, - true, - 5, - "/path/to/dir", - "prefix", - 5, - 512}); + LinuxMemoryChecker memChecker( + PeriodicMemoryChecker::Config{ + 1'000, + true, + 130000000000, + 32, + true, + 5, + "/path/to/dir", + "prefix", + 5, + 512}); memChecker.setMemInfoFile(memInfoPath); memChecker.setMemMaxFile(memMaxFilePath); ASSERT_NO_THROW(memChecker.start()); @@ -172,17 +178,18 @@ TEST_F(LinuxMemoryCheckerTest, sysMemLimitBytesCheck) { // systemMemLimitBytes = 131,000,000,001 bytes. // available memory of deployment = 131,000,000,000 bytes. - LinuxMemoryChecker memChecker2(PeriodicMemoryChecker::Config{ - 1'000, - true, - 131000000001, - 32, - true, - 5, - "/path/to/dir", - "prefix", - 5, - 512}); + LinuxMemoryChecker memChecker2( + PeriodicMemoryChecker::Config{ + 1'000, + true, + 131000000001, + 32, + true, + 5, + "/path/to/dir", + "prefix", + 5, + 512}); memChecker2.setMemInfoFile(memInfoPath); memChecker2.setMemMaxFile(memMaxFilePath); VELOX_ASSERT_THROW(memChecker2.start(), "(131000000001 vs. 131000000000)"); diff --git a/presto-native-execution/presto_cpp/main/tests/PeriodicMemoryCheckerTest.cpp b/presto-native-execution/presto_cpp/main/tests/PeriodicMemoryCheckerTest.cpp index 4c625412cba37..68c5615656d7b 100644 --- a/presto-native-execution/presto_cpp/main/tests/PeriodicMemoryCheckerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PeriodicMemoryCheckerTest.cpp @@ -37,8 +37,8 @@ class PeriodicMemoryCheckerTest : public testing::Test { mallocBytes_(mallocBytes), periodicCb_(std::move(periodicCb)), heapDumpCb_(std::move(heapDumpCb)) { - cachedSystemUsedMemoryBytes_ = systemUsedMemoryBytes; - } + cachedSystemUsedMemoryBytes_ = systemUsedMemoryBytes; + } ~TestPeriodicMemoryChecker() override {} @@ -79,22 +79,27 @@ TEST_F(PeriodicMemoryCheckerTest, basic) { // Default config ASSERT_NO_THROW(TestPeriodicMemoryChecker(PeriodicMemoryChecker::Config{})); - ASSERT_NO_THROW(TestPeriodicMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "/path/to/dir", "prefix", 5, 512})); + ASSERT_NO_THROW(TestPeriodicMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "/path/to/dir", "prefix", 5, 512})); VELOX_ASSERT_THROW( - TestPeriodicMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 0, 32, true, 5, "/path/to/dir", "prefix", 5, 512}), + TestPeriodicMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 0, 32, true, 5, "/path/to/dir", "prefix", 5, 512}), "(0 vs. 0)"); VELOX_ASSERT_THROW( - TestPeriodicMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "", "prefix", 5, 512}), + TestPeriodicMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "", "prefix", 5, 512}), "heapDumpLogDir cannot be empty when heap dump is enabled."); VELOX_ASSERT_THROW( - TestPeriodicMemoryChecker(PeriodicMemoryChecker::Config{ - 1'000, true, 1024, 32, true, 5, "/path/to/dir", "", 5, 512}), + TestPeriodicMemoryChecker( + PeriodicMemoryChecker::Config{ + 1'000, true, 1024, 32, true, 5, "/path/to/dir", "", 5, 512}), "heapDumpFilePrefix cannot be empty when heap dump is enabled."); - TestPeriodicMemoryChecker memChecker(PeriodicMemoryChecker::Config{ - 1'000, false, 0, 0, false, 5, "/path/to/dir", "prefix", 5, 512}); + TestPeriodicMemoryChecker memChecker( + PeriodicMemoryChecker::Config{ + 1'000, false, 0, 0, false, 5, "/path/to/dir", "prefix", 5, 512}); ASSERT_NO_THROW(memChecker.start()); VELOX_ASSERT_THROW(memChecker.start(), "start() called more than once"); ASSERT_NO_THROW(memChecker.stop()); diff --git a/presto-native-execution/presto_cpp/main/tests/PrestoExchangeSourceTest.cpp b/presto-native-execution/presto_cpp/main/tests/PrestoExchangeSourceTest.cpp index a1e76624a6ebc..8940091f2e31b 100644 --- a/presto-native-execution/presto_cpp/main/tests/PrestoExchangeSourceTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PrestoExchangeSourceTest.cpp @@ -19,8 +19,8 @@ #include #include "folly/experimental/EventCount.h" #include "presto_cpp/main/PrestoExchangeSource.h" -#include "presto_cpp/main/common/tests/MutableConfigs.h" #include "presto_cpp/main/common/Utils.h" +#include "presto_cpp/main/common/tests/MutableConfigs.h" #include "presto_cpp/main/tests/HttpServerWrapper.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/common/base/tests/GTestUtils.h" diff --git a/presto-native-execution/presto_cpp/main/tests/PrestoTaskTest.cpp b/presto-native-execution/presto_cpp/main/tests/PrestoTaskTest.cpp index 91eae94718df9..f1334ef201a17 100644 --- a/presto-native-execution/presto_cpp/main/tests/PrestoTaskTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PrestoTaskTest.cpp @@ -24,8 +24,8 @@ DECLARE_bool(velox_memory_leak_check_enabled); using namespace facebook::velox; using namespace facebook::presto; -using facebook::velox::exec::test::PlanBuilder; using facebook::presto::PrestoTaskId; +using facebook::velox::exec::test::PlanBuilder; namespace { // Create a simple velox task for testing. diff --git a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp index daffa64442580..d391df273c844 100644 --- a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp @@ -506,7 +506,8 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionAndExtraCredentialsOverload) { EXPECT_EQ("custom_value", raw.at("custom_credential")); } - // --- Test 3: Merge behavior: session system properties + more credentials --- + // --- Test 3: Merge behavior: session system properties + more credentials + // --- { auto session = createBasicSession(); // Verify that typed options reflect session settings. @@ -541,7 +542,8 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionStartTimeConfiguration) { // The startTime is already set in createBasicSession() to 1234567890 auto veloxConfig = QueryConfig{toVeloxConfigs(session)}; - // Verify that session start time is properly passed through to VeloxQueryConfig + // Verify that session start time is properly passed through to + // VeloxQueryConfig EXPECT_EQ(1234567890, veloxConfig.sessionStartTimeMs()); // Test with different session start time diff --git a/presto-native-execution/presto_cpp/main/tests/QueryContextCacheTest.cpp b/presto-native-execution/presto_cpp/main/tests/QueryContextCacheTest.cpp index beac11c0203fd..b62c225dcfb1f 100644 --- a/presto-native-execution/presto_cpp/main/tests/QueryContextCacheTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/QueryContextCacheTest.cpp @@ -166,4 +166,4 @@ TEST_F(QueryContextCacheTest, eviction) { verifyQueryCtxCache(queryContextCache, queryCtxs, 0, 20); EXPECT_EQ(queryContextCache.size(), 0); } -} // namespace facebook::presto \ No newline at end of file +} // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp index af056aa289f2f..1c501e662f9ae 100644 --- a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp @@ -22,7 +22,8 @@ namespace facebook::presto { class QueryContextManagerTest : public testing::Test { protected: static void SetUpTestCase() { - velox::memory::MemoryManager::testingSetInstance(velox::memory::MemoryManager::Options{}); + velox::memory::MemoryManager::testingSetInstance( + velox::memory::MemoryManager::Options{}); } void SetUp() override { @@ -123,20 +124,26 @@ TEST_F(QueryContextManagerTest, defaultSessionProperties) { EXPECT_EQ( queryConfig.spillCompressionKind(), defaultQC->spillCompressionKind()); EXPECT_EQ(queryConfig.spillEnabled(), defaultQC->spillEnabled()); - EXPECT_EQ(queryConfig.aggregationSpillEnabled(), defaultQC->aggregationSpillEnabled()); + EXPECT_EQ( + queryConfig.aggregationSpillEnabled(), + defaultQC->aggregationSpillEnabled()); EXPECT_EQ(queryConfig.joinSpillEnabled(), defaultQC->joinSpillEnabled()); - EXPECT_EQ(queryConfig.orderBySpillEnabled(), defaultQC->orderBySpillEnabled()); + EXPECT_EQ( + queryConfig.orderBySpillEnabled(), defaultQC->orderBySpillEnabled()); EXPECT_EQ( queryConfig.validateOutputFromOperators(), defaultQC->validateOutputFromOperators()); EXPECT_EQ( queryConfig.spillWriteBufferSize(), defaultQC->spillWriteBufferSize()); EXPECT_EQ( - queryConfig.requestDataSizesMaxWaitSec(), defaultQC->requestDataSizesMaxWaitSec()); + queryConfig.requestDataSizesMaxWaitSec(), + defaultQC->requestDataSizesMaxWaitSec()); EXPECT_EQ( - queryConfig.maxSplitPreloadPerDriver(), defaultQC->maxSplitPreloadPerDriver()); + queryConfig.maxSplitPreloadPerDriver(), + defaultQC->maxSplitPreloadPerDriver()); EXPECT_EQ( - queryConfig.maxLocalExchangePartitionBufferSize(), defaultQC->maxLocalExchangePartitionBufferSize()); + queryConfig.maxLocalExchangePartitionBufferSize(), + defaultQC->maxLocalExchangePartitionBufferSize()); } TEST_F(QueryContextManagerTest, overridingSessionProperties) { @@ -149,7 +156,7 @@ TEST_F(QueryContextManagerTest, overridingSessionProperties) { auto queryCtx = taskManager_->getQueryContextManager()->findOrCreateQueryCtx( taskId, updateRequest); - // When session properties are not explicitly set, they should be set to + // When session properties are not explicitly set, they should be set to // system config values. EXPECT_EQ( queryCtx->queryConfig().queryMaxMemoryPerNode(), @@ -158,8 +165,7 @@ TEST_F(QueryContextManagerTest, overridingSessionProperties) { queryCtx->queryConfig().spillFileCreateConfig(), systemConfig->spillerFileCreateConfig()); EXPECT_EQ( - queryCtx->queryConfig().spillEnabled(), - systemConfig->spillEnabled()); + queryCtx->queryConfig().spillEnabled(), systemConfig->spillEnabled()); EXPECT_EQ( queryCtx->queryConfig().aggregationSpillEnabled(), systemConfig->aggregationSpillEnabled()); @@ -193,20 +199,16 @@ TEST_F(QueryContextManagerTest, overridingSessionProperties) { EXPECT_EQ( queryCtx->queryConfig().spillFileCreateConfig(), "encoding:replica_2"); // Override with different value - EXPECT_EQ( - queryCtx->queryConfig().spillEnabled(), true); + EXPECT_EQ(queryCtx->queryConfig().spillEnabled(), true); EXPECT_NE( - queryCtx->queryConfig().spillEnabled(), - systemConfig->spillEnabled()); + queryCtx->queryConfig().spillEnabled(), systemConfig->spillEnabled()); // Override with different value - EXPECT_EQ( - queryCtx->queryConfig().aggregationSpillEnabled(), false); + EXPECT_EQ(queryCtx->queryConfig().aggregationSpillEnabled(), false); EXPECT_NE( queryCtx->queryConfig().aggregationSpillEnabled(), systemConfig->aggregationSpillEnabled()); // Override with same value - EXPECT_EQ( - queryCtx->queryConfig().joinSpillEnabled(), true); + EXPECT_EQ(queryCtx->queryConfig().joinSpillEnabled(), true); EXPECT_EQ( queryCtx->queryConfig().joinSpillEnabled(), systemConfig->joinSpillEnabled()); diff --git a/presto-native-execution/presto_cpp/main/tests/TaskInfoTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskInfoTest.cpp index 38b8150634d0f..1cf019f0b62a5 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskInfoTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskInfoTest.cpp @@ -13,15 +13,15 @@ */ #include -#include "presto_cpp/main/thrift/ProtocolToThrift.h" -#include "presto_cpp/presto_protocol/core/Duration.h" #include "presto_cpp/main/common/tests/test_json.h" #include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" +#include "presto_cpp/main/thrift/ProtocolToThrift.h" +#include "presto_cpp/presto_protocol/core/Duration.h" using namespace facebook::presto; class TaskInfoTest : public ::testing::Test { - protected: + protected: void SetUp() override { registerPrestoToVeloxConnector( std::make_unique("hive")); @@ -32,11 +32,13 @@ class TaskInfoTest : public ::testing::Test { } }; -const std::string BASE_DATA_PATH = "/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/"; +const std::string BASE_DATA_PATH = + "/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/"; TEST_F(TaskInfoTest, duration) { double thrift = 0; - thrift::toThrift(protocol::Duration(123, protocol::TimeUnit::MILLISECONDS), thrift); + thrift::toThrift( + protocol::Duration(123, protocol::TimeUnit::MILLISECONDS), thrift); ASSERT_EQ(thrift, 123); } @@ -51,17 +53,24 @@ TEST_F(TaskInfoTest, taskInfo) { ASSERT_EQ(thriftTaskInfo.outputBuffers()->buffers()->size(), 2); ASSERT_EQ(thriftTaskInfo.outputBuffers()->buffers()[0].bufferId()->id(), 100); ASSERT_EQ(thriftTaskInfo.outputBuffers()->buffers()[1].bufferId()->id(), 200); - ASSERT_EQ(thriftTaskInfo.stats()->blockedReasons()->count(thrift::BlockedReason::WAITING_FOR_MEMORY), 1); + ASSERT_EQ( + thriftTaskInfo.stats()->blockedReasons()->count( + thrift::BlockedReason::WAITING_FOR_MEMORY), + 1); ASSERT_EQ(thriftTaskInfo.stats()->runtimeStats()->metrics()->size(), 2); - ASSERT_EQ(thriftTaskInfo.stats()->runtimeStats()->metrics()["test_metric1"].sum(), 123); - ASSERT_EQ(thriftTaskInfo.stats()->runtimeStats()->metrics()["test_metric2"].name(), "test_metric2"); + ASSERT_EQ( + thriftTaskInfo.stats()->runtimeStats()->metrics()["test_metric1"].sum(), + 123); + ASSERT_EQ( + thriftTaskInfo.stats()->runtimeStats()->metrics()["test_metric2"].name(), + "test_metric2"); } TEST_F(TaskInfoTest, taskId) { protocol::TaskId taskId = "queryId.1.2.3.4"; thrift::TaskId thriftTaskId; thrift::toThrift(taskId, thriftTaskId); - + ASSERT_EQ(thriftTaskId.stageExecutionId()->stageId()->queryId(), "queryId"); ASSERT_EQ(thriftTaskId.stageExecutionId()->stageId()->id(), 1); ASSERT_EQ(thriftTaskId.stageExecutionId()->id(), 2); @@ -69,14 +78,14 @@ TEST_F(TaskInfoTest, taskId) { ASSERT_EQ(thriftTaskId.attemptNumber(), 4); } - TEST_F(TaskInfoTest, operatorStatsEmptyBlockedReason) { - std::string str = slurp(getDataPath(BASE_DATA_PATH, "OperatorStatsEmptyBlockedReason.json")); + std::string str = slurp( + getDataPath(BASE_DATA_PATH, "OperatorStatsEmptyBlockedReason.json")); json j = json::parse(str); protocol::OperatorStats operatorStats = j; thrift::OperatorStats thriftOperatorStats; thrift::toThrift(operatorStats, thriftOperatorStats); - + ASSERT_EQ(thriftOperatorStats.blockedReason().has_value(), false); ASSERT_EQ(thriftOperatorStats.blockedWall(), 80); ASSERT_EQ(thriftOperatorStats.finishCpu(), 1000); @@ -88,6 +97,8 @@ TEST_F(TaskInfoTest, operatorStats) { protocol::OperatorStats operatorStats = j; thrift::OperatorStats thriftOperatorStats; thrift::toThrift(operatorStats, thriftOperatorStats); - - ASSERT_EQ(thriftOperatorStats.blockedReason(), thrift::BlockedReason::WAITING_FOR_MEMORY); + + ASSERT_EQ( + thriftOperatorStats.blockedReason(), + thrift::BlockedReason::WAITING_FOR_MEMORY); } diff --git a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp index fc8cd9575862b..c6b6313ac12cf 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp @@ -19,8 +19,8 @@ #include "presto_cpp/main/PrestoExchangeSource.h" #include "presto_cpp/main/TaskResource.h" #include "presto_cpp/main/common/tests/MutableConfigs.h" -#include "presto_cpp/main/connectors/PrestoToVeloxConnector.h" #include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" +#include "presto_cpp/main/connectors/PrestoToVeloxConnector.h" #include "presto_cpp/main/tests/HttpServerWrapper.h" #include "velox/common/base/Fs.h" #include "velox/common/base/tests/GTestUtils.h" @@ -148,10 +148,11 @@ class Cursor { std::vector deserialize(folly::IOBuf* buffer) { std::vector byteRanges; for (auto& range : *buffer) { - byteRanges.emplace_back(ByteRange{ - const_cast(range.data()), - static_cast(range.size()), - 0}); + byteRanges.emplace_back( + ByteRange{ + const_cast(range.data()), + static_cast(range.size()), + 0}); } const auto input = @@ -227,8 +228,9 @@ class TaskManagerTest : public exec::test::OperatorTestBase, nullptr); }); - registerPrestoToVeloxConnector(std::make_unique( - connector::hive::HiveConnectorFactory::kHiveConnectorName)); + registerPrestoToVeloxConnector( + std::make_unique( + connector::hive::HiveConnectorFactory::kHiveConnectorName)); connector::hive::HiveConnectorFactory factory; auto hiveConnector = factory.newConnector( kHiveConnectorId, @@ -259,10 +261,11 @@ class TaskManagerTest : public exec::test::OperatorTestBase, std::move(httpServer)); auto serverAddress = httpServerWrapper_->start().get(); - taskManager_->setBaseUri(fmt::format( - "http://{}:{}", - serverAddress.getAddressStr(), - serverAddress.getPort())); + taskManager_->setBaseUri( + fmt::format( + "http://{}:{}", + serverAddress.getAddressStr(), + serverAddress.getPort())); writerFactory_ = dwio::common::getWriterFactory(dwio::common::FileFormat::DWRF); } @@ -614,12 +617,13 @@ class TaskManagerTest : public exec::test::OperatorTestBase, auto nodeConfigFilePath = fmt::format("{}/node.properties", spillDirectory->getPath()); auto nodeConfigFile = fileSystem->openFileForWrite(nodeConfigFilePath); - nodeConfigFile->append(fmt::format( - "{}={}\n{}={}", - NodeConfig::kNodeInternalAddress, - "192.16.7.66", - NodeConfig::kNodeId, - "12")); + nodeConfigFile->append( + fmt::format( + "{}={}\n{}={}", + NodeConfig::kNodeInternalAddress, + "192.16.7.66", + NodeConfig::kNodeId, + "12")); nodeConfigFile->close(); NodeConfig::instance()->initialize(nodeConfigFilePath); @@ -718,11 +722,12 @@ TEST_P(TaskManagerTest, addSplitsWithSameSourceNode) { } duckDbQueryRunner_.createTable("tmp", vectors); - const auto planFragment = exec::test::PlanBuilder() - .tableScan(rowType_) - .filter("c0 % 5 = 0") - .partitionedOutput({}, 1, {"c0", "c1"}, GetParam()) - .planFragment(); + const auto planFragment = + exec::test::PlanBuilder() + .tableScan(rowType_) + .filter("c0 % 5 = 0") + .partitionedOutput({}, 1, {"c0", "c1"}, GetParam()) + .planFragment(); protocol::TaskUpdateRequest updateRequest; // Create multiple task sources with the same source node id. @@ -730,7 +735,8 @@ TEST_P(TaskManagerTest, addSplitsWithSameSourceNode) { taskSources.reserve(filePaths.size()); long splitSequenceId{0}; for (const auto& filePath : filePaths) { - taskSources.push_back(makeSource("0", {filePath}, /*noMoreSplits=*/true, splitSequenceId)); + taskSources.push_back( + makeSource("0", {filePath}, /*noMoreSplits=*/true, splitSequenceId)); } taskSources.reserve(filePaths.size()); updateRequest.sources = std::move(taskSources); @@ -1588,8 +1594,9 @@ TEST_P(TaskManagerTest, buildSpillDirectoryFailure) { // Cleanup old tasks between test iterations. taskManager_->setOldTaskCleanUpMs(0); for (bool buildSpillDirectoryFailure : {false}) { - SCOPED_TRACE(fmt::format( - "buildSpillDirectoryFailure: {}", buildSpillDirectoryFailure)); + SCOPED_TRACE( + fmt::format( + "buildSpillDirectoryFailure: {}", buildSpillDirectoryFailure)); auto spillDir = setupSpillPath(); std::vector batches = makeVectors(1, 1'000); diff --git a/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp index 00aba3d3ea1ae..3fbbea4c4f4e3 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp @@ -13,8 +13,8 @@ */ #include -#include "presto_cpp/main/thrift/ProtocolToThrift.h" #include "presto_cpp/main/common/tests/test_json.h" +#include "presto_cpp/main/thrift/ProtocolToThrift.h" using namespace facebook::presto; @@ -78,16 +78,24 @@ TEST_F(TaskStatusTest, executionFailureInfoOptionalFieldsEmpty) { ASSERT_EQ(thriftExecutionFailureInfo.type(), "type"); ASSERT_EQ(thriftExecutionFailureInfo.errorLocation()->columnNumber(), 2); - ASSERT_EQ(thriftExecutionFailureInfo.remoteHost()->hostPortString(), "localhost:8080"); - ASSERT_EQ(thriftExecutionFailureInfo.errorCode()->type(), thrift::ErrorType::INTERNAL_ERROR); + ASSERT_EQ( + thriftExecutionFailureInfo.remoteHost()->hostPortString(), + "localhost:8080"); + ASSERT_EQ( + thriftExecutionFailureInfo.errorCode()->type(), + thrift::ErrorType::INTERNAL_ERROR); ASSERT_EQ(thriftExecutionFailureInfo.errorCode()->retriable(), false); - ASSERT_EQ(thriftExecutionFailureInfo.errorCause(), thrift::ErrorCause::EXCEEDS_BROADCAST_MEMORY_LIMIT); + ASSERT_EQ( + thriftExecutionFailureInfo.errorCause(), + thrift::ErrorCause::EXCEEDS_BROADCAST_MEMORY_LIMIT); ASSERT_EQ(thriftExecutionFailureInfo.cause(), nullptr); ASSERT_EQ(thriftExecutionFailureInfo.suppressed()->size(), 0); } TEST_F(TaskStatusTest, executionFailureInfoOptionalFieldsNonempty) { - std::string str = slurp(getDataPath("/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/", "ExecutionFailureInfo.json")); + std::string str = slurp(getDataPath( + "/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/", + "ExecutionFailureInfo.json")); json j = json::parse(str); protocol::ExecutionFailureInfo executionFailureInfo = j; @@ -95,13 +103,28 @@ TEST_F(TaskStatusTest, executionFailureInfoOptionalFieldsNonempty) { thrift::toThrift(executionFailureInfo, thriftExecutionFailureInfo); ASSERT_EQ((*thriftExecutionFailureInfo.cause()).type(), "cause"); - ASSERT_EQ((*thriftExecutionFailureInfo.cause()).errorCause(), thrift::ErrorCause::UNKNOWN); - ASSERT_EQ((*thriftExecutionFailureInfo.cause()).errorCode()->type(), thrift::ErrorType::INSUFFICIENT_RESOURCES); - ASSERT_EQ((*thriftExecutionFailureInfo.cause()).errorCode()->retriable(), true); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[0].type(), "suppressed1"); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[0].errorCause(), thrift::ErrorCause::LOW_PARTITION_COUNT); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[0].errorCode()->type(), thrift::ErrorType::EXTERNAL); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[1].type(), "suppressed2"); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[1].errorCause(), thrift::ErrorCause::EXCEEDS_BROADCAST_MEMORY_LIMIT); - ASSERT_EQ((*thriftExecutionFailureInfo.suppressed())[1].errorCode()->type(), thrift::ErrorType::INTERNAL_ERROR); + ASSERT_EQ( + (*thriftExecutionFailureInfo.cause()).errorCause(), + thrift::ErrorCause::UNKNOWN); + ASSERT_EQ( + (*thriftExecutionFailureInfo.cause()).errorCode()->type(), + thrift::ErrorType::INSUFFICIENT_RESOURCES); + ASSERT_EQ( + (*thriftExecutionFailureInfo.cause()).errorCode()->retriable(), true); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[0].type(), "suppressed1"); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[0].errorCause(), + thrift::ErrorCause::LOW_PARTITION_COUNT); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[0].errorCode()->type(), + thrift::ErrorType::EXTERNAL); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[1].type(), "suppressed2"); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[1].errorCause(), + thrift::ErrorCause::EXCEEDS_BROADCAST_MEMORY_LIMIT); + ASSERT_EQ( + (*thriftExecutionFailureInfo.suppressed())[1].errorCode()->type(), + thrift::ErrorType::INTERNAL_ERROR); } diff --git a/presto-native-execution/presto_cpp/main/tests/TaskUpdateRequestTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskUpdateRequestTest.cpp index 065ce62996e2e..25c0de8c1ecf7 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskUpdateRequestTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskUpdateRequestTest.cpp @@ -13,16 +13,16 @@ */ #include -#include "presto_cpp/main/thrift/ProtocolToThrift.h" -#include "presto_cpp/main/thrift/ThriftIO.h" #include "presto_cpp/main/common/tests/test_json.h" -#include "presto_cpp/main/connectors/PrestoToVeloxConnector.h" #include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" +#include "presto_cpp/main/connectors/PrestoToVeloxConnector.h" +#include "presto_cpp/main/thrift/ProtocolToThrift.h" +#include "presto_cpp/main/thrift/ThriftIO.h" using namespace facebook::presto; class TaskUpdateRequestTest : public ::testing::Test { - protected: + protected: void SetUp() override { registerPrestoToVeloxConnector( std::make_unique("hive")); @@ -33,7 +33,8 @@ class TaskUpdateRequestTest : public ::testing::Test { } }; -const std::string BASE_DATA_PATH = "/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/"; +const std::string BASE_DATA_PATH = + "/github/presto-trunk/presto-native-execution/presto_cpp/main/tests/data/"; TEST_F(TaskUpdateRequestTest, connectorId) { protocol::ConnectorId connectorId; @@ -49,9 +50,13 @@ TEST_F(TaskUpdateRequestTest, optionalField) { thriftResourceEstimates.executionTime_ref() = 100; thriftResourceEstimates.peakMemory_ref() = 1024 * 1024 * 1024; thrift::fromThrift(thriftResourceEstimates, resourceEstimates); - ASSERT_EQ(*resourceEstimates.executionTime, protocol::Duration(100, protocol::TimeUnit::MILLISECONDS)); + ASSERT_EQ( + *resourceEstimates.executionTime, + protocol::Duration(100, protocol::TimeUnit::MILLISECONDS)); ASSERT_EQ(resourceEstimates.cpuTime, nullptr); - ASSERT_EQ(*resourceEstimates.peakMemory, protocol::DataSize(1024 * 1024 * 1024, protocol::DataUnit::BYTE)); + ASSERT_EQ( + *resourceEstimates.peakMemory, + protocol::DataSize(1024 * 1024 * 1024, protocol::DataUnit::BYTE)); ASSERT_EQ(resourceEstimates.peakTaskMemory, nullptr); } @@ -71,12 +76,18 @@ TEST_F(TaskUpdateRequestTest, routineCharacteristics) { thrift::Language thriftLanguage; thriftLanguage.language_ref() = "English"; thriftRroutineCharacteristics.language_ref() = std::move(thriftLanguage); - thriftRroutineCharacteristics.determinism_ref() = thrift::Determinism::NOT_DETERMINISTIC; - thriftRroutineCharacteristics.nullCallClause_ref() = thrift::NullCallClause::RETURNS_NULL_ON_NULL_INPUT; + thriftRroutineCharacteristics.determinism_ref() = + thrift::Determinism::NOT_DETERMINISTIC; + thriftRroutineCharacteristics.nullCallClause_ref() = + thrift::NullCallClause::RETURNS_NULL_ON_NULL_INPUT; thrift::fromThrift(thriftRroutineCharacteristics, routineCharacteristics); ASSERT_EQ((*routineCharacteristics.language).language, "English"); - ASSERT_EQ(*routineCharacteristics.determinism, protocol::Determinism::NOT_DETERMINISTIC); - ASSERT_EQ(*routineCharacteristics.nullCallClause, protocol::NullCallClause::RETURNS_NULL_ON_NULL_INPUT); + ASSERT_EQ( + *routineCharacteristics.determinism, + protocol::Determinism::NOT_DETERMINISTIC); + ASSERT_EQ( + *routineCharacteristics.nullCallClause, + protocol::NullCallClause::RETURNS_NULL_ON_NULL_INPUT); } TEST_F(TaskUpdateRequestTest, mapOutputBuffers) { @@ -89,10 +100,8 @@ TEST_F(TaskUpdateRequestTest, mapOutputBuffers) { thrift::OutputBufferId outputBufferId2; outputBufferId1.id_ref() = 1; outputBufferId2.id_ref() = 2; - thriftOutputBuffers.buffers_ref() = { - {outputBufferId1, 10}, - {outputBufferId2, 20} - }; + thriftOutputBuffers.buffers_ref() = { + {outputBufferId1, 10}, {outputBufferId2, 20}}; thrift::fromThrift(thriftOutputBuffers, outputBuffers); ASSERT_EQ(outputBuffers.type, protocol::BufferType::ARBITRARY); @@ -109,20 +118,20 @@ TEST_F(TaskUpdateRequestTest, binaryHiveSplitFromThrift) { "@type": "hive", "uuid": "8a4d6c83-60ee-46de-9715-bc91755619fa" })"; - thriftSplit.connectorSplit()->jsonValue_ref() = slurp(getDataPath(BASE_DATA_PATH, "HiveSplit.json")); + thriftSplit.connectorSplit()->jsonValue_ref() = + slurp(getDataPath(BASE_DATA_PATH, "HiveSplit.json")); protocol::Split split; thrift::fromThrift(thriftSplit, split); - + // Verify that connector specific fields are set correctly with json codec auto transactionHandle = std::dynamic_pointer_cast( split.transactionHandle); ASSERT_EQ(transactionHandle->uuid, "8a4d6c83-60ee-46de-9715-bc91755619fa"); - auto hiveSplit = - std::dynamic_pointer_cast( - split.connectorSplit); + auto hiveSplit = std::dynamic_pointer_cast( + split.connectorSplit); ASSERT_EQ(hiveSplit->database, "tpch"); ASSERT_EQ( hiveSplit->nodeSelectionStrategy, @@ -137,13 +146,19 @@ TEST_F(TaskUpdateRequestTest, binaryRemoteSplitFromThrift) { thriftSplit.connectorId()->catalogName_ref() = "$remote"; thriftSplit.transactionHandle()->customSerializedValue_ref() = thriftWrite(thriftTransactionHandle); - + thriftRemoteSplit.location()->location_ref() = "/test_location"; thriftRemoteSplit.remoteSourceTaskId()->id_ref() = 100; thriftRemoteSplit.remoteSourceTaskId()->attemptNumber_ref() = 200; thriftRemoteSplit.remoteSourceTaskId()->stageExecutionId()->id_ref() = 300; - thriftRemoteSplit.remoteSourceTaskId()->stageExecutionId()->stageId()->id_ref() = 400; - thriftRemoteSplit.remoteSourceTaskId()->stageExecutionId()->stageId()->queryId_ref() = "test_query_id"; + thriftRemoteSplit.remoteSourceTaskId() + ->stageExecutionId() + ->stageId() + ->id_ref() = 400; + thriftRemoteSplit.remoteSourceTaskId() + ->stageExecutionId() + ->stageId() + ->queryId_ref() = "test_query_id"; thriftSplit.connectorSplit()->connectorId_ref() = "$remote"; thriftSplit.connectorSplit()->customSerializedValue_ref() = @@ -153,8 +168,8 @@ TEST_F(TaskUpdateRequestTest, binaryRemoteSplitFromThrift) { thrift::fromThrift(thriftSplit, split); // Verify that connector specific fields are set correctly with thrift codec - auto remoteSplit = std::dynamic_pointer_cast( - split.connectorSplit); + auto remoteSplit = + std::dynamic_pointer_cast(split.connectorSplit); ASSERT_EQ((remoteSplit->location).location, "/test_location"); ASSERT_EQ(remoteSplit->remoteSourceTaskId, "test_query_id.400.300.100.200"); } @@ -170,29 +185,38 @@ TEST_F(TaskUpdateRequestTest, unionExecutionWriterTargetFromThrift) { "@type": "hive", "uuid": "8a4d6c83-60ee-46de-9715-bc91755619fa" })"; - thriftCreateHandle.handle()->connectorHandle()->jsonValue_ref() = slurp(getDataPath(BASE_DATA_PATH, "HiveOutputTableHandle.json"));; + thriftCreateHandle.handle()->connectorHandle()->jsonValue_ref() = + slurp(getDataPath(BASE_DATA_PATH, "HiveOutputTableHandle.json")); + ; thriftWriterTarget.set_createHandle(std::move(thriftCreateHandle)); - + // Convert from thrift to protocol and verify fields auto writerTarget = std::make_shared(); thrift::fromThrift(thriftWriterTarget, writerTarget); ASSERT_EQ(writerTarget->_type, "CreateHandle"); - auto createHandle = std::dynamic_pointer_cast(writerTarget); + auto createHandle = + std::dynamic_pointer_cast(writerTarget); ASSERT_NE(createHandle, nullptr); ASSERT_EQ(createHandle->schemaTableName.schema, "test_schema"); ASSERT_EQ(createHandle->schemaTableName.table, "test_table"); - auto* hiveTxnHandle = dynamic_cast(createHandle->handle.transactionHandle.get()); + auto* hiveTxnHandle = dynamic_cast( + createHandle->handle.transactionHandle.get()); ASSERT_NE(hiveTxnHandle, nullptr); ASSERT_EQ(hiveTxnHandle->uuid, "8a4d6c83-60ee-46de-9715-bc91755619fa"); - auto* hiveOutputTableHandle = dynamic_cast(createHandle->handle.connectorHandle.get()); + auto* hiveOutputTableHandle = + dynamic_cast( + createHandle->handle.connectorHandle.get()); ASSERT_NE(hiveOutputTableHandle, nullptr); ASSERT_EQ(hiveOutputTableHandle->schemaName, "test_schema"); ASSERT_EQ(hiveOutputTableHandle->tableName, "test_table"); - ASSERT_EQ(hiveOutputTableHandle->tableStorageFormat, protocol::hive::HiveStorageFormat::ORC); - ASSERT_EQ(hiveOutputTableHandle->locationHandle.targetPath, "/path/to/target"); + ASSERT_EQ( + hiveOutputTableHandle->tableStorageFormat, + protocol::hive::HiveStorageFormat::ORC); + ASSERT_EQ( + hiveOutputTableHandle->locationHandle.targetPath, "/path/to/target"); } TEST_F(TaskUpdateRequestTest, unionExecutionWriterTargetToThrift) { @@ -200,28 +224,35 @@ TEST_F(TaskUpdateRequestTest, unionExecutionWriterTargetToThrift) { auto createHandle = std::make_shared(); createHandle->schemaTableName.schema = "test_schema"; createHandle->schemaTableName.table = "test_table"; - + auto writerTarget = std::make_shared(); writerTarget->_type = "CreateHandle"; writerTarget = createHandle; - - // Convert to thrift and verify fields. Note that toThrift functions for connector fields are not implemented. + + // Convert to thrift and verify fields. Note that toThrift functions for + // connector fields are not implemented. thrift::ExecutionWriterTargetUnion thriftWriterTarget; thrift::toThrift(writerTarget, thriftWriterTarget); ASSERT_TRUE(thriftWriterTarget.createHandle_ref().has_value()); - const auto& thriftCreateHandle = thriftWriterTarget.createHandle_ref().value(); - ASSERT_EQ(thriftCreateHandle.schemaTableName()->schema_ref().value(), "test_schema"); - ASSERT_EQ(thriftCreateHandle.schemaTableName()->table_ref().value(), "test_table"); + const auto& thriftCreateHandle = + thriftWriterTarget.createHandle_ref().value(); + ASSERT_EQ( + thriftCreateHandle.schemaTableName()->schema_ref().value(), + "test_schema"); + ASSERT_EQ( + thriftCreateHandle.schemaTableName()->table_ref().value(), "test_table"); } TEST_F(TaskUpdateRequestTest, fragment) { - std::string str = slurp(getDataPath(BASE_DATA_PATH, "Fragment.thrift.base64")); + std::string str = + slurp(getDataPath(BASE_DATA_PATH, "Fragment.thrift.base64")); const auto strEnd = str.find_last_not_of(" \t\n\r"); if (strEnd != std::string::npos) { str.erase(strEnd + 1); } - protocol::PlanFragment f = json::parse(facebook::velox::encoding::Base64::decode(str)); + protocol::PlanFragment f = + json::parse(facebook::velox::encoding::Base64::decode(str)); ASSERT_EQ(f.root->_type, ".AggregationNode"); @@ -247,25 +278,23 @@ TEST_F(TaskUpdateRequestTest, sessionRepresentation) { thrift::SessionRepresentation thriftSessionRepresentation; std::map> thriftMap; thriftMap["Person1"] = { - {"Name", "John Doe"}, - {"Age", "30"}, - {"City", "New York"} - }; + {"Name", "John Doe"}, {"Age", "30"}, {"City", "New York"}}; thriftMap["Person2"] = { - {"Name", "Jane Doe"}, - {"Age", "25"}, - {"City", "Los Angeles"} - }; + {"Name", "Jane Doe"}, {"Age", "25"}, {"City", "Los Angeles"}}; thriftMap["Person3"] = { - {"Name", "Bob Smith"}, - {"Age", "40"}, - {"City", "Chicago"} - }; - thriftSessionRepresentation.unprocessedCatalogProperties_ref() = std::move(thriftMap); + {"Name", "Bob Smith"}, {"Age", "40"}, {"City", "Chicago"}}; + thriftSessionRepresentation.unprocessedCatalogProperties_ref() = + std::move(thriftMap); thrift::fromThrift(thriftSessionRepresentation, sessionRepresentation); ASSERT_EQ(sessionRepresentation.unprocessedCatalogProperties.size(), 3); - ASSERT_EQ(sessionRepresentation.unprocessedCatalogProperties["Person1"]["City"], "New York"); - ASSERT_EQ(sessionRepresentation.unprocessedCatalogProperties["Person2"]["Age"], "25"); - ASSERT_EQ(sessionRepresentation.unprocessedCatalogProperties["Person3"]["Name"], "Bob Smith"); + ASSERT_EQ( + sessionRepresentation.unprocessedCatalogProperties["Person1"]["City"], + "New York"); + ASSERT_EQ( + sessionRepresentation.unprocessedCatalogProperties["Person2"]["Age"], + "25"); + ASSERT_EQ( + sessionRepresentation.unprocessedCatalogProperties["Person3"]["Name"], + "Bob Smith"); } diff --git a/presto-native-execution/presto_cpp/main/tests/certs/client_ca.pem b/presto-native-execution/presto_cpp/main/tests/certs/client_ca.pem index 5b982399a1c0a..87c78fee2dd56 100644 --- a/presto-native-execution/presto_cpp/main/tests/certs/client_ca.pem +++ b/presto-native-execution/presto_cpp/main/tests/certs/client_ca.pem @@ -43,4 +43,4 @@ oJVHuDzW6KxQh4tV8Fg5hhbwJ7Y4AP99AY/WXYZKqjSu7lv9LO4ie4zBEgJol6Bs u4shAoGBAKbPrKUobbhsihDZsS8hjDQ9g0eMagxOJq57WqTEzAqih0k1YQQhjhHL fiANKrBbAFFOxdn1JfJNBdBXSEeVdJ+AMy7wj0K6dWGNP3A4FKqs5g5zb+grdPEJ l8qRFT+gODcZT3iSokEUuPnG0y07sM7CUS5rc94NqV/nE1dgQenA ------END RSA PRIVATE KEY----- \ No newline at end of file +-----END RSA PRIVATE KEY----- diff --git a/presto-native-execution/presto_cpp/main/tests/certs/test_cert1.pem b/presto-native-execution/presto_cpp/main/tests/certs/test_cert1.pem index ee5551d60cd60..db38566a56633 100644 --- a/presto-native-execution/presto_cpp/main/tests/certs/test_cert1.pem +++ b/presto-native-execution/presto_cpp/main/tests/certs/test_cert1.pem @@ -16,4 +16,4 @@ LDv5K7msbk4RvrWlM2rVVx41e03c+VxdUF6sBVgYANbE0uQ6ULYVWIPRz8lItWKl a19JzTyU8NQeOfXDlcxPg7KgdRGyTcbGlzOFr30ZQCK3OoL351XhRr/jlHQdScVA JCusLYHJHpBBFs4ZT7sbkBSk7W80oUpCleklLjPTAPwAfWVYNBCsVWba++mvmmWI QYrc4pXNR3oEDUZz02Zo28rvXJgXj3WHg1TBOf6y42KnHH9i1ejre7fomg== ------END CERTIFICATE----- \ No newline at end of file +-----END CERTIFICATE----- diff --git a/presto-native-execution/presto_cpp/main/tests/certs/test_key1.pem b/presto-native-execution/presto_cpp/main/tests/certs/test_key1.pem index 5819b04258c32..18d9c01cb262a 100644 --- a/presto-native-execution/presto_cpp/main/tests/certs/test_key1.pem +++ b/presto-native-execution/presto_cpp/main/tests/certs/test_key1.pem @@ -24,4 +24,4 @@ oJVHuDzW6KxQh4tV8Fg5hhbwJ7Y4AP99AY/WXYZKqjSu7lv9LO4ie4zBEgJol6Bs u4shAoGBAKbPrKUobbhsihDZsS8hjDQ9g0eMagxOJq57WqTEzAqih0k1YQQhjhHL fiANKrBbAFFOxdn1JfJNBdBXSEeVdJ+AMy7wj0K6dWGNP3A4FKqs5g5zb+grdPEJ l8qRFT+gODcZT3iSokEUuPnG0y07sM7CUS5rc94NqV/nE1dgQenA ------END RSA PRIVATE KEY----- \ No newline at end of file +-----END RSA PRIVATE KEY----- diff --git a/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt b/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt index e5a7f044f32ee..9deb725d94d06 100644 --- a/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt @@ -16,8 +16,7 @@ find_library(THRIFT_CORE thrift-core) find_library(THRIFT_PROTOCOL thriftprotocol) find_library(THRIFT_METADATA thriftmetadata) find_library(THRIFT_TRANSPORT transport) -find_path(THRIFT_INCLUDES thrift/lib/cpp2/gen/module_data_h.h - PATH_SUFFIXES include REQUIRED) +find_path(THRIFT_INCLUDES thrift/lib/cpp2/gen/module_data_h.h PATH_SUFFIXES include REQUIRED) include(ThriftLibrary.cmake) @@ -28,14 +27,18 @@ thrift_library( "" ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/presto_cpp/main/thrift - "..") -target_link_libraries(presto_thrift-cpp2 ${THRIFT_PROTOCOL} ${THRIFT_METADATA} - ${THRIFT_CORE} ${THRIFT_TRANSPORT}) + ".." +) +target_link_libraries( + presto_thrift-cpp2 + ${THRIFT_PROTOCOL} + ${THRIFT_METADATA} + ${THRIFT_CORE} + ${THRIFT_TRANSPORT} +) set(presto_thrift_INCLUDES ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(presto_thrift-cpp2 PUBLIC ${presto_thrift_INCLUDES} - ${GLOG_INCLUDE_DIR}) -target_include_directories(presto_thrift-cpp2-obj PUBLIC ${THRIFT_INCLUDES} - ${GLOG_INCLUDE_DIR}) +target_include_directories(presto_thrift-cpp2 PUBLIC ${presto_thrift_INCLUDES} ${GLOG_INCLUDE_DIR}) +target_include_directories(presto_thrift-cpp2-obj PUBLIC ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR}) thrift_library( presto_native @@ -44,19 +47,24 @@ thrift_library( "" ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/presto_cpp/main/thrift - "..") -target_link_libraries(presto_native-cpp2 ${THRIFT_PROTOCOL} ${THRIFT_METADATA} - ${THRIFT_CORE} ${THRIFT_TRANSPORT}) + ".." +) +target_link_libraries( + presto_native-cpp2 + ${THRIFT_PROTOCOL} + ${THRIFT_METADATA} + ${THRIFT_CORE} + ${THRIFT_TRANSPORT} +) set(presto_native_INCLUDES ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(presto_native-cpp2 PUBLIC ${presto_native_INCLUDES} - ${GLOG_INCLUDE_DIR}) -target_include_directories(presto_native-cpp2-obj PUBLIC ${THRIFT_INCLUDES} - ${GLOG_INCLUDE_DIR}) +target_include_directories(presto_native-cpp2 PUBLIC ${presto_native_INCLUDES} ${GLOG_INCLUDE_DIR}) +target_include_directories(presto_native-cpp2-obj PUBLIC ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR}) add_library(presto_thrift_extra ProtocolToThrift.cpp) target_include_directories( - presto_thrift_extra PUBLIC ${presto_thrift_INCLUDES} ${THRIFT_INCLUDES} - ${GLOG_INCLUDE_DIR}) + presto_thrift_extra + PUBLIC ${presto_thrift_INCLUDES} ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR} +) add_dependencies(presto_thrift_extra presto_thrift-cpp2) if(PRESTO_ENABLE_TESTING) diff --git a/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.py b/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.py index 7b9428c877ff8..86fefdbe7e481 100755 --- a/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.py +++ b/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.py @@ -74,19 +74,19 @@ def verify(thrift_item, protocol_item): + str(protocol_field_set - thrift_field_set) ) + def process_fields(thrift_item, config_item): for field in thrift_item.fields: if ( - config_item is not None + config_item is not None and "fields" in config_item and field.field_name in config_item.fields ): - field["proto_name"] = config_item.fields[ - field.field_name - ].field_name + field["proto_name"] = config_item.fields[field.field_name].field_name else: field["proto_name"] = field.field_name + def main(): args = parse_args() config = util.load_yaml(args.config) @@ -102,23 +102,32 @@ def main(): result = [{"comment": comment}] # Skip structs that are not used in cpp - thrift = [item for item in thrift if "class_name" in item and item.class_name not in config.SkipStruct] + thrift = [ + item + for item in thrift + if "class_name" in item and item.class_name not in config.SkipStruct + ] for thrift_item in thrift: if "class_name" not in thrift_item: continue - # For structs that are defined in presto_protocol_core.h + # For structs that are defined in presto_protocol_core.h if thrift_item.class_name in config.StructInProtocolCore: thrift_item["core"] = "true" - + # For union structs if "union" in thrift_item and thrift_item.class_name not in config.Special: thrift_item["proto_name"] = thrift_item.class_name.removesuffix("Union") - if thrift_item.class_name in config.StructMap and "fields" in config.StructMap[thrift_item.class_name]: + if ( + thrift_item.class_name in config.StructMap + and "fields" in config.StructMap[thrift_item.class_name] + ): config_item = config.StructMap[thrift_item.class_name].fields for field in thrift_item.fields: if field.field_name in config_item: - field["proto_field_type"] = config_item[field.field_name].field_type + field["proto_field_type"] = config_item[ + field.field_name + ].field_type continue # For structs that have a single field in IDL but defined using type aliases in cpp @@ -131,7 +140,7 @@ def main(): if thrift_item.class_name in config.ConnectorStruct: thrift_item["connector"] = "true" del thrift_item["struct"] - + # For structs that need special implementations if thrift_item.class_name in config.Special: hfile = "./special/" + thrift_item.class_name + ".hpp.inc" @@ -140,8 +149,15 @@ def main(): cfile = "./special/" + thrift_item.class_name + ".cpp.inc" special_file(cfile, thrift_item, "cinc") - elif (thrift_item.class_name in pmap) or (thrift_item.class_name in config.StructMap and config.StructMap[thrift_item.class_name].class_name in pmap): - protocol_item = pmap[thrift_item.class_name] if thrift_item.class_name in pmap else pmap[config.StructMap[thrift_item.class_name].class_name] + elif (thrift_item.class_name in pmap) or ( + thrift_item.class_name in config.StructMap + and config.StructMap[thrift_item.class_name].class_name in pmap + ): + protocol_item = ( + pmap[thrift_item.class_name] + if thrift_item.class_name in pmap + else pmap[config.StructMap[thrift_item.class_name].class_name] + ) thrift_item["proto_name"] = protocol_item.class_name config_item = None @@ -156,7 +172,9 @@ def main(): if "struct" in protocol_item: verify(thrift_item, protocol_item) else: - eprint("Thrift item missing from presto_protocol: " + thrift_item.class_name) + eprint( + "Thrift item missing from presto_protocol: " + thrift_item.class_name + ) result.extend(thrift) print(util.to_json(result)) diff --git a/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.yml b/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.yml index 03f9edd0d63d8..b79b348f9878e 100644 --- a/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.yml +++ b/presto-native-execution/presto_cpp/main/thrift/presto_protocol-to-thrift-json.yml @@ -1,11 +1,11 @@ -# Licensed under the Apache License, Version 2.0 (the License); +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an AS IS BASIS, +# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. diff --git a/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift b/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift new file mode 100644 index 0000000000000..035e40d5b2823 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift @@ -0,0 +1,710 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace cpp2 facebook.presto.thrift + +enum TaskState { + PLANNED = 0, + RUNNING = 1, + FINISHED = 2, + CANCELED = 3, + ABORTED = 4, + FAILED = 5, +} +enum ErrorType { + USER_ERROR = 0, + INTERNAL_ERROR = 1, + INSUFFICIENT_RESOURCES = 2, + EXTERNAL = 3, +} +enum ErrorCause { + UNKNOWN = 0, + LOW_PARTITION_COUNT = 1, + EXCEEDS_BROADCAST_MEMORY_LIMIT = 2, +} +enum BufferState { + OPEN = 0, + NO_MORE_BUFFERS = 1, + NO_MORE_PAGES = 2, + FLUSHING = 3, + FINISHED = 4, + FAILED = 5, +} +enum BlockedReason { + WAITING_FOR_MEMORY = 0, +} +enum RuntimeUnit { + NONE = 0, + NANO = 1, + BYTE = 2, +} +enum JoinType { + INNER = 0, + PROBE_OUTER = 1, + LOOKUP_OUTER = 2, + FULL_OUTER = 3, +} +enum Type { + ROLE = 0, + ALL = 1, + NONE = 2, +} +enum Determinism { + DETERMINISTIC = 0, + NOT_DETERMINISTIC = 1, +} +enum NullCallClause { + RETURNS_NULL_ON_NULL_INPUT = 0, + CALLED_ON_NULL_INPUT = 1, +} +enum FunctionKind { + SCALAR = 0, + AGGREGATE = 1, + WINDOW = 2, +} +enum BufferType { + PARTITIONED = 0, + BROADCAST = 1, + ARBITRARY = 2, + DISCARDING = 3, + SPOOLING = 4, +} + +struct ConnectorSplit { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorTransactionHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorOutputTableHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorDeleteTableHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorInsertTableHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorTableHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct ConnectorTableLayoutHandle { + 1: optional string connectorId; + 2: optional binary customSerializedValue; + 3: optional string jsonValue; +} +struct RemoteTransactionHandle {} +struct Lifespan { + 1: bool grouped; + 2: i32 groupId; +} +struct ErrorLocation { + 1: i32 lineNumber; + 2: i32 columnNumber; +} +struct HostAddress { + 1: string hostPortString; +} +struct StageId { + 1: string queryId; + 2: i32 id; +} +struct OutputBufferId { + 1: i32 id; +} +struct PageBufferInfo { + 1: i32 partition; + 2: i64 bufferedPages; + 3: i64 bufferedBytes; + 4: i64 rowsAdded; + 5: i64 pagesAdded; +} +struct PlanNodeId { + 1: string id; +} +struct DistributionSnapshot { + 1: double maxError; + 2: double count; + 3: double total; + 4: i64 p01; + 5: i64 p05; + 6: i64 p10; + 7: i64 p25; + 8: i64 p50; + 9: i64 p75; + 10: i64 p90; + 11: i64 p95; + 12: i64 p99; + 13: i64 min; + 14: i64 max; + 15: double avg; +} +struct RuntimeStats { + 1: map metrics; +} +struct ExchangeClientStatus { + 1: i64 bufferedBytes; + 2: i64 maxBufferedBytes; + 3: i64 averageBytesPerRequest; + 4: i64 successfulRequestsCount; + 5: i32 bufferedPages; + 6: bool noMoreLocations; + 7: list pageBufferClientStatuses; +} +struct PageBufferClientStatus { + 1: string uri; + 2: string state; + 3: i64 lastUpdate; + 4: i64 rowsReceived; + 5: i32 pagesReceived; + 6: i64 rowsRejected; + 7: i32 pagesRejected; + 8: i32 requestsScheduled; + 9: i32 requestsCompleted; + 10: i32 requestsFailed; + 11: string httpRequestState; +} +struct LocalExchangeBufferInfo { + 1: i64 bufferedBytes; + 2: i32 bufferedPages; +} +struct TableFinishInfo { + 1: string serializedConnectorOutputMetadata; + 2: bool jsonLengthLimitExceeded; + 3: double statisticsWallTime; + 4: double statisticsCpuTime; +} +struct SplitOperatorInfo { + 1: map splitInfoMap; +} +struct HashCollisionsInfo { + 1: double weightedSumSquaredHashCollisions; + 2: double weightedHashCollisions; + 3: double weightedExpectedHashCollisions; +} +struct PartitionedOutputInfo { + 1: i64 rowsAdded; + 2: i64 pagesAdded; + 3: i64 outputBufferPeakMemoryUsage; +} +struct WindowInfo { + 1: list windowInfos; +} +struct DriverWindowInfo { + 1: double sumSquaredDifferencesPositionsOfIndex; + 2: double sumSquaredDifferencesSizeOfIndex; + 3: double sumSquaredDifferencesSizeInPartition; + 4: i64 totalPartitionsCount; + 5: i64 totalRowsCount; + 6: i64 numberOfIndexes; +} +struct TableWriterInfo { + 1: i64 pageSinkPeakMemoryUsage; + 2: double statisticsWallTime; + 3: double statisticsCpuTime; + 4: double validationCpuTime; +} +struct TableWriterMergeInfo { + 1: double statisticsWallTime; + 2: double statisticsCpuTime; +} +struct DynamicFilterStats { + 1: set producerNodeIds; +} +struct DriverStats { + 1: Lifespan lifespan; + 2: i64 createTimeInMillis; + 3: i64 startTimeInMillis; + 4: i64 endTimeInMillis; + 5: double queuedTime; + 6: double elapsedTime; + 7: i64 userMemoryReservationInBytes; + 8: i64 revocableMemoryReservationInBytes; + 9: i64 systemMemoryReservationInBytes; + 10: double totalScheduledTime; + 11: double totalCpuTime; + 12: double totalBlockedTime; + 13: bool fullyBlocked; + 14: set blockedReasons; + 15: i64 totalAllocationInBytes; + 16: i64 rawInputDataSizeInBytes; + 17: double rawInputReadTime; + 18: i64 rawInputPositions; + 19: i64 processedInputDataSizeInBytes; + 20: i64 processedInputPositions; + 21: i64 outputDataSizeInBytes; + 22: i64 outputPositions; + 23: i64 physicalWrittenDataSizeInBytes; + 24: list operatorStats; +} +struct TransactionId { + 1: string uuid; +} +struct TimeZoneKey { + 1: i16 timeZoneKey; +} +struct ResourceEstimates { + 1: optional double executionTime; + 2: optional double cpuTime; + 3: optional double peakMemory; + 4: optional double peakTaskMemory; +} +struct ConnectorId { + 1: string catalogName; +} +struct SqlFunctionId { + 1: string signature; +} +struct TypeSignature { + 1: string signature; + 2: bool ignore; +} +struct Language { + 1: string language; +} +struct QualifiedObjectName { + 1: string catalogName; + 2: string schemaName; + 3: string objectName; +} +struct TypeVariableConstraint { + 1: string name; + 2: bool comparableRequired; + 3: bool orderableRequired; + 4: string variadicBound; + 5: bool nonDecimalNumericRequired; +} +struct LongVariableConstraint { + 1: string name; + 2: string expression; +} +struct TaskSource { + 1: PlanNodeId planNodeId; + 2: set splits; + 3: set noMoreSplitsForLifespan; + 4: bool noMoreSplits; +} +struct SplitContext { + 1: bool cacheable; +} +struct OutputTableHandle { + 1: ConnectorId connectorId; + 2: ConnectorTransactionHandle transactionHandle; + 3: ConnectorOutputTableHandle connectorHandle; +} +struct SchemaTableName { + 1: string schema; + 2: string table; +} +struct InsertTableHandle { + 1: ConnectorId connectorId; + 2: ConnectorTransactionHandle transactionHandle; + 3: ConnectorInsertTableHandle connectorHandle; +} +struct DeleteTableHandle { + 1: ConnectorId connectorId; + 2: ConnectorTransactionHandle transactionHandle; + 3: ConnectorDeleteTableHandle connectorHandle; +} +struct RefreshMaterializedViewHandle { + 1: InsertTableHandle handle; + 2: SchemaTableName schemaTableName; +} +struct TableHandle { + 1: ConnectorId connectorId; + 2: ConnectorTableHandle connectorHandle; + 3: ConnectorTransactionHandle transaction; + 4: ConnectorTableLayoutHandle connectorTableLayout; +} +struct AnalyzeTableHandle { + 1: ConnectorId connectorId; + 2: ConnectorTransactionHandle transactionHandle; + 3: ConnectorTableHandle connectorHandle; +} +struct Location { + 1: string location; +} +struct TaskStatus { + 1: i64 taskInstanceIdLeastSignificantBits; + 2: i64 taskInstanceIdMostSignificantBits; + 3: i64 version; + 4: TaskState state; + 5: string selfUri; + 6: set completedDriverGroups; + 7: list failures; + 8: i32 queuedPartitionedDrivers; + 9: i32 runningPartitionedDrivers; + 10: double outputBufferUtilization; + 11: bool outputBufferOverutilized; + 12: i64 physicalWrittenDataSizeInBytes; + 13: i64 memoryReservationInBytes; + 14: i64 systemMemoryReservationInBytes; + 15: i64 fullGcCount; + 16: i64 fullGcTimeInMillis; + 17: i64 peakNodeTotalMemoryReservationInBytes; + 18: i64 totalCpuTimeInNanos; + 19: i64 taskAgeInMillis; + 20: i64 queuedPartitionedSplitsWeight; + 21: i64 runningPartitionedSplitsWeight; +} +struct ErrorCode { + 1: i32 code; + 2: string name; + 3: ErrorType type; + 4: bool retriable; +} +struct StageExecutionId { + 1: StageId stageId; + 2: i32 id; +} +struct OutputBufferInfo { + 1: string type; + 2: BufferState state; + 3: list buffers; + 4: bool canAddBuffers; + 5: bool canAddPages; + 6: i64 totalBufferedBytes; + 7: i64 totalBufferedPages; + 8: i64 totalRowsSent; + 9: i64 totalPagesSent; +} +struct BufferInfo { + 1: OutputBufferId bufferId; + 2: bool finished; + 3: i32 bufferedPages; + 4: i64 pagesSent; + 5: PageBufferInfo pageBufferInfo; +} +struct TaskStats { + 1: i64 createTimeInMillis; + 2: i64 firstStartTimeInMillis; + 3: i64 lastStartTimeInMillis; + 4: i64 lastEndTimeInMillis; + 5: i64 endTimeInMillis; + 6: i64 elapsedTimeInNanos; + 7: i64 queuedTimeInNanos; + 8: i32 totalDrivers; + 9: i32 queuedDrivers; + 10: i32 runningDrivers; + 11: i32 blockedDrivers; + 12: i32 completedDrivers; + 13: double cumulativeUserMemory; + 14: double cumulativeTotalMemory; + 15: i64 userMemoryReservationInBytes; + 16: i64 revocableMemoryReservationInBytes; + 17: i64 systemMemoryReservationInBytes; + 18: i64 peakUserMemoryInBytes; + 19: i64 peakTotalMemoryInBytes; + 20: i64 peakNodeTotalMemoryInBytes; + 21: i64 totalScheduledTimeInNanos; + 22: i64 totalCpuTimeInNanos; + 23: i64 totalBlockedTimeInNanos; + 24: bool fullyBlocked; + 25: set blockedReasons; + 26: i64 totalAllocationInBytes; + 27: i64 rawInputDataSizeInBytes; + 28: i64 rawInputPositions; + 29: i64 processedInputDataSizeInBytes; + 30: i64 processedInputPositions; + 31: i64 outputDataSizeInBytes; + 32: i64 outputPositions; + 33: i64 physicalWrittenDataSizeInBytes; + 34: list pipelines; + 35: i32 queuedPartitionedDrivers; + 36: i64 queuedPartitionedSplitsWeight; + 37: i32 runningPartitionedDrivers; + 38: i64 runningPartitionedSplitsWeight; + 39: i32 fullGcCount; + 40: i64 fullGcTimeInMillis; + 41: RuntimeStats runtimeStats; + 42: i32 totalSplits; + 43: i32 queuedSplits; + 44: i32 runningSplits; + 45: i32 completedSplits; + 46: i32 totalNewDrivers; + 47: i32 queuedNewDrivers; + 48: i32 runningNewDrivers; + 49: i32 completedNewDrivers; +} +struct PipelineStats { + 1: i32 pipelineId; + 2: i64 firstStartTimeInMillis; + 3: i64 lastStartTimeInMillis; + 4: i64 lastEndTimeInMillis; + 5: bool inputPipeline; + 6: bool outputPipeline; + 7: i32 totalDrivers; + 8: i32 queuedDrivers; + 9: i32 queuedPartitionedDrivers; + 10: i64 queuedPartitionedSplitsWeight; + 11: i32 runningDrivers; + 12: i32 runningPartitionedDrivers; + 13: i64 runningPartitionedSplitsWeight; + 14: i32 blockedDrivers; + 15: i32 completedDrivers; + 16: i64 userMemoryReservationInBytes; + 17: i64 revocableMemoryReservationInBytes; + 18: i64 systemMemoryReservationInBytes; + 19: DistributionSnapshot queuedTime; + 20: DistributionSnapshot elapsedTime; + 21: i64 totalScheduledTimeInNanos; + 22: i64 totalCpuTimeInNanos; + 23: i64 totalBlockedTimeInNanos; + 24: bool fullyBlocked; + 25: set blockedReasons; + 26: i64 totalAllocationInBytes; + 27: i64 rawInputDataSizeInBytes; + 28: i64 rawInputPositions; + 29: i64 processedInputDataSizeInBytes; + 30: i64 processedInputPositions; + 31: i64 outputDataSizeInBytes; + 32: i64 outputPositions; + 33: i64 physicalWrittenDataSizeInBytes; + 34: list operatorSummaries; + 35: list drivers; +} +struct RuntimeMetric { + 1: string name; + 2: i64 sum; + 3: i64 count; + 4: i64 max; + 5: i64 min; + 6: RuntimeUnit unit; +} +struct JoinOperatorInfo { + 1: JoinType joinType; + 2: list logHistogramProbes; + 3: list logHistogramOutput; + 4: optional i64 lookupSourcePositions; +} +struct SessionRepresentation { + 1: string queryId; + 2: optional TransactionId transactionId; + 3: bool clientTransactionSupport; + 4: string user; + 5: optional string principal; + 6: optional string source; + 7: optional string catalog; + 8: optional string schema; + 9: optional string traceToken; + 10: TimeZoneKey timeZoneKey; + 11: string locale; + 12: optional string remoteUserAddress; + 13: optional string userAgent; + 14: optional string clientInfo; + 15: set clientTags; + 16: ResourceEstimates resourceEstimates; + 17: i64 startTime; + 18: map systemProperties; + 19: map> catalogProperties; + 20: map> unprocessedCatalogProperties; + 21: map roles; + 22: map preparedStatements; + 23: map sessionFunctions; +} +struct SelectedRole { + 1: Type type; + 2: optional string role; +} +struct Parameter { + 1: string name; + 2: TypeSignature type; +} +struct RoutineCharacteristics { + 1: Language language; + 2: Determinism determinism; + 3: NullCallClause nullCallClause; +} +struct Signature { + 1: QualifiedObjectName name; + 2: FunctionKind kind; + 3: TypeSignature returnType; + 4: list argumentTypes; + 5: bool variableArity; + 6: list typeVariableConstraints; + 7: list longVariableConstraints; +} +struct Split { + 1: ConnectorId connectorId; + 2: ConnectorTransactionHandle transactionHandle; + 3: ConnectorSplit connectorSplit; + 4: Lifespan lifespan; + 5: SplitContext splitContext; +} +struct OutputBuffers { + 1: BufferType type; + 2: i64 version; + 3: bool noMoreBufferIds; + 4: map buffers; +} +struct CreateHandle { + 1: OutputTableHandle handle; + 2: SchemaTableName schemaTableName; +} +struct InsertHandle { + 1: InsertTableHandle handle; + 2: SchemaTableName schemaTableName; +} +struct DeleteHandle { + 1: DeleteTableHandle handle; + 2: SchemaTableName schemaTableName; +} +struct UpdateHandle { + 1: TableHandle handle; + 2: SchemaTableName schemaTableName; +} +struct ExecutionFailureInfo { + 1: string type; + 2: string message; + 3: optional ExecutionFailureInfo cause ( + cpp.ref_type = "shared", + drift.recursive_reference = true, + ); + 4: list suppressed; + 5: list stack; + 6: ErrorLocation errorLocation; + 7: ErrorCode errorCode; + 8: HostAddress remoteHost; + 9: ErrorCause errorCause; +} +struct TaskId { + 1: StageExecutionId stageExecutionId; + 2: i32 id; + 3: i32 attemptNumber; +} +union OperatorInfoUnion { + 1: ExchangeClientStatus exchangeClientStatus; + 2: LocalExchangeBufferInfo localExchangeBufferInfo; + 3: TableFinishInfo tableFinishInfo; + 4: SplitOperatorInfo splitOperatorInfo; + 5: HashCollisionsInfo hashCollisionsInfo; + 6: PartitionedOutputInfo partitionedOutputInfo; + 7: JoinOperatorInfo joinOperatorInfo; + 8: WindowInfo windowInfo; + 9: TableWriterInfo tableWriterInfo; + 10: TableWriterMergeInfo tableWriterMergeInfo; +} +struct SqlInvokedFunction { + 1: list parameters; + 2: string description; + 3: RoutineCharacteristics routineCharacteristics; + 4: string body; + 5: bool variableArity; + 6: Signature signature; + 7: SqlFunctionId functionId; +} +struct ScheduledSplit { + 1: i64 sequenceId; + 2: PlanNodeId planNodeId; + 3: Split split; +} +union ExecutionWriterTargetUnion { + 1: CreateHandle createHandle; + 2: InsertHandle insertHandle; + 3: DeleteHandle deleteHandle; + 4: RefreshMaterializedViewHandle refreshMaterializedViewHandle; + 5: UpdateHandle updateHandle; +} +struct TaskInfo { + 1: TaskId taskId; + 2: TaskStatus taskStatus; + 3: i64 lastHeartbeatInMillis; + 4: OutputBufferInfo outputBuffers; + 5: set noMoreSplits; + 6: TaskStats stats; + 7: bool needsPlan; + 8: string nodeId; +} +struct RemoteSplit { + 1: Location location; + 2: TaskId remoteSourceTaskId; +} +struct OperatorStats { + 1: i32 stageId; + 2: i32 stageExecutionId; + 3: i32 pipelineId; + 4: i32 operatorId; + 5: PlanNodeId planNodeId; + 6: string operatorType; + 7: i64 totalDrivers; + 8: i64 addInputCalls; + 9: double addInputWall; + 10: double addInputCpu; + 11: i64 addInputAllocationInBytes; + 12: i64 rawInputDataSizeInBytes; + 13: i64 rawInputPositions; + 14: i64 inputDataSizeInBytes; + 15: i64 inputPositions; + 16: double sumSquaredInputPositions; + 17: i64 getOutputCalls; + 18: double getOutputWall; + 19: double getOutputCpu; + 20: i64 getOutputAllocationInBytes; + 21: i64 outputDataSizeInBytes; + 22: i64 outputPositions; + 23: i64 physicalWrittenDataSizeInBytes; + 24: double additionalCpu; + 25: double blockedWall; + 26: i64 finishCalls; + 27: double finishWall; + 28: double finishCpu; + 29: i64 finishAllocationInBytes; + 30: i64 userMemoryReservationInBytes; + 31: i64 revocableMemoryReservationInBytes; + 32: i64 systemMemoryReservationInBytes; + 33: i64 peakUserMemoryReservationInBytes; + 34: i64 peakSystemMemoryReservationInBytes; + 35: i64 peakTotalMemoryReservationInBytes; + 36: i64 spilledDataSizeInBytes; + 37: RuntimeStats runtimeStats; + 38: optional BlockedReason blockedReason; + 39: OperatorInfoUnion infoUnion; + 40: i64 nullJoinBuildKeyCount; + 41: i64 joinBuildKeyCount; + 42: i64 nullJoinProbeKeyCount; + 43: i64 joinProbeKeyCount; + 44: DynamicFilterStats dynamicFilterStats; + 45: i64 isBlockedCalls; + 46: double isBlockedWall; + 47: double isBlockedCpu; + 48: i64 isBlockedAllocationInBytes; +} +struct TableWriteInfo { + 1: ExecutionWriterTargetUnion writerTargetUnion; + 2: optional AnalyzeTableHandle analyzeTableHandle; +} +struct TaskUpdateRequest { + 1: SessionRepresentation session; + 2: map extraCredentials; + 3: optional binary fragment; + 4: list sources; + 5: OutputBuffers outputIds; + 6: optional TableWriteInfo tableWriteInfo; +} + +service PrestoThrift { + void fake(); +} diff --git a/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt index 6dff496d1ac7b..f75598ff1d4d7 100644 --- a/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt @@ -10,20 +10,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable( - thrift_io_test - ThriftIOTest.cpp) +add_executable(thrift_io_test ThriftIOTest.cpp) -add_test( - NAME thrift_io_test - COMMAND thrift_io_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +add_test(NAME thrift_io_test COMMAND thrift_io_test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -target_include_directories(thrift_io_test PRIVATE - ${presto_thrift_INCLUDES} - ${presto_native_INCLUDES} - ${THRIFT_INCLUDES} - ${GLOG_INCLUDE_DIR}) +target_include_directories( + thrift_io_test + PRIVATE ${presto_thrift_INCLUDES} ${presto_native_INCLUDES} ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR} +) target_link_libraries( thrift_io_test @@ -39,7 +33,7 @@ target_link_libraries( ${RE2} GTest::gmock GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) -set_property(TARGET thrift_io_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET thrift_io_test PROPERTY JOB_POOL_LINK presto_link_job_pool) diff --git a/presto-native-execution/presto_cpp/main/thrift/thrift2json.py b/presto-native-execution/presto_cpp/main/thrift/thrift2json.py index 12eda1eaebbe6..16aaff969eac1 100755 --- a/presto-native-execution/presto_cpp/main/thrift/thrift2json.py +++ b/presto-native-execution/presto_cpp/main/thrift/thrift2json.py @@ -34,14 +34,14 @@ class attrdict(dict): def preprocess(file_path, output_temp_path): - with open(file_path, 'r') as file: + with open(file_path, "r") as file: lines = file.readlines() modified_lines = [] for line in lines: - modified_line = re.sub(r'\s*,*\s*drift\.recursive_reference=true\s*', '', line) - modified_line = re.sub(r'\(\s*\)$', '', modified_line) + modified_line = re.sub(r"\s*,*\s*drift\.recursive_reference=true\s*", "", line) + modified_line = re.sub(r"\(\s*\)$", "", modified_line) modified_lines.append(modified_line) - with open(output_temp_path, 'w') as file: + with open(output_temp_path, "w") as file: file.writelines(modified_lines) diff --git a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt index a129cf84a19d5..8f11b63083536 100644 --- a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt @@ -14,11 +14,9 @@ add_library(presto_type_converter OBJECT TypeParser.cpp) target_link_libraries(presto_type_converter velox_presto_type_parser) add_library(presto_velox_expr_conversion OBJECT PrestoToVeloxExpr.cpp) -target_link_libraries(presto_velox_expr_conversion velox_presto_types - velox_vector velox_exception) +target_link_libraries(presto_velox_expr_conversion velox_presto_types velox_vector velox_exception) -add_library(presto_types PrestoToVeloxQueryPlan.cpp VeloxPlanValidator.cpp - PrestoToVeloxSplit.cpp) +add_library(presto_types PrestoToVeloxQueryPlan.cpp VeloxPlanValidator.cpp PrestoToVeloxSplit.cpp) target_link_libraries( presto_types presto_velox_expr_conversion @@ -27,7 +25,8 @@ target_link_libraries( presto_type_converter presto_session_properties velox_type - velox_type_fbhive) + velox_type_fbhive +) set_property(TARGET presto_types PROPERTY JOB_POOL_LINK presto_link_job_pool) diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp index 0973a2f0ae3ca..b83f12ca0e36b 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp @@ -13,7 +13,6 @@ */ #include "presto_cpp/main/types/PrestoToVeloxExpr.h" - #include #include #include "presto_cpp/main/common/Configs.h" @@ -204,7 +203,8 @@ std::optional convertCastToVarcharWithMaxLength( static const std::string prestoDefaultNamespacePrefix = SystemConfig::instance()->prestoDefaultNamespacePrefix(); if (nullOnFailure) { - VELOX_UNSUPPORTED("TRY_CAST of varchar to {} is not supported.", returnType); + VELOX_UNSUPPORTED( + "TRY_CAST of varchar to {} is not supported.", returnType); } // Parse the max length from the return type string in the format of diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp index 227feb506fed7..0285c16ea48c7 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp @@ -29,12 +29,12 @@ #include "presto_cpp/main/SessionProperties.h" #include "presto_cpp/main/common/Utils.h" +#include "presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h" #include "presto_cpp/main/operators/BroadcastWrite.h" #include "presto_cpp/main/operators/PartitionAndSerialize.h" #include "presto_cpp/main/operators/ShuffleRead.h" #include "presto_cpp/main/operators/ShuffleWrite.h" #include "presto_cpp/main/types/TypeParser.h" -#include "presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h" using namespace facebook::velox; using namespace facebook::velox::exec; @@ -411,8 +411,9 @@ core::PlanNodePtr VeloxQueryPlanConverterBase::toVeloxQueryPlan( const auto desiredSourceOutput = toRowType(node->inputs[i], typeParser_); for (auto j = 0; j < outputType->size(); ++j) { - projections.emplace_back(std::make_shared( - outputType->childAt(j), desiredSourceOutput->nameOf(j))); + projections.emplace_back( + std::make_shared( + outputType->childAt(j), desiredSourceOutput->nameOf(j))); } sourceNodes[i] = std::make_shared( @@ -659,8 +660,9 @@ core::PlanNodePtr VeloxQueryPlanConverterBase::toVeloxQueryPlan( std::vector projections; projections.reserve(leftNames.size() + 1); for (auto i = 0; i < leftNames.size(); i++) { - projections.emplace_back(std::make_shared( - leftTypes[i], leftNames[i])); + projections.emplace_back( + std::make_shared( + leftTypes[i], leftNames[i])); } const bool constantValue = joinType.value() == core::JoinType::kLeftSemiFilter; @@ -1126,8 +1128,9 @@ VeloxQueryPlanConverterBase::toVeloxQueryPlan( std::vector groupingKeys; groupingKeys.reserve(node->groupingColumns.size()); for (const auto& [output, input] : node->groupingColumns) { - groupingKeys.emplace_back(core::GroupIdNode::GroupingKeyInfo{ - output.name, exprConverter_.toVeloxExpr(input)}); + groupingKeys.emplace_back( + core::GroupIdNode::GroupingKeyInfo{ + output.name, exprConverter_.toVeloxExpr(input)}); } return std::make_shared( diff --git a/presto-native-execution/presto_cpp/main/types/TypeParser.cpp b/presto-native-execution/presto_cpp/main/types/TypeParser.cpp index 15d7301c347d4..5a8ee896dc027 100644 --- a/presto-native-execution/presto_cpp/main/types/TypeParser.cpp +++ b/presto-native-execution/presto_cpp/main/types/TypeParser.cpp @@ -28,7 +28,8 @@ velox::TypePtr TypeParser::parse(const std::string& text) const { } } if (!SystemConfig::instance()->enumTypesEnabled()) { - if (text.find("BigintEnum") != std::string::npos || text.find("VarcharEnum") != std::string::npos) { + if (text.find("BigintEnum") != std::string::npos || + text.find("VarcharEnum") != std::string::npos) { VELOX_UNSUPPORTED("Unsupported type: {}", text); } } diff --git a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt index 2494dc2826e74..0e1d27d1b18a6 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt @@ -33,18 +33,24 @@ target_link_libraries( presto_type_converter presto_types velox_type_fbhive - velox_hive_partition_function) + velox_hive_partition_function +) -add_executable(presto_expressions_test RowExpressionTest.cpp ValuesPipeTest.cpp - PlanConverterTest.cpp TypeParserTest.cpp) +add_executable( + presto_expressions_test + RowExpressionTest.cpp + ValuesPipeTest.cpp + PlanConverterTest.cpp + TypeParserTest.cpp +) add_test( NAME presto_expressions_test COMMAND presto_expressions_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) -add_dependencies(presto_expressions_test presto_operators presto_protocol - presto_types) +add_dependencies(presto_expressions_test presto_operators presto_protocol presto_types) target_link_libraries( presto_expressions_test @@ -77,17 +83,18 @@ target_link_libraries( ${FOLLY_WITH_DEPENDENCIES} ${GLOG} ${GFLAGS_LIBRARIES} - pthread) + pthread +) -set_property(TARGET presto_expressions_test PROPERTY JOB_POOL_LINK - presto_link_job_pool) +set_property(TARGET presto_expressions_test PROPERTY JOB_POOL_LINK presto_link_job_pool) add_executable(presto_to_velox_connector_test PrestoToVeloxConnectorTest.cpp) add_test( NAME presto_to_velox_connector_test COMMAND presto_to_velox_connector_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) target_link_libraries( presto_to_velox_connector_test @@ -101,14 +108,16 @@ target_link_libraries( velox_tpch_connector velox_tpcds_connector GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) add_executable(presto_to_velox_query_plan_test PrestoToVeloxQueryPlanTest.cpp) add_test( NAME presto_to_velox_query_plan_test COMMAND presto_to_velox_query_plan_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) target_link_libraries( presto_to_velox_query_plan_test @@ -122,4 +131,5 @@ target_link_libraries( velox_hive_connector velox_tpch_connector GTest::gtest - GTest::gtest_main) + GTest::gtest_main +) diff --git a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp index 1f9ced81d675a..59913ae90e437 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp @@ -12,8 +12,8 @@ * limitations under the License. */ -#include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" #include +#include "presto_cpp/main/connectors/HivePrestoToVeloxConnector.h" #include "presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h" #include "presto_cpp/main/types/PrestoToVeloxExpr.h" #include "presto_cpp/presto_protocol/connector/hive/HiveConnectorProtocol.h" @@ -45,12 +45,15 @@ TEST_F(PrestoToVeloxConnectorTest, registerVariousConnectors) { connectorList; connectorList.emplace_back( std::pair("hive", std::make_unique("hive"))); - connectorList.emplace_back(std::pair( - "hive-hadoop2", + connectorList.emplace_back( + std::pair( + "hive-hadoop2", - std::make_unique("hive-hadoop2"))); - connectorList.emplace_back(std::pair( - "iceberg", std::make_unique("iceberg"))); + std::make_unique("hive-hadoop2"))); + connectorList.emplace_back( + std::pair( + "iceberg", + std::make_unique("iceberg"))); connectorList.emplace_back( std::pair("tpch", std::make_unique("tpch"))); diff --git a/presto-native-execution/presto_cpp/main/types/tests/RowExpressionTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/RowExpressionTest.cpp index 6cb1d7d100815..09d0066272136 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/RowExpressionTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/RowExpressionTest.cpp @@ -419,7 +419,8 @@ TEST_F(RowExpressionTest, varbinary5) { } TEST_F(RowExpressionTest, char) { - SystemConfig::instance()->setValue(std::string(SystemConfig::kCharNToVarcharImplicitCast), "true"); + SystemConfig::instance()->setValue( + std::string(SystemConfig::kCharNToVarcharImplicitCast), "true"); std::string str = R"##( { "@type": "constant", @@ -656,8 +657,8 @@ TEST_F(RowExpressionTest, castToVarchar) { ASSERT_NE(returnExpr, nullptr); ASSERT_EQ(returnExpr->name(), "presto.default.substr"); - auto returnArg1 = std::dynamic_pointer_cast( - returnExpr->inputs()[0]); + auto returnArg1 = + std::dynamic_pointer_cast(returnExpr->inputs()[0]); auto returnArg2 = std::dynamic_pointer_cast( returnExpr->inputs()[1]); auto returnArg3 = std::dynamic_pointer_cast( @@ -1024,8 +1025,7 @@ TEST_F(RowExpressionTest, likeWithEscape) { ASSERT_NE(callExpr, nullptr); auto callExprToString = callExpr->toString(); - ASSERT_EQ( - callExpr->toString(), "presto.default.like(\"type\",%BRASS,#)"); + ASSERT_EQ(callExpr->toString(), "presto.default.like(\"type\",%BRASS,#)"); } TEST_F(RowExpressionTest, dereference) { diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp index b6be024e5527d..b341789aab7b3 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp @@ -19,21 +19,23 @@ using namespace std; namespace facebook::presto::test::utils { namespace { - const std::string applyClionDirFix(std::string& currentPath, const std::string& fileName) { - // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ - // directory. Hard-coded json files are not copied there and test fails with - // file not found. Fixing the path so that we can trigger these tests from - // CLion. - boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); - boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); - - return currentPath + "/data/" + fileName; - } +const std::string applyClionDirFix( + std::string& currentPath, + const std::string& fileName) { + // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ + // directory. Hard-coded json files are not copied there and test fails with + // file not found. Fixing the path so that we can trigger these tests from + // CLion. + boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); + boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); + + return currentPath + "/data/" + fileName; } +} // namespace const std::string getDataPath(const std::string& fileName) { std::string currentPath = boost::filesystem::current_path().c_str(); - if (boost::algorithm::ends_with(currentPath, "fbcode")) { + if (boost::algorithm::ends_with(currentPath, "fbcode")) { return currentPath + "/github/presto-trunk/presto-native-execution/presto_cpp/main/types/tests/data/" + fileName; @@ -46,7 +48,9 @@ const std::string getDataPath(const std::string& fileName) { return applyClionDirFix(currentPath, fileName); } -const std::string getDataPath(const std::string& testDataDir, const std::string& fileName) { +const std::string getDataPath( + const std::string& testDataDir, + const std::string& fileName) { std::string currentPath = boost::filesystem::current_path().c_str(); if (boost::algorithm::ends_with(currentPath, "fbcode")) { return currentPath + testDataDir + fileName; diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h index 652648bef068a..660c43f03784c 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h @@ -13,5 +13,7 @@ */ namespace facebook::presto::test::utils { const std::string getDataPath(const std::string& fileName); -const std::string getDataPath(const std::string& testDataDir, const std::string& fileName); -} +const std::string getDataPath( + const std::string& testDataDir, + const std::string& fileName); +} // namespace facebook::presto::test::utils diff --git a/presto-native-execution/presto_cpp/main/types/tests/TypeParserTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/TypeParserTest.cpp index ee9036e5943bc..bda24b62cc998 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/TypeParserTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/TypeParserTest.cpp @@ -17,47 +17,53 @@ #include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/common/tests/MutableConfigs.h" #include "presto_cpp/main/types/TypeParser.h" -#include "velox/common/file/FileSystems.h" #include "velox/common/base/tests/GTestUtils.h" +#include "velox/common/file/FileSystems.h" #include "velox/functions/prestosql/types/BigintEnumRegistration.h" #include "velox/functions/prestosql/types/VarcharEnumRegistration.h" - using namespace facebook::presto; using namespace facebook::velox; class TypeParserTest : public ::testing::Test { void SetUp() override { filesystems::registerLocalFileSystem(); - test::setupMutableSystemConfig(); + test::setupMutableSystemConfig(); registerBigintEnumType(); registerVarcharEnumType(); } }; // Test basical functionality of TypeParser. -// More detailed tests for Presto TypeParser are in velox/functions/prestosql/types/parser/tests/TypeParserTest. +// More detailed tests for Presto TypeParser are in +// velox/functions/prestosql/types/parser/tests/TypeParserTest. TEST_F(TypeParserTest, parseEnumTypes) { TypeParser typeParser = TypeParser(); ASSERT_EQ( - typeParser.parse( - "test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})")->toString(), - "test.enum.mood:BigintEnum({\"CURIOUS\": 2, \"HAPPY\": 0})"); + typeParser + .parse( + "test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})") + ->toString(), + "test.enum.mood:BigintEnum({\"CURIOUS\": 2, \"HAPPY\": 0})"); ASSERT_EQ( - typeParser.parse( - "test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})")->toString(), - "test.enum.mood:VarcharEnum({\"CURIOUS\": \"someValue\", \"HAPPY\": \"some value\", \"SAD\": \"SOME VALUE\"})"); + typeParser + .parse( + "test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})") + ->toString(), + "test.enum.mood:VarcharEnum({\"CURIOUS\": \"someValue\", \"HAPPY\": \"some value\", \"SAD\": \"SOME VALUE\"})"); - // When set to false, TypeParser will throw an unsupported error when it receives an enum type. - SystemConfig::instance()->setValue(std::string(SystemConfig::kEnumTypesEnabled), "false"); + // When set to false, TypeParser will throw an unsupported error when it + // receives an enum type. + SystemConfig::instance()->setValue( + std::string(SystemConfig::kEnumTypesEnabled), "false"); VELOX_ASSERT_THROW( - typeParser.parse( - "test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})"), - "Unsupported type: test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})"); + typeParser.parse( + "test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})"), + "Unsupported type: test.enum.mood:BigintEnum(test.enum.mood{\"CURIOUS\":2, \"HAPPY\":0})"); VELOX_ASSERT_THROW( - typeParser.parse( - "test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})"), - "Unsupported type: test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})"); + typeParser.parse( + "test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})"), + "Unsupported type: test.enum.mood:VarcharEnum(test.enum.mood{\"CURIOUS\":\"ONXW2ZKWMFWHKZI=\", \"HAPPY\":\"ONXW2ZJAOZQWY5LF\" , \"SAD\":\"KNHU2RJAKZAUYVKF\"})"); } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/PartitionedOutput.json b/presto-native-execution/presto_cpp/main/types/tests/data/PartitionedOutput.json index 4c7d5dd63b611..e7e9c23b62285 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/PartitionedOutput.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/PartitionedOutput.json @@ -481,4 +481,4 @@ "totalLifespans":1 }, "outputTableWriterFragment":false -} \ No newline at end of file +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ValuesNode.json b/presto-native-execution/presto_cpp/main/types/tests/data/ValuesNode.json index 4feaf38f2c304..61e37f9d1fffe 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/ValuesNode.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ValuesNode.json @@ -51,4 +51,4 @@ } ] ] -} \ No newline at end of file +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ValuesPipeTest.json b/presto-native-execution/presto_cpp/main/types/tests/data/ValuesPipeTest.json index b172ec8bf5952..7a84686d163bf 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/ValuesPipeTest.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ValuesPipeTest.json @@ -219,4 +219,3 @@ "outputTableWriterFragment": false, "jsonRepresentation": "{\n \"id\" : \"9\",\n \"name\" : \"Output\",\n \"identifier\" : \"[a, b]\",\n \"details\" : \"a := field\\nb := field_0\\n\",\n \"children\" : [ {\n \"id\" : \"4\",\n \"name\" : \"Filter\",\n \"identifier\" : \"[filterPredicate = (field) = (INTEGER 1)]\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"133\",\n \"name\" : \"LocalExchange\",\n \"identifier\" : \"[ROUND_ROBIN] ()\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"0\",\n \"name\" : \"Values\",\n \"identifier\" : \"\",\n \"details\" : \"(INTEGER 1, VARCHAR(1) a)\\n(INTEGER 2, VARCHAR(1) b)\\n(INTEGER 3, VARCHAR(1) c)\\n\",\n \"children\" : [ ],\n \"remoteSources\" : [ ]\n } ],\n \"remoteSources\" : [ ]\n } ],\n \"remoteSources\" : [ ]\n } ],\n \"remoteSources\" : [ ]\n}" } - diff --git a/presto-native-execution/presto_cpp/presto_protocol/CMakeLists.txt b/presto-native-execution/presto_cpp/presto_protocol/CMakeLists.txt index 15ebb198c164a..d29267e434410 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/presto_protocol/CMakeLists.txt @@ -10,8 +10,14 @@ # See the License for the specific language governing permissions and # limitations under the License. add_library( - presto_protocol OBJECT presto_protocol.cpp Base64Util.cpp core/DataSize.cpp - core/Duration.cpp core/ConnectorProtocol.cpp) + presto_protocol + OBJECT + presto_protocol.cpp + Base64Util.cpp + core/DataSize.cpp + core/Duration.cpp + core/ConnectorProtocol.cpp +) target_link_libraries(presto_protocol velox_type velox_presto_serializer ${RE2}) diff --git a/presto-native-execution/presto_cpp/presto_protocol/Makefile b/presto-native-execution/presto_cpp/presto_protocol/Makefile index 09b43df28b4f5..8576bd741ba2f 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/Makefile +++ b/presto-native-execution/presto_cpp/presto_protocol/Makefile @@ -52,7 +52,7 @@ presto_protocol-cpp: presto_protocol-json chevron -d connector/arrow_flight/presto_protocol_arrow_flight.json connector/arrow_flight/presto_protocol-json-hpp.mustache >> connector/arrow_flight/presto_protocol_arrow_flight.h clang-format -style=file -i connector/arrow_flight/presto_protocol_arrow_flight.h connector/arrow_flight/presto_protocol_arrow_flight.cpp -presto_protocol-json: +presto_protocol-json: ./java-to-struct-json.py --config core/presto_protocol_core.yml core/special/*.java core/special/*.inc -j | jq . > core/presto_protocol_core.json ./java-to-struct-json.py --config connector/hive/presto_protocol_hive.yml connector/hive/special/*.inc -j | jq . > connector/hive/presto_protocol_hive.json ./java-to-struct-json.py --config connector/iceberg/presto_protocol_iceberg.yml connector/iceberg/special/*.inc -j | jq . > connector/iceberg/presto_protocol_iceberg.json diff --git a/presto-native-execution/presto_cpp/presto_protocol/README.md b/presto-native-execution/presto_cpp/presto_protocol/README.md index d4117938e45d2..579449aca04f8 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/README.md +++ b/presto-native-execution/presto_cpp/presto_protocol/README.md @@ -41,7 +41,7 @@ The `presto_protocol.yml` config file is further described below. [mustache](http://mustache.github.io/mustache.5.html) files `presto_protocol-json-hpp.mustache`, `presto_protocol-json-cpp.mustache` and generates the C++ protocol files `presto_protocol.h`, `presto_protocol.cpp`. - + ### presto_protocol.yml The config file contains the following yaml format entries. @@ -63,4 +63,4 @@ To properly order the resulting C++ declarations, the `.inc` files can be annota with `//dependency` comments. These comments may appear in two places. Annotation following a field declaration creates a dependency of this class on the data type of that field. Annotations of the form `//dependency other_class` -create a dependency of this class on `other_class`. \ No newline at end of file +create a dependency of this class on `other_class`. diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/hive/presto_protocol_hive.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/hive/presto_protocol_hive.cpp index 8011da82eee47..24e90b78e2e8c 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/hive/presto_protocol_hive.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/hive/presto_protocol_hive.cpp @@ -370,9 +370,10 @@ namespace facebook::presto::protocol::hive { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - BucketFunctionType_enum_table[] = { // NOLINT: cert-err58-cpp - {BucketFunctionType::HIVE_COMPATIBLE, "HIVE_COMPATIBLE"}, - {BucketFunctionType::PRESTO_NATIVE, "PRESTO_NATIVE"}}; + BucketFunctionType_enum_table[] = + { // NOLINT: cert-err58-cpp + {BucketFunctionType::HIVE_COMPATIBLE, "HIVE_COMPATIBLE"}, + {BucketFunctionType::PRESTO_NATIVE, "PRESTO_NATIVE"}}; void to_json(json& j, const BucketFunctionType& e) { static_assert( std::is_enum::value, @@ -598,12 +599,13 @@ namespace facebook::presto::protocol::hive { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - HiveCompressionCodec_enum_table[] = { // NOLINT: cert-err58-cpp - {HiveCompressionCodec::NONE, "NONE"}, - {HiveCompressionCodec::SNAPPY, "SNAPPY"}, - {HiveCompressionCodec::GZIP, "GZIP"}, - {HiveCompressionCodec::LZ4, "LZ4"}, - {HiveCompressionCodec::ZSTD, "ZSTD"}}; + HiveCompressionCodec_enum_table[] = + { // NOLINT: cert-err58-cpp + {HiveCompressionCodec::NONE, "NONE"}, + {HiveCompressionCodec::SNAPPY, "SNAPPY"}, + {HiveCompressionCodec::GZIP, "GZIP"}, + {HiveCompressionCodec::LZ4, "LZ4"}, + {HiveCompressionCodec::ZSTD, "ZSTD"}}; void to_json(json& j, const HiveCompressionCodec& e) { static_assert( std::is_enum::value, diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol-json-hpp.mustache b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol-json-hpp.mustache index 47a338f8459e0..4527c9716a74d 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol-json-hpp.mustache +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol-json-hpp.mustache @@ -58,4 +58,4 @@ namespace facebook::presto::protocol::iceberg { } {{/enum}} {{/hinc}} -{{/.}} \ No newline at end of file +{{/.}} diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp index 3229da2e88d07..6d03a5ce52b12 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.cpp @@ -25,11 +25,12 @@ namespace facebook::presto::protocol::iceberg { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - ChangelogOperation_enum_table[] = { // NOLINT: cert-err58-cpp - {ChangelogOperation::INSERT, "INSERT"}, - {ChangelogOperation::DELETE, "DELETE"}, - {ChangelogOperation::UPDATE_BEFORE, "UPDATE_BEFORE"}, - {ChangelogOperation::UPDATE_AFTER, "UPDATE_AFTER"}}; + ChangelogOperation_enum_table[] = + { // NOLINT: cert-err58-cpp + {ChangelogOperation::INSERT, "INSERT"}, + {ChangelogOperation::DELETE, "DELETE"}, + {ChangelogOperation::UPDATE_BEFORE, "UPDATE_BEFORE"}, + {ChangelogOperation::UPDATE_AFTER, "UPDATE_AFTER"}}; void to_json(json& j, const ChangelogOperation& e) { static_assert( std::is_enum::value, @@ -508,14 +509,15 @@ namespace facebook::presto::protocol::iceberg { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - PartitionTransformType_enum_table[] = { // NOLINT: cert-err58-cpp - {PartitionTransformType::IDENTITY, "IDENTITY"}, - {PartitionTransformType::YEAR, "YEAR"}, - {PartitionTransformType::MONTH, "MONTH"}, - {PartitionTransformType::DAY, "DAY"}, - {PartitionTransformType::HOUR, "HOUR"}, - {PartitionTransformType::BUCKET, "BUCKET"}, - {PartitionTransformType::TRUNCATE, "TRUNCATE"}}; + PartitionTransformType_enum_table[] = + { // NOLINT: cert-err58-cpp + {PartitionTransformType::IDENTITY, "IDENTITY"}, + {PartitionTransformType::YEAR, "YEAR"}, + {PartitionTransformType::MONTH, "MONTH"}, + {PartitionTransformType::DAY, "DAY"}, + {PartitionTransformType::HOUR, "HOUR"}, + {PartitionTransformType::BUCKET, "BUCKET"}, + {PartitionTransformType::TRUNCATE, "TRUNCATE"}}; void to_json(json& j, const PartitionTransformType& e) { static_assert( std::is_enum::value, diff --git a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml index f04db9b31ce3f..1a8be3d90b3b6 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.yml @@ -72,4 +72,3 @@ JavaClasses: - presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogOperation.java - presto-iceberg/src/main/java/com/facebook/presto/iceberg/changelog/ChangelogSplitInfo.java - presto-iceberg/src/main/java/com/facebook/presto/iceberg/SortField.java - diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h b/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h index f349c1a6644af..d51a20dc496a0 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/ConnectorProtocol.h @@ -41,8 +41,8 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json( json& j, @@ -54,8 +54,8 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json(json& j, const std::shared_ptr& p) const = 0; @@ -72,8 +72,8 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json( json& j, @@ -85,19 +85,19 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json(json& j, const std::shared_ptr& p) const = 0; virtual void from_json(const json& j, std::shared_ptr& p) const = 0; virtual void serialize( - const std::shared_ptr& proto, - std::string& thrift) const = 0; + const std::shared_ptr& proto, + std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json( json& j, @@ -116,8 +116,8 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; virtual void to_json( json& j, @@ -129,12 +129,11 @@ class ConnectorProtocol { const std::shared_ptr& proto, std::string& thrift) const = 0; virtual void deserialize( - const std::string& thrift, - std::shared_ptr& proto) const = 0; + const std::string& thrift, + std::shared_ptr& proto) const = 0; - virtual void to_json( - json& j, - const std::shared_ptr& p) const = 0; + virtual void to_json(json& j, const std::shared_ptr& p) + const = 0; virtual void from_json( const json& j, std::shared_ptr& p) const = 0; @@ -290,9 +289,8 @@ class ConnectorProtocolTemplate final : public ConnectorProtocol { const final { to_json_template(j, p); } - void from_json( - const json& j, - std::shared_ptr& p) const final { + void from_json(const json& j, std::shared_ptr& p) + const final { from_json_template(j, p); } void serialize( @@ -310,9 +308,8 @@ class ConnectorProtocolTemplate final : public ConnectorProtocol { const final { to_json_template(j, p); } - void from_json( - const json& j, - std::shared_ptr& p) const final { + void from_json(const json& j, std::shared_ptr& p) + const final { from_json_template(j, p); } @@ -359,39 +356,39 @@ class ConnectorProtocolTemplate final : public ConnectorProtocol { template static void serializeTemplate( - const std::shared_ptr&, - std::string&, - typename std::enable_if< - std::is_same::value, - BASE>::type* = 0) { - VELOX_NYI("Not implemented: {}", typeid(BASE).name()); + const std::shared_ptr&, + std::string&, + typename std::enable_if< + std::is_same::value, + BASE>::type* = 0) { + VELOX_NYI("Not implemented: {}", typeid(BASE).name()); } template static void deserializeTemplate( - const std::string&, - std::shared_ptr&, - typename std::enable_if< - std::is_same::value, - BASE>::type* = 0) { - VELOX_NYI("Not implemented: {}", typeid(BASE).name()); + const std::string&, + std::shared_ptr&, + typename std::enable_if< + std::is_same::value, + BASE>::type* = 0) { + VELOX_NYI("Not implemented: {}", typeid(BASE).name()); } template static void serializeTemplate( - const std::shared_ptr& proto, - std::string& thrift, - typename std::enable_if::value, BASE>:: - type* = 0) { + const std::shared_ptr& proto, + std::string& thrift, + typename std::enable_if::value, BASE>:: + type* = 0) { auto derived = *std::static_pointer_cast(proto); thrift = derived.serialize(derived); } template static void deserializeTemplate( - const std::string& thrift, - std::shared_ptr& proto, - typename std::enable_if::value, BASE>:: - type* = 0) { + const std::string& thrift, + std::shared_ptr& proto, + typename std::enable_if::value, BASE>:: + type* = 0) { std::shared_ptr derived; proto = derived->deserialize(thrift, derived); } diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index efb585849ef31..874e2275577c1 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -36,10 +36,11 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - NodeSelectionStrategy_enum_table[] = { // NOLINT: cert-err58-cpp - {NodeSelectionStrategy::HARD_AFFINITY, "HARD_AFFINITY"}, - {NodeSelectionStrategy::SOFT_AFFINITY, "SOFT_AFFINITY"}, - {NodeSelectionStrategy::NO_PREFERENCE, "NO_PREFERENCE"}}; + NodeSelectionStrategy_enum_table[] = + { // NOLINT: cert-err58-cpp + {NodeSelectionStrategy::HARD_AFFINITY, "HARD_AFFINITY"}, + {NodeSelectionStrategy::SOFT_AFFINITY, "SOFT_AFFINITY"}, + {NodeSelectionStrategy::NO_PREFERENCE, "NO_PREFERENCE"}}; void to_json(json& j, const NodeSelectionStrategy& e) { static_assert( std::is_enum::value, @@ -558,11 +559,12 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - AggregationNodeStep_enum_table[] = { // NOLINT: cert-err58-cpp - {AggregationNodeStep::PARTIAL, "PARTIAL"}, - {AggregationNodeStep::FINAL, "FINAL"}, - {AggregationNodeStep::INTERMEDIATE, "INTERMEDIATE"}, - {AggregationNodeStep::SINGLE, "SINGLE"}}; + AggregationNodeStep_enum_table[] = + { // NOLINT: cert-err58-cpp + {AggregationNodeStep::PARTIAL, "PARTIAL"}, + {AggregationNodeStep::FINAL, "FINAL"}, + {AggregationNodeStep::INTERMEDIATE, "INTERMEDIATE"}, + {AggregationNodeStep::SINGLE, "SINGLE"}}; void to_json(json& j, const AggregationNodeStep& e) { static_assert( std::is_enum::value, @@ -2808,10 +2810,11 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - BuiltInFunctionKind_enum_table[] = { // NOLINT: cert-err58-cpp - {BuiltInFunctionKind::ENGINE, "ENGINE"}, - {BuiltInFunctionKind::PLUGIN, "PLUGIN"}, - {BuiltInFunctionKind::WORKER, "WORKER"}}; + BuiltInFunctionKind_enum_table[] = + { // NOLINT: cert-err58-cpp + {BuiltInFunctionKind::ENGINE, "ENGINE"}, + {BuiltInFunctionKind::PLUGIN, "PLUGIN"}, + {BuiltInFunctionKind::WORKER, "WORKER"}}; void to_json(json& j, const BuiltInFunctionKind& e) { static_assert( std::is_enum::value, @@ -6144,9 +6147,10 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - JoinDistributionType_enum_table[] = { // NOLINT: cert-err58-cpp - {JoinDistributionType::PARTITIONED, "PARTITIONED"}, - {JoinDistributionType::REPLICATED, "REPLICATED"}}; + JoinDistributionType_enum_table[] = + { // NOLINT: cert-err58-cpp + {JoinDistributionType::PARTITIONED, "PARTITIONED"}, + {JoinDistributionType::REPLICATED, "REPLICATED"}}; void to_json(json& j, const JoinDistributionType& e) { static_assert( std::is_enum::value, @@ -8211,14 +8215,17 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - StageExecutionStrategy_enum_table[] = { // NOLINT: cert-err58-cpp - {StageExecutionStrategy::UNGROUPED_EXECUTION, "UNGROUPED_EXECUTION"}, - {StageExecutionStrategy::FIXED_LIFESPAN_SCHEDULE_GROUPED_EXECUTION, - "FIXED_LIFESPAN_SCHEDULE_GROUPED_EXECUTION"}, - {StageExecutionStrategy::DYNAMIC_LIFESPAN_SCHEDULE_GROUPED_EXECUTION, - "DYNAMIC_LIFESPAN_SCHEDULE_GROUPED_EXECUTION"}, - {StageExecutionStrategy::RECOVERABLE_GROUPED_EXECUTION, - "RECOVERABLE_GROUPED_EXECUTION"}}; + StageExecutionStrategy_enum_table[] = + { // NOLINT: cert-err58-cpp + {StageExecutionStrategy::UNGROUPED_EXECUTION, + "UNGROUPED_EXECUTION"}, + {StageExecutionStrategy::FIXED_LIFESPAN_SCHEDULE_GROUPED_EXECUTION, + "FIXED_LIFESPAN_SCHEDULE_GROUPED_EXECUTION"}, + {StageExecutionStrategy:: + DYNAMIC_LIFESPAN_SCHEDULE_GROUPED_EXECUTION, + "DYNAMIC_LIFESPAN_SCHEDULE_GROUPED_EXECUTION"}, + {StageExecutionStrategy::RECOVERABLE_GROUPED_EXECUTION, + "RECOVERABLE_GROUPED_EXECUTION"}}; void to_json(json& j, const StageExecutionStrategy& e) { static_assert( std::is_enum::value, @@ -9930,12 +9937,13 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - SystemPartitionFunction_enum_table[] = { // NOLINT: cert-err58-cpp - {SystemPartitionFunction::SINGLE, "SINGLE"}, - {SystemPartitionFunction::HASH, "HASH"}, - {SystemPartitionFunction::ROUND_ROBIN, "ROUND_ROBIN"}, - {SystemPartitionFunction::BROADCAST, "BROADCAST"}, - {SystemPartitionFunction::UNKNOWN, "UNKNOWN"}}; + SystemPartitionFunction_enum_table[] = + { // NOLINT: cert-err58-cpp + {SystemPartitionFunction::SINGLE, "SINGLE"}, + {SystemPartitionFunction::HASH, "HASH"}, + {SystemPartitionFunction::ROUND_ROBIN, "ROUND_ROBIN"}, + {SystemPartitionFunction::BROADCAST, "BROADCAST"}, + {SystemPartitionFunction::UNKNOWN, "UNKNOWN"}}; void to_json(json& j, const SystemPartitionFunction& e) { static_assert( std::is_enum::value, @@ -9972,13 +9980,14 @@ namespace facebook::presto::protocol { // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays static const std::pair - SystemPartitioning_enum_table[] = { // NOLINT: cert-err58-cpp - {SystemPartitioning::SINGLE, "SINGLE"}, - {SystemPartitioning::FIXED, "FIXED"}, - {SystemPartitioning::SOURCE, "SOURCE"}, - {SystemPartitioning::SCALED, "SCALED"}, - {SystemPartitioning::COORDINATOR_ONLY, "COORDINATOR_ONLY"}, - {SystemPartitioning::ARBITRARY, "ARBITRARY"}}; + SystemPartitioning_enum_table[] = + { // NOLINT: cert-err58-cpp + {SystemPartitioning::SINGLE, "SINGLE"}, + {SystemPartitioning::FIXED, "FIXED"}, + {SystemPartitioning::SOURCE, "SOURCE"}, + {SystemPartitioning::SCALED, "SCALED"}, + {SystemPartitioning::COORDINATOR_ONLY, "COORDINATOR_ONLY"}, + {SystemPartitioning::ARBITRARY, "ARBITRARY"}}; void to_json(json& j, const SystemPartitioning& e) { static_assert( std::is_enum::value, diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/special/SystemPartitioningHandle.java b/presto-native-execution/presto_cpp/presto_protocol/core/special/SystemPartitioningHandle.java index a2abcdb8cd356..86b0a52a1831b 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/special/SystemPartitioningHandle.java +++ b/presto-native-execution/presto_cpp/presto_protocol/core/special/SystemPartitioningHandle.java @@ -187,7 +187,7 @@ public enum SystemPartitionFunction HASH , ROUND_ROBIN , BROADCAST , - UNKNOWN + UNKNOWN } public abstract BucketFunction createBucketFunction(List partitionChannelTypes, boolean isHashPrecomputed, int bucketCount); diff --git a/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py b/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py index 439f47e58f584..9ac2d4eb4e2dd 100755 --- a/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py +++ b/presto-native-execution/presto_cpp/presto_protocol/java-to-struct-json.py @@ -13,7 +13,6 @@ # import argparse -import json import os import sys from collections import defaultdict @@ -114,7 +113,7 @@ def add_field( field_local = False for key, value in lang.items(): - if type(value) == str: + if isinstance(value, str): field_text = re.sub(key, value, field_text) else: field_text, n = re.subn(key, value["replace"], field_text) @@ -342,7 +341,6 @@ def main(): subclasses = {} for abstract_name, abstract_value in config.AbstractClasses.items(): - classes[abstract_name].class_name = abstract_name classes[abstract_name].field_name = member_name(abstract_name) classes[abstract_name].abstract = True diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol-protobuf.mustache b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol-protobuf.mustache index 6fe747b225f0d..5a145cf1df350 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol-protobuf.mustache +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol-protobuf.mustache @@ -29,4 +29,3 @@ enum {{class_name}} { {{/enum}} {{/.}} - diff --git a/presto-native-execution/presto_cpp/presto_protocol/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/presto_protocol/tests/CMakeLists.txt index 893ab732a597c..6011a8e4f3949 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/presto_protocol/tests/CMakeLists.txt @@ -27,11 +27,13 @@ add_executable( TupleDomainTest.cpp TypeErrorTest.cpp VariableReferenceExpressionTest.cpp - PlanFragmentTest.cpp) + PlanFragmentTest.cpp +) add_test( NAME presto_protocol_test COMMAND presto_protocol_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) add_dependencies(presto_protocol_test presto_protocol) target_link_libraries( @@ -50,4 +52,5 @@ target_link_libraries( ${DOUBLE_CONVERSION} ${GLOG} ${GFLAGS_LIBRARIES} - pthread) + pthread +) diff --git a/presto-native-execution/presto_cpp/presto_protocol/tests/PlanFragmentTest.cpp b/presto-native-execution/presto_cpp/presto_protocol/tests/PlanFragmentTest.cpp index 48da464e3f4b4..956094c87690b 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/tests/PlanFragmentTest.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/tests/PlanFragmentTest.cpp @@ -33,7 +33,9 @@ void testJsonRoundTrip(const std::string& str) { template void testJsonRoundTripFile(const std::string& filename) { - testJsonRoundTrip(slurp(getDataPath("/github/presto-trunk/presto-native-execution/presto_cpp/presto_protocol/tests/data/", filename))); + testJsonRoundTrip(slurp(getDataPath( + "/github/presto-trunk/presto-native-execution/presto_cpp/presto_protocol/tests/data/", + filename))); } } // namespace diff --git a/presto-native-execution/presto_cpp/presto_protocol/tests/RowExpressionTest.cpp b/presto-native-execution/presto_cpp/presto_protocol/tests/RowExpressionTest.cpp index 6ef1f31e311ab..5459b60ef4eab 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/tests/RowExpressionTest.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/tests/RowExpressionTest.cpp @@ -68,7 +68,7 @@ class RowExpressionTest : public ::testing::Test {}; TEST_F(RowExpressionTest, constant) { std::string str = R"( { - "@type": "constant", + "@type": "constant", "valueBlock":"CgAAAExPTkdfQVJSQVkBAAAAAAAAAAAAAAAA", "type":"bigint" } diff --git a/presto-native-execution/presto_cpp/presto_protocol/tests/TupleDomainTest.cpp b/presto-native-execution/presto_cpp/presto_protocol/tests/TupleDomainTest.cpp index 38357cfd36020..51b07d9cba962 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/tests/TupleDomainTest.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/tests/TupleDomainTest.cpp @@ -22,7 +22,7 @@ class TestTupleDomain : public ::testing::Test {}; TEST_F(TestTupleDomain, TestTupleDomainSubfield) { std::string str = R"( - { "columnDomains":[ + { "columnDomains":[ { "column":"valid_experiment_reading", "domain":{ @@ -46,7 +46,7 @@ TEST_F(TestTupleDomain, TestTupleDomainSubfield) { }, "nullAllowed":false } - } + } ] } )"; diff --git a/presto-native-execution/presto_cpp/presto_protocol/tests/data/ValuesNode.json b/presto-native-execution/presto_cpp/presto_protocol/tests/data/ValuesNode.json index 4feaf38f2c304..61e37f9d1fffe 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/tests/data/ValuesNode.json +++ b/presto-native-execution/presto_cpp/presto_protocol/tests/data/ValuesNode.json @@ -51,4 +51,4 @@ } ] ] -} \ No newline at end of file +} diff --git a/presto-native-execution/presto_cpp/presto_protocol/util.py b/presto-native-execution/presto_cpp/presto_protocol/util.py index 2c9407aab1f1f..af0b375d7c6a9 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/util.py +++ b/presto-native-execution/presto_cpp/presto_protocol/util.py @@ -37,7 +37,7 @@ def run(command, compressed=False, **kwargs): if "input" in kwargs: input = kwargs["input"] - if type(input) == list: + if isinstance(input, list): input = "\n".join(input) + "\n" kwargs["input"] = input.encode("utf-8") diff --git a/presto-native-execution/rhel_build.log b/presto-native-execution/rhel_build.log new file mode 100644 index 0000000000000..03ad634034dbc --- /dev/null +++ b/presto-native-execution/rhel_build.log @@ -0,0 +1 @@ +Error: the specified Containerfile or Dockerfile does not exist, scripts/dockerfiles/rhel-dependency.dockerfile: no such file or directory diff --git a/presto-native-execution/scripts/check.py b/presto-native-execution/scripts/check.py deleted file mode 100755 index 471c5dc175c12..0000000000000 --- a/presto-native-execution/scripts/check.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 2024-07-17: Copied from https://github.com/facebookincubator/velox/blob/main/scripts/license-header.py - -import argparse -from collections import OrderedDict -import os -import regex -import subprocess -import sys - -from util import attrdict -import util - -EXTENSIONS = "cpp,h,inc,prolog" -SCRIPTS = util.script_path() - - -def get_diff(file, formatted): - if not formatted.endswith("\n"): - formatted = formatted + "\n" - - status, stdout, stderr = util.run( - f"diff -u {file} --label {file} --label {file} -", input=formatted - ) - if stdout != "": - stdout = f"diff a/{file} b/{file}\n" + stdout - - return status, stdout, stderr - - -class CppFormatter(str): - def diff(self, commit): - if commit == "": - return get_diff(self, util.run(f"clang-format --style=file {self}")[1]) - else: - return util.run( - f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --diff --style=file {commit} {self}" - ) - - def fix(self, commit): - if commit == "": - return util.run(f"clang-format -i --style=file {self}")[0] == 0 - else: - return ( - util.run( - f"{SCRIPTS}/git-clang-format -q --extensions='{EXTENSIONS}' --style=file {commit} {self}" - )[0] - == 0 - ) - - -class CMakeFormatter(str): - def diff(self, commit): - return get_diff( - self, util.run(f"cmake-format --first-comment-is-literal True {self}")[1] - ) - - def fix(self, commit): - return ( - util.run(f"cmake-format --first-comment-is-literal True -i {self}")[0] == 0 - ) - - -class PythonFormatter(str): - def diff(self, commit): - return util.run(f"black -q --diff {self}") - - def fix(self, commit): - return util.run(f"black -q {self}")[0] == 0 - - -format_file_types = OrderedDict( - { - "CMakeLists.txt": attrdict({"formatter": CMakeFormatter}), - "*.cmake": attrdict({"formatter": CMakeFormatter}), - "*.cpp": attrdict({"formatter": CppFormatter}), - "*.h": attrdict({"formatter": CppFormatter}), - "*.inc": attrdict({"formatter": CppFormatter}), - "*.prolog": attrdict({"formatter": CppFormatter}), - "*.py": attrdict({"formatter": PythonFormatter}), - } -) - - -def get_formatter(filename): - if filename in format_file_types: - return format_file_types[filename] - - return format_file_types.get("*" + util.get_fileextn(filename), None) - - -def format_command(commit, files, fix): - ok = 0 - for filepath in files: - filename = util.get_filename(filepath) - filetype = get_formatter(filename) - - if filetype is None: - print("Skip : " + filepath, file=sys.stderr) - continue - - file = filetype.formatter(filepath) - - if fix == "show": - status, diff, stderr = file.diff(commit) - - if stderr != "": - ok = 1 - print(f"Error: {file}", file=sys.stderr) - continue - - if diff != "" and diff != "no modified files to format": - ok = 1 - print(f"Fix : {file}", file=sys.stderr) - print(diff) - else: - print(f"Ok : {file}", file=sys.stderr) - - else: - print(f"Fix : {file}", file=sys.stderr) - if not file.fix(commit): - ok = 1 - print(f"Error: {file}", file=sys.stderr) - - return ok - - -def header_command(commit, files, fix): - options = "-vk" if fix == "show" else "-i" - - status, stdout, stderr = util.run( - f"{SCRIPTS}/license-header.py {options} -", input=files - ) - - if stdout != "": - print(stdout) - - return status - - -def tidy_command(commit, files, fix): - files = [file for file in files if regex.match(r".*\.cpp$", file)] - - if not files: - return 0 - - commit = f"--commit {commit}" if commit != "" else "" - fix = "--fix" if fix == "fix" else "" - - status, stdout, stderr = util.run( - f"{SCRIPTS}/run-clang-tidy.py {commit} {fix} -", input=files - ) - - if stdout != "": - print(stdout) - - return status - - -def get_commit(files): - if files == "commit": - return "HEAD^" - - if files == "main" or files == "master": - return util.run(f"git merge-base origin/{files} HEAD")[1] - - return "" - - -def get_files(commit, path): - filelist = [] - - if commit != "": - status, stdout, stderr = util.run( - f"git diff --relative --name-only --diff-filter='ACMR' {commit}" - ) - filelist = stdout.splitlines() - else: - if os.path.isfile(path): - filelist.append(path) - else: - for root, dirs, files in os.walk(path): - for name in files: - filelist.append(os.path.join(root, name)) - - return [ - file - for file in filelist - if "/data/" not in file - and "velox/external/" not in file - and "build/fbcode_builder" not in file - and "build/deps" not in file - and "cmake-build-debug" not in file - and "NOTICE.txt" != file - ] - - -def help(args): - parser.print_help() - return 0 - - -def add_check_options(subparser, name): - parser = subparser.add_parser(name) - parser.add_argument("--fix", action="store_const", default="show", const="fix") - return parser - - -def add_options(parser): - files = parser.add_subparsers(dest="files") - - tree_parser = add_check_options(files, "tree") - tree_parser.add_argument("path", default="") - - branch_parser = add_check_options(files, "main") - branch_parser = add_check_options(files, "master") - commit_parser = add_check_options(files, "commit") - - -def add_check_command(parser, name): - subparser = parser.add_parser(name) - add_options(subparser) - - return subparser - - -def parse_args(): - global parser - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, - description="""Check format/header/tidy - - check.py {format,header,tidy} {commit,branch} [--fix] - check.py {format,header,tidy} {tree} [--fix] PATH -""", - ) - command = parser.add_subparsers(dest="command") - command.add_parser("help") - - format_command_parser = add_check_command(command, "format") - header_command_parser = add_check_command(command, "header") - tidy_command_parser = add_check_command(command, "tidy") - - parser.set_defaults(path="") - parser.set_defaults(command="help") - - return parser.parse_args() - - -def run_command(args, command): - commit = get_commit(args.files) - files = get_files(commit, args.path) - - return command(commit, files, args.fix) - - -def format(args): - return run_command(args, format_command) - - -def header(args): - return run_command(args, header_command) - - -def tidy(args): - return run_command(args, tidy_command) - - -def main(): - args = parse_args() - return globals()[args.command](args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/presto-native-execution/scripts/setup-adapters.sh b/presto-native-execution/scripts/setup-adapters.sh index c5a1721e87815..d91845532ab9d 100755 --- a/presto-native-execution/scripts/setup-adapters.sh +++ b/presto-native-execution/scripts/setup-adapters.sh @@ -18,8 +18,7 @@ JWT_VERSION="v0.6.0" PROMETHEUS_VERSION="v1.2.4" SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") -if [ -f "${SCRIPT_DIR}/setup-common.sh" ] -then +if [ -f "${SCRIPT_DIR}/setup-common.sh" ]; then source "${SCRIPT_DIR}/setup-common.sh" else source "${SCRIPT_DIR}/../velox/scripts/setup-common.sh" @@ -59,30 +58,30 @@ install_prometheus_cpp=0 install_arrow_flight=0 if [ "$#" -eq 0 ]; then - # Install all adapters by default - install_jwt=1 - install_prometheus_cpp=1 - install_arrow_flight=1 + # Install all adapters by default + install_jwt=1 + install_prometheus_cpp=1 + install_arrow_flight=1 fi while [[ $# -gt 0 ]]; do case $1 in - jwt) - install_jwt=1 - shift # past argument - ;; - prometheus) - install_prometheus_cpp=1; - shift - ;; - arrow_flight) - install_arrow_flight=1; - shift - ;; - *) - echo "ERROR: Unknown option $1! will be ignored!" - shift - ;; + jwt) + install_jwt=1 + shift # past argument + ;; + prometheus) + install_prometheus_cpp=1 + shift + ;; + arrow_flight) + install_arrow_flight=1 + shift + ;; + *) + echo "ERROR: Unknown option $1! will be ignored!" + shift + ;; esac done @@ -100,6 +99,6 @@ if [ $install_arrow_flight -eq 1 ]; then fi _ret=$? -if [ $_ret -eq 0 ] ; then - echo "All deps for Presto adapters installed!" +if [ $_ret -eq 0 ]; then + echo "All deps for Presto adapters installed!" fi diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 523a35f4f2bfe..015ee736be1fc 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -21,8 +21,7 @@ GPERF_VERSION="3.1" CPU_TARGET="${CPU_TARGET:-avx}" SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") -if [ -f "${SCRIPT_DIR}/setup-centos9.sh" ] -then +if [ -f "${SCRIPT_DIR}/setup-centos9.sh" ]; then source "${SCRIPT_DIR}/setup-centos9.sh" else source "${SCRIPT_DIR}/../velox/scripts/setup-centos9.sh" @@ -41,9 +40,9 @@ function install_gperf { wget_and_untar https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz gperf ( cd ${DEPENDENCY_DIR}/gperf || exit && - ./configure --prefix=/usr/local/gperf/3_1 && - make "-j${NPROC}" && - make install + ./configure --prefix=/usr/local/gperf/3_1 && + make "-j${NPROC}" && + make install if [ -f /usr/local/bin/gperf ]; then echo "Did not create '/usr/local/bin/gperf' symlink as file already exists." else diff --git a/presto-native-execution/scripts/setup-macos.sh b/presto-native-execution/scripts/setup-macos.sh index 4f031e6b125e9..a78fb96d08c51 100755 --- a/presto-native-execution/scripts/setup-macos.sh +++ b/presto-native-execution/scripts/setup-macos.sh @@ -22,7 +22,7 @@ PYTHON_VENV=${PYTHON_VENV:-"${SCRIPTDIR}/../.venv"} # Set DEPENDENCY_DIR to a directory outside of Presto # to build DuckDB. BUILD_DUCKDB="${BUILD_DUCKDB:-false}" -source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../velox/scripts/setup-macos.sh" GPERF_VERSION="3.1" function install_proxygen { @@ -37,8 +37,8 @@ function install_gperf { wget_and_untar https://mirrors.ocf.berkeley.edu/gnu/gperf/gperf-${GPERF_VERSION}.tar.gz gperf ( cd ${DEPENDENCY_DIR}/gperf || exit && - ./configure --prefix=${INSTALL_PREFIX} && - make install + ./configure --prefix=${INSTALL_PREFIX} && + make install ) } @@ -47,7 +47,7 @@ function install_presto_deps { run_and_time install_proxygen } -(return 2> /dev/null) && return # If script was sourced, don't run commands. +(return 2>/dev/null) && return # If script was sourced, don't run commands. if [[ $# -ne 0 ]]; then for cmd in "$@"; do diff --git a/presto-native-execution/scripts/setup-ubuntu.sh b/presto-native-execution/scripts/setup-ubuntu.sh index 947f11ded94e7..c2af776ea2971 100755 --- a/presto-native-execution/scripts/setup-ubuntu.sh +++ b/presto-native-execution/scripts/setup-ubuntu.sh @@ -17,7 +17,7 @@ set -eufx -o pipefail # Run the velox setup script first. -source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-ubuntu.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../velox/scripts/setup-ubuntu.sh" SUDO="${SUDO:-"sudo --preserve-env"}" function install_proxygen { diff --git a/presto-native-execution/scripts/util.py b/presto-native-execution/scripts/util.py index aeaee8fb3ce1a..020fbb5622afc 100644 --- a/presto-native-execution/scripts/util.py +++ b/presto-native-execution/scripts/util.py @@ -37,7 +37,7 @@ def run(command, compressed=False, **kwargs): if "input" in kwargs: input = kwargs["input"] - if type(input) == list: + if isinstance(input, list): input = "\n".join(input) + "\n" kwargs["input"] = input.encode("utf-8")