diff --git a/.github/workflows/arrow-flight-tests.yml b/.github/workflows/arrow-flight-tests.yml index 3aab27ff09c58..c8dec648623d5 100644 --- a/.github/workflows/arrow-flight-tests.yml +++ b/.github/workflows/arrow-flight-tests.yml @@ -1,9 +1,6 @@ name: arrow flight tests -on: - pull_request: - paths-ignore: - - presto-docs/** +on: pull_request permissions: contents: read @@ -17,12 +14,30 @@ env: RETRY: .github/bin/retry jobs: + changes: + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + # Set job outputs to values from filter step + outputs: + codechange: ${{ steps.filter.outputs.codechange }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' + arrowflight-java-tests: runs-on: ubuntu-latest + needs: changes strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] modules: - :presto-base-arrow-flight # Only run tests for the `presto-base-arrow-flight` module @@ -34,33 +49,52 @@ jobs: steps: # Checkout the code only if there are changes in the relevant files - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: show-progress: false persist-credentials: false # Set up Java and dependencies for the build environment - uses: actions/setup-java@v4 + if: needs.changes.outputs.codechange == 'true' with: distribution: temurin java-version: ${{ matrix.java }} cache: maven + + # Cleanup before build + - name: Clean up before build + if: needs.changes.outputs.codechange == 'true' + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + df -h + docker system prune -af || true + - name: Download nodejs to maven cache + if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs # Install dependencies for the target module - name: Maven Install + if: needs.changes.outputs.codechange == 'true' run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -e -am -pl ${{ matrix.modules }} # Run Maven tests for the target module, excluding native tests - name: Maven Tests + if: needs.changes.outputs.codechange == 'true' run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*" prestocpp-linux-build-for-test: runs-on: ubuntu-22.04 + needs: changes container: - image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84 + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 + volumes: + - /usr:/host_usr + - /opt:/host_opt concurrency: group: ${{ github.workflow }}-prestocpp-linux-build-for-test-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -71,21 +105,42 @@ jobs: permissions: actions: write steps: + # We cannot use the github action to free disk space from the runner + # because we are in the container and not on the runner anymore. + - name: Free Disk Space + run: | + # Re-used from free-disk-space github action. + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + # Show before + echo "Original available disk space: " $(getAvailableSpace) + # Remove DotNet. + rm -rf /host_usr/share/dotnet || true + # Remove android + rm -rf /host_usr/local/lib/android || true + # Remove CodeQL + rm -rf /host_opt/hostedtoolcache/CodeQL || true + # Show after + echo "New available disk space: " $(getAvailableSpace) + - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: persist-credentials: false - name: Fix git permissions + if: needs.changes.outputs.codechange == 'true' # Usually actions/checkout does this but as we run in a container # it doesn't work run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Update velox + if: needs.changes.outputs.codechange == 'true' run: | cd presto-native-execution make velox-submodule - name: Install Arrow Flight + if: needs.changes.outputs.codechange == 'true' run: | mkdir -p ${DEPENDENCY_DIR}/adapter-deps/download mkdir -p ${INSTALL_PREFIX}/adapter-deps/install @@ -95,19 +150,23 @@ jobs: PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh arrow_flight - name: Install Github CLI for using apache/infrastructure-actions/stash + if: needs.changes.outputs.codechange == 'true' run: | curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm rpm -iv gh_2.63.2_linux_amd64.rpm - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + if: needs.changes.outputs.codechange == 'true' with: path: '${{ env.CCACHE_DIR }}' key: ccache-prestocpp-linux-build-for-test - name: Zero ccache statistics + if: needs.changes.outputs.codechange == 'true' run: ccache -sz - name: Build engine + if: needs.changes.outputs.codechange == 'true' run: | source /opt/rh/gcc-toolset-12/enable cd presto-native-execution @@ -122,39 +181,47 @@ jobs: -DCMAKE_PREFIX_PATH=/usr/local \ -DThrift_ROOT=/usr/local \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DMAX_LINK_JOBS=4 + -DMAX_LINK_JOBS=3 ninja -C _build/release -j 4 - name: Ccache after + if: needs.changes.outputs.codechange == 'true' run: ccache -s - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + if: needs.changes.outputs.codechange == 'true' with: path: '${{ env.CCACHE_DIR }}' key: ccache-prestocpp-linux-build-for-test - name: Run Unit Tests for the Arrow Flight connector only + if: needs.changes.outputs.codechange == 'true' run: | cd presto-native-execution/_build/release ctest -j 4 -VV --output-on-failure --tests-regex ^presto_flight.* - name: Upload artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/upload-artifact@v4 with: name: arrow-flight-presto-native-build path: presto-native-execution/_build/release/presto_cpp/main/presto_server - name: Upload Arrow Flight install artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/upload-artifact@v4 with: name: arrow-flight-install path: ${{ env.INSTALL_PREFIX }}/lib64/libarrow_flight* arrowflight-native-e2e-tests: - needs: prestocpp-linux-build-for-test + needs: [changes, prestocpp-linux-build-for-test] runs-on: ubuntu-22.04 container: - image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84 + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 + volumes: + - /usr:/host_usr + - /opt:/host_opt env: INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install" strategy: @@ -169,22 +236,43 @@ jobs: cancel-in-progress: true steps: + # We cannot use the github action to free disk space from the runner + # because we are in the container and not on the runner anymore. + - name: Free Disk Space + run: | + # Re-used from free-disk-space github action. + getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); } + # Show before + echo "Original available disk space: " $(getAvailableSpace) + # Remove DotNet. + rm -rf /host_usr/share/dotnet || true + # Remove android + rm -rf /host_usr/local/lib/android || true + # Remove CodeQL + rm -rf /host_opt/hostedtoolcache/CodeQL || true + # Show after + echo "New available disk space: " $(getAvailableSpace) + - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: persist-credentials: false - name: Fix git permissions + if: needs.changes.outputs.codechange == 'true' # Usually actions/checkout does this but as we run in a container # it doesn't work run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Download artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/download-artifact@v4 with: name: arrow-flight-presto-native-build path: presto-native-execution/_build/release/presto_cpp/main - name: Download Arrow Flight install artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/download-artifact@v4 with: name: arrow-flight-install @@ -192,21 +280,25 @@ jobs: # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - name: Restore execute permissions and library path + if: needs.changes.outputs.codechange == 'true' run: | chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server # Ensure transitive dependency libboost-iostreams is found. ldconfig /usr/local/lib - name: Install OpenJDK8 + if: needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache + if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs - name: Maven install + if: needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError @@ -215,6 +307,7 @@ jobs: ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }} - name: Run arrowflight native e2e tests + if: needs.changes.outputs.codechange == 'true' run: | export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" mvn test \ diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 74e148fe4105e..ec9c9d7cb33c9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Maven Install run: | diff --git a/.github/workflows/hive-tests.yml b/.github/workflows/hive-tests.yml index 05eb630e5a3fe..08228bfd827ee 100644 --- a/.github/workflows/hive-tests.yml +++ b/.github/workflows/hive-tests.yml @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest needs: changes timeout-minutes: 60 @@ -92,7 +92,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest needs: changes timeout-minutes: 20 diff --git a/.github/workflows/jdbc-connector-tests.yml b/.github/workflows/jdbc-connector-tests.yml index 342fb01ef1c25..8818cf3d76b11 100644 --- a/.github/workflows/jdbc-connector-tests.yml +++ b/.github/workflows/jdbc-connector-tests.yml @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest needs: changes timeout-minutes: 60 diff --git a/.github/workflows/kudu.yml b/.github/workflows/kudu.yml index 831ce311087cd..6d711f94b28ff 100644 --- a/.github/workflows/kudu.yml +++ b/.github/workflows/kudu.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest needs: changes timeout-minutes: 60 diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 79782713bb0ed..9a4f7b5693fce 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest timeout-minutes: 45 concurrency: diff --git a/.github/workflows/owasp-dependency-check.yml b/.github/workflows/owasp-dependency-check.yml index 426187e40b678..04b87e6c57779 100644 --- a/.github/workflows/owasp-dependency-check.yml +++ b/.github/workflows/owasp-dependency-check.yml @@ -12,23 +12,44 @@ on: type: string jobs: + changes: + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + # Set job outputs to values from filter step + outputs: + codechange: ${{ steps.filter.outputs.codechange }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' + - 'presto-docs/pom.xml' + dependency-check: + needs: changes runs-on: ubuntu-latest concurrency: group: ${{ github.workflow }}-owasp-dependency-check-${{ github.event.pull_request.number }} cancel-in-progress: true env: - CVSS_THRESHOLD: ${{ github.event.inputs.cvss-threshold || '7.0' }} + CVSS_THRESHOLD: ${{ github.event.inputs.cvss-threshold || '0.1' }} OWASP_VERSION: 12.1.3 steps: # Checkout PR branch first to get access to the composite action - name: Checkout PR branch + if: needs.changes.outputs.codechange == 'true' uses: actions/checkout@v4 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - name: Find merge base + if: needs.changes.outputs.codechange == 'true' id: merge-base env: GH_TOKEN: ${{ github.token }} @@ -42,6 +63,7 @@ jobs: echo "Using merge base: $merge_base" - name: Checkout base branch + if: needs.changes.outputs.codechange == 'true' uses: actions/checkout@v4 with: persist-credentials: false @@ -49,6 +71,7 @@ jobs: path: base - name: Set up Java + if: needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: distribution: temurin @@ -56,10 +79,12 @@ jobs: cache: maven - name: Get date for cache key + if: needs.changes.outputs.codechange == 'true' id: get-date run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - name: Restore OWASP database cache + if: needs.changes.outputs.codechange == 'true' uses: actions/cache/restore@v4 id: cache-owasp-restore with: @@ -70,6 +95,7 @@ jobs: owasp-cache-${{ runner.os }}- - name: Run OWASP check on base branch + if: needs.changes.outputs.codechange == 'true' uses: ./.github/actions/maven-owasp-scan with: working-directory: base @@ -77,13 +103,14 @@ jobs: data-directory: /tmp/.owasp/dependency-check-data - name: Save OWASP cache after base scan - if: steps.cache-owasp-restore.outputs.cache-hit != 'true' + if: needs.changes.outputs.codechange == 'true' && steps.cache-owasp-restore.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: /tmp/.owasp/dependency-check-data key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }}-partial - name: Run OWASP check on PR branch + if: needs.changes.outputs.codechange == 'true' uses: ./.github/actions/maven-owasp-scan with: working-directory: . @@ -91,6 +118,7 @@ jobs: data-directory: /tmp/.owasp/dependency-check-data - name: Compare and fail on new CVEs above threshold + if: needs.changes.outputs.codechange == 'true' run: | # Extract CVEs above threshold from both branches (CVSS >= $CVSS_THRESHOLD) threshold=$CVSS_THRESHOLD @@ -154,14 +182,14 @@ jobs: fi - name: Save OWASP database cache - if: always() + if: needs.changes.outputs.codechange == 'true' && always() uses: actions/cache/save@v4 with: path: /tmp/.owasp/dependency-check-data key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }} - name: Upload reports - if: always() + if: needs.changes.outputs.codechange == 'true' && always() uses: actions/upload-artifact@v4 with: name: owasp-reports diff --git a/.github/workflows/presto-release-publish.yml b/.github/workflows/presto-release-publish.yml index 36720303fa17b..3c9e1058fb51c 100644 --- a/.github/workflows/presto-release-publish.yml +++ b/.github/workflows/presto-release-publish.yml @@ -432,7 +432,7 @@ jobs: working-directory: presto-native-execution run: | df -h - docker compose build --build-arg EXTRA_CMAKE_FLAGS=" + docker compose build --build-arg EXTRA_CMAKE_FLAGS=" \ -DPRESTO_ENABLE_PARQUET=ON \ -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ -DPRESTO_ENABLE_JWT=ON \ diff --git a/.github/workflows/prestocpp-linux-adapters-build.yml b/.github/workflows/prestocpp-linux-adapters-build.yml index 362b3fcbbacee..d45bd35a27f6d 100644 --- a/.github/workflows/prestocpp-linux-adapters-build.yml +++ b/.github/workflows/prestocpp-linux-adapters-build.yml @@ -11,7 +11,7 @@ jobs: prestocpp-linux-adapters-build: runs-on: ubuntu-22.04 container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 concurrency: group: ${{ github.workflow }}-prestocpp-linux-adapters-build-${{ github.event.pull_request.number }} cancel-in-progress: true diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index e77a4bb152fd6..566146da290fd 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-22.04 needs: changes container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 concurrency: group: ${{ github.workflow }}-prestocpp-linux-build-test-${{ github.event.pull_request.number }} cancel-in-progress: true @@ -135,7 +135,7 @@ jobs: needs: [changes, prestocpp-linux-build-for-test] runs-on: ubuntu-22.04 container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: - /usr:/host_usr - /opt:/host_opt @@ -200,7 +200,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache if: | @@ -260,7 +260,7 @@ jobs: storage-format: [PARQUET, DWRF] enable-sidecar: [true, false] container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: - /usr:/host_usr - /opt:/host_opt @@ -332,7 +332,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache if: | @@ -388,7 +388,7 @@ jobs: group: ${{ github.workflow }}-prestocpp-linux-presto-on-spark-e2e-tests-${{ matrix.storage-format }}-${{ matrix.enable-sidecar }}-${{ github.event.pull_request.number }} cancel-in-progress: true container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: - /usr:/host_usr - /opt:/host_opt @@ -453,7 +453,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache if: | @@ -501,7 +501,7 @@ jobs: needs: [changes, prestocpp-linux-build-for-test] runs-on: ubuntu-22.04 container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: - /usr:/host_usr - /opt:/host_opt @@ -568,7 +568,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache if: | @@ -613,7 +613,7 @@ jobs: needs: [changes, prestocpp-linux-build-for-test] runs-on: ubuntu-22.04 container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: - /usr:/host_usr - /opt:/host_opt @@ -680,7 +680,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17.0.15 + java-version: '17' cache: maven - name: Download nodejs to maven cache if: | diff --git a/.github/workflows/prestocpp-linux-build.yml b/.github/workflows/prestocpp-linux-build.yml index 872b43adf90e0..f87b63fe760f3 100644 --- a/.github/workflows/prestocpp-linux-build.yml +++ b/.github/workflows/prestocpp-linux-build.yml @@ -20,13 +20,13 @@ jobs: codechange: - '!presto-docs/**' - prestocpp-linux-build-engine: + prestocpp-linux-build-gpu-engine: runs-on: ubuntu-22.04 permissions: contents: read needs: changes container: - image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea + image: prestodb/presto-native-dependency:0.297-202602190453-8d6d9543 volumes: - /usr:/host_usr - /opt:/host_opt @@ -35,16 +35,20 @@ jobs: cancel-in-progress: true env: CCACHE_DIR: "${{ github.workspace }}/ccache" - CC: /usr/bin/clang-15 - CXX: /usr/bin/clang++-15 + cudf_SOURCE: BUNDLED + CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc + # Set compiler to GCC 14 + CUDA_FLAGS: -ccbin /opt/rh/gcc-toolset-14/root/usr/bin BUILD_SCRIPT: | + unset CC && unset CXX + source /opt/rh/gcc-toolset-14/enable cd presto-native-execution cmake \ - -B _build/debug \ + -B _build/release \ -GNinja \ -DTREAT_WARNINGS_AS_ERRORS=1 \ -DENABLE_ALL_WARNINGS=1 \ - -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_BUILD_TYPE=Release \ -DPRESTO_ENABLE_S3=ON \ -DPRESTO_ENABLE_GCS=ON \ -DPRESTO_ENABLE_ABFS=OFF \ @@ -54,11 +58,13 @@ jobs: -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ -DPRESTO_ENABLE_TESTING=OFF \ + -DPRESTO_ENABLE_CUDF=ON \ + -DCMAKE_CUDA_ARCHITECTURES=75 \ -DCMAKE_PREFIX_PATH=/usr/local \ -DThrift_ROOT=/usr/local \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DMAX_LINK_JOBS=4 - ninja -C _build/debug -j 4 + ninja -C _build/release -j 4 steps: # We cannot use the github action to free disk space from the runner diff --git a/.github/workflows/prestocpp-macos-build.yml b/.github/workflows/prestocpp-macos-build.yml index 7c2b6421a43ba..32bf241d58670 100644 --- a/.github/workflows/prestocpp-macos-build.yml +++ b/.github/workflows/prestocpp-macos-build.yml @@ -76,6 +76,10 @@ jobs: install_velox_deps_from_brew install_double_conversion + # Install glog/gflags because they are not installed from homebrew. + install_gflags + install_glog + # Velox deps needed by proxygen, a presto dependency. install_boost install_fmt diff --git a/.github/workflows/product-tests-basic-environment.yml b/.github/workflows/product-tests-basic-environment.yml index faa60abf11d3a..7a2dfd1a6cb17 100644 --- a/.github/workflows/product-tests-basic-environment.yml +++ b/.github/workflows/product-tests-basic-environment.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index fea69491625d8..cbc0dde9e0e4c 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read @@ -98,7 +98,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/singlestore-tests.yml b/.github/workflows/singlestore-tests.yml index f18d1a6e554b3..6e1f0b275c1ff 100644 --- a/.github/workflows/singlestore-tests.yml +++ b/.github/workflows/singlestore-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/spark-integration.yml b/.github/workflows/spark-integration.yml index 6dc61358e9c2e..36f43b97d5198 100644 --- a/.github/workflows/spark-integration.yml +++ b/.github/workflows/spark-integration.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/test-other-modules.yml b/.github/workflows/test-other-modules.yml index ed2bf2e77727c..38a34323ddc01 100644 --- a/.github/workflows/test-other-modules.yml +++ b/.github/workflows/test-other-modules.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 028c9ba9378ff..7419e3e2083de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false matrix: - java: [17.0.15] + java: ['17'] modules: - :presto-tests -P presto-tests-execution-memory - :presto-tests -P presto-tests-general diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index b96797e3c6d15..b5217474cf3a6 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1,2 +1,2 @@ -distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.8.8/apache-maven-3.8.8-bin.zip +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.zip wrapperUrl=https://repo1.maven.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/README.md b/README.md index 4b3b1fcdedb39..a664980d3687b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ See the [Presto documentation](https://prestodb.io/docs/current/) for general do ## Mission and Architecture -See [PrestoDB: Mission and Architecture](ARCHITECTURE.md). +See [PrestoDB: Mission and Architecture](ARCHITECTURE.md). ## Requirements @@ -67,7 +67,7 @@ To modify the loaded plugins in IntelliJ, modify the `config.properties` located ### Additional configuration for Java 17 -When running with Java 17, additional `--add-opens` flags are required to allow reflective access used by certain catalogs based on which catalogs are configured. +When running with Java 17, additional `--add-opens` flags are required to allow reflective access used by certain catalogs based on which catalogs are configured. For the default set of catalogs loaded when starting the Presto server in IntelliJ without changes, add the following flags to the **VM Options**: --add-opens=java.base/java.io=ALL-UNNAMED @@ -157,6 +157,10 @@ resources will be hot-reloaded and changes are reflected on browser refresh. Check out [building instructions](https://github.com/prestodb/presto/tree/master/presto-native-execution#build-from-source) to get started. +## Using development containers + +The PrestoDB project provides support for development containers in its own repository. +Please visit the [presto-dev README](https://github.com/prestodb/presto-dev/blob/main/README.md) for details.
diff --git a/pom.xml b/pom.xml index 880d2928cc193..534f3ef5e91e7 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ 3.3.9 4.13.2 - 0.225 + 0.227 ${dep.airlift.version} 0.38 0.6 @@ -55,23 +55,24 @@ ${dep.airlift.version} - 2.13.1 + 2.14.0 1.55 7.5 9.12.0 3.8.0 - 1.13.1 + 1.16.0 + 1.10.0 9.7.1 1.9.17 313 2.0.16 3.9.1 - 1.3.0 - 30.0.1 + 1.4.0 + 35.0.1 2.3.1 - 4.0.5 + 4.0.6 0.14.0 - 1.20.5 + 2.0.3 3.4.1 2.9.0 3.1.3 @@ -79,17 +80,17 @@ 32.1.0-jre 2.15.4 3.0.0 - 1.11.4 + 1.12.1 1.27.1 - 4.29.0 + 4.30.2 12.0.29 - 4.1.130.Final - 1.2.8 + 4.2.10.Final + 1.3.3 2.5 2.12.1 3.18.0 6.0.0 - 17.0.0 + 18.3.0 3.5.4 2.0.2-6 3.4.1-1 @@ -110,11 +111,12 @@ -missing 1.17.2 2.32.9 - 1.19.0 + 1.20.0 8.5.2 2.2.0 5.0.1 + 1.58.0 true @@ -160,6 +162,7 @@ presto-bytecode presto-client presto-parser + presto-internal-communication presto-main-base presto-main-tests presto-main @@ -224,17 +227,20 @@ presto-native-tests presto-router presto-open-telemetry + presto-openlineage-event-listener redis-hbo-provider presto-singlestore presto-hana presto-openapi presto-native-sidecar-plugin + presto-common-arrow presto-base-arrow-flight presto-function-server presto-router-example-plugin-scheduler presto-plan-checker-router-plugin presto-sql-helpers/presto-sql-invoked-functions-plugin presto-sql-helpers/presto-native-sql-invoked-functions-plugin + presto-lance @@ -597,6 +603,12 @@ ${project.version} + + com.facebook.presto + presto-internal-communication + ${project.version} + + io.grpc grpc-context @@ -1010,6 +1022,12 @@ provided + + com.facebook.presto + presto-common-arrow + ${project.version} + + com.facebook.presto presto-base-arrow-flight @@ -1089,7 +1107,7 @@ com.facebook.presto.hive hive-apache - 3.0.0-10 + 3.0.0-12 @@ -1192,6 +1210,12 @@ ${project.version} + + com.facebook.presto + presto-openlineage-event-listener + ${project.version} + + com.facebook.presto presto-native-sidecar-plugin @@ -1207,7 +1231,7 @@ io.airlift aircompressor - 0.27 + 2.0.3 @@ -1536,7 +1560,7 @@ org.postgresql postgresql - 42.6.1 + 42.7.9 @@ -1640,6 +1664,12 @@ ${dep.reactor-netty.version} + + io.projectreactor + reactor-core + 3.8.3 + + org.apache.thrift libthrift @@ -2062,6 +2092,13 @@ + + + at.yawk.lz4 + lz4-java + 1.10.2 + + org.apache.httpcomponents httpclient @@ -2313,7 +2350,7 @@ org.xerial.snappy snappy-java - 1.1.10.7 + 1.1.10.8 @@ -2582,19 +2619,19 @@ io.opentelemetry opentelemetry-api - 1.19.0 + ${dep.io.opentelemetry.version} io.opentelemetry opentelemetry-context - 1.19.0 + ${dep.io.opentelemetry.version} io.opentelemetry opentelemetry-exporter-otlp - 1.19.0 + ${dep.io.opentelemetry.version} com.squareup.okhttp3 @@ -2606,31 +2643,31 @@ io.opentelemetry opentelemetry-extension-trace-propagators - 1.19.0 + ${dep.io.opentelemetry.version} io.opentelemetry opentelemetry-sdk - 1.19.0 + ${dep.io.opentelemetry.version} io.opentelemetry opentelemetry-sdk-common - 1.19.0 + ${dep.io.opentelemetry.version} io.opentelemetry opentelemetry-sdk-trace - 1.19.0 + ${dep.io.opentelemetry.version} - io.opentelemetry + io.opentelemetry.semconv opentelemetry-semconv - 1.19.0-alpha + 1.37.0 diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java index d5329e375d58f..c3c1f43f8e142 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.spi.analyzer.AccessControlRole; import com.facebook.presto.spi.analyzer.UpdateInfo; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.eventlistener.OutputColumnMetadata; import com.facebook.presto.spi.function.FunctionHandle; @@ -241,11 +242,14 @@ public class Analysis // Row id field used for MERGE INTO command. private final Map, FieldReference> rowIdField = new LinkedHashMap<>(); - public Analysis(@Nullable Statement root, Map, Expression> parameters, boolean isDescribe) + private final ViewDefinitionReferences viewDefinitionReferences; + + public Analysis(@Nullable Statement root, Map, Expression> parameters, boolean isDescribe, ViewDefinitionReferences viewDefinitionReferences) { this.root = root; this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameterMap is null")); this.isDescribe = isDescribe; + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } public Statement getStatement() @@ -957,9 +961,9 @@ public AccessControlReferences getAccessControlReferences() return accessControlReferences; } - public void addQueryAccessControlInfo(AccessControlInfo accessControlInfo) + public ViewDefinitionReferences getViewDefinitionReferences() { - accessControlReferences.setQueryAccessControlInfo(accessControlInfo); + return viewDefinitionReferences; } public void addAccessControlCheckForTable(AccessControlRole accessControlRole, AccessControlInfoForTable accessControlInfoForTable) diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalysis.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalysis.java index 69b55766ca693..3a70ce4e66b0b 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalysis.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalysis.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.spi.analyzer.QueryAnalysis; import com.facebook.presto.spi.analyzer.UpdateInfo; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.sql.tree.Explain; import com.google.common.collect.ImmutableSet; @@ -65,6 +66,12 @@ public AccessControlReferences getAccessControlReferences() return analysis.getAccessControlReferences(); } + @Override + public ViewDefinitionReferences getViewDefinitionReferences() + { + return analysis.getViewDefinitionReferences(); + } + @Override public boolean isExplainAnalyzeQuery() { diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java index e777e219eae6e..1be20437e8a1c 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java @@ -21,12 +21,14 @@ import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; +import com.facebook.presto.sql.tree.CreateTag; import com.facebook.presto.sql.tree.CreateType; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; @@ -131,6 +133,8 @@ private StatementUtils() {} builder.put(CreateType.class, QueryType.DATA_DEFINITION); builder.put(AddColumn.class, QueryType.DATA_DEFINITION); builder.put(CreateTable.class, QueryType.DATA_DEFINITION); + builder.put(CreateBranch.class, QueryType.DATA_DEFINITION); + builder.put(CreateTag.class, QueryType.DATA_DEFINITION); builder.put(RenameTable.class, QueryType.DATA_DEFINITION); builder.put(RenameColumn.class, QueryType.DATA_DEFINITION); builder.put(DropColumn.class, QueryType.DATA_DEFINITION); diff --git a/presto-base-arrow-flight/pom.xml b/presto-base-arrow-flight/pom.xml index 004f690c12ffb..2bec314c35953 100644 --- a/presto-base-arrow-flight/pom.xml +++ b/presto-base-arrow-flight/pom.xml @@ -29,6 +29,11 @@ + + com.facebook.presto + presto-common-arrow + + org.apache.arrow arrow-memory-core @@ -250,6 +255,7 @@ com.fasterxml.jackson.core:jackson-databind com.facebook.airlift:log-manager javax.inject:javax.inject + io.airlift:slice diff --git a/presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/testingConnector/TestingArrowBlockBuilder.java b/presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/testingConnector/TestingArrowBlockBuilder.java index f42941a96387c..b72e2100339c1 100644 --- a/presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/testingConnector/TestingArrowBlockBuilder.java +++ b/presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/testingConnector/TestingArrowBlockBuilder.java @@ -34,7 +34,7 @@ public TestingArrowBlockBuilder(TypeManager typeManager) } @Override - protected Type getPrestoTypeFromArrowField(Field field) + public Type getPrestoTypeFromArrowField(Field field) { String columnLength = field.getMetadata().get("columnLength"); int length = columnLength != null ? Integer.parseInt(columnLength) : 0; diff --git a/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java b/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java index d880429f7489f..3ecd353d21327 100644 --- a/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java +++ b/presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionUtil.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.NamedTypeSignature; +import com.facebook.presto.common.type.RowFieldName; import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; @@ -154,10 +155,12 @@ private static List getTypeSignatureParameters( parameterTypeSignature.getStandardTypeSignature(), parameterTypeSignature.getParameters())); if (isNamedTypeSignature) { + // Preserve the original field name if present, otherwise use Optional.empty() + Optional fieldName = parameter.getNamedTypeSignature().getFieldName(); newParameterTypeList.add( TypeSignatureParameter.of( new NamedTypeSignature( - Optional.empty(), + fieldName, newTypeSignature))); } else { diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java index c6a3242f0b7ff..b851cdc81146a 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraClientModule.java @@ -57,7 +57,6 @@ public void configure(Binder binder) { binder.bind(CassandraConnectorId.class).toInstance(new CassandraConnectorId(connectorId)); binder.bind(CassandraConnector.class).in(Scopes.SINGLETON); - binder.bind(CassandraMetadata.class).in(Scopes.SINGLETON); binder.bind(CassandraSplitManager.class).in(Scopes.SINGLETON); binder.bind(CassandraTokenSplitManager.class).in(Scopes.SINGLETON); binder.bind(CassandraRecordSetProvider.class).in(Scopes.SINGLETON); diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java index 016f2c8022465..9562a9afb6971 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnector.java @@ -14,8 +14,10 @@ package com.facebook.presto.cassandra; import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.json.JsonCodec; import com.facebook.airlift.log.Logger; import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorCommitHandle; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; @@ -26,9 +28,13 @@ import jakarta.inject.Inject; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import static com.facebook.presto.spi.connector.EmptyConnectorCommitHandle.INSTANCE; import static com.facebook.presto.spi.transaction.IsolationLevel.READ_UNCOMMITTED; import static com.facebook.presto.spi.transaction.IsolationLevel.checkConnectorSupports; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public class CassandraConnector @@ -36,35 +42,66 @@ public class CassandraConnector { private static final Logger log = Logger.get(CassandraConnector.class); + private final CassandraConnectorId connectorId; private final LifeCycleManager lifeCycleManager; - private final CassandraMetadata metadata; + private final CassandraPartitionManager partitionManager; + private final CassandraClientConfig config; + private final CassandraSession cassandraSession; private final CassandraSplitManager splitManager; private final ConnectorRecordSetProvider recordSetProvider; private final ConnectorPageSinkProvider pageSinkProvider; private final List> sessionProperties; + private final JsonCodec> extraColumnMetadataCodec; + private final ConcurrentMap transactions = new ConcurrentHashMap<>(); @Inject public CassandraConnector( + CassandraConnectorId connectorId, LifeCycleManager lifeCycleManager, - CassandraMetadata metadata, CassandraSplitManager splitManager, CassandraRecordSetProvider recordSetProvider, CassandraPageSinkProvider pageSinkProvider, - CassandraSessionProperties sessionProperties) + CassandraSessionProperties sessionProperties, + CassandraSession cassandraSession, + CassandraPartitionManager partitionManager, + JsonCodec> extraColumnMetadataCodec, + CassandraClientConfig config) { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadata = requireNonNull(metadata, "metadata is null"); this.splitManager = requireNonNull(splitManager, "splitManager is null"); this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); this.sessionProperties = requireNonNull(sessionProperties.getSessionProperties(), "sessionProperties is null"); + this.partitionManager = requireNonNull(partitionManager, "partitionManager is null"); + this.cassandraSession = requireNonNull(cassandraSession, "cassandraSession is null"); + this.config = requireNonNull(config, "config is null"); + this.extraColumnMetadataCodec = requireNonNull(extraColumnMetadataCodec, "extraColumnMetadataCodec is null"); } @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { checkConnectorSupports(READ_UNCOMMITTED, isolationLevel); - return CassandraTransactionHandle.INSTANCE; + CassandraTransactionHandle transaction = new CassandraTransactionHandle(); + transactions.put(transaction, + new CassandraMetadata(connectorId, cassandraSession, partitionManager, extraColumnMetadataCodec, config)); + return transaction; + } + + @Override + public ConnectorCommitHandle commit(ConnectorTransactionHandle transaction) + { + checkArgument(transactions.remove(transaction) != null, "no such transaction: %s", transaction); + return INSTANCE; + } + + @Override + public void rollback(ConnectorTransactionHandle transaction) + { + CassandraMetadata metadata = transactions.remove(transaction); + checkArgument(metadata != null, "no such transaction: %s", transaction); + metadata.rollback(); } @Override @@ -74,8 +111,10 @@ public boolean isSingleStatementWritesOnly() } @Override - public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) { + CassandraMetadata metadata = transactions.get(transaction); + checkArgument(metadata != null, "no such transaction: %s", transaction); return metadata; } diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java index 3a2b2cda4a574..e63707de290ee 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraConnectorFactory.java @@ -15,6 +15,7 @@ import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorContext; diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java index da86b2293fdcc..b4ea65a7ecb48 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraMetadata.java @@ -41,13 +41,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airlift.slice.Slice; -import jakarta.inject.Inject; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static com.facebook.presto.cassandra.CassandraType.toCassandraType; @@ -57,6 +57,7 @@ import static com.facebook.presto.spi.StandardErrorCode.PERMISSION_DENIED; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Locale.ROOT; import static java.util.Objects.requireNonNull; @@ -72,8 +73,8 @@ public class CassandraMetadata private boolean caseSensitiveNameMatchingEnabled; private final JsonCodec> extraColumnMetadataCodec; + private final AtomicReference rollbackAction = new AtomicReference<>(); - @Inject public CassandraMetadata( CassandraConnectorId connectorId, CassandraSession cassandraSession, @@ -319,6 +320,9 @@ private CassandraOutputTableHandle createTable(ConnectorSession session, Connect // We need to create the Cassandra table before commit because the record needs to be written to the table. cassandraSession.execute(queryBuilder.toString()); + + // set a rollback to delete the created table in case of an abort / failure. + setRollback(schemaName, tableName); return new CassandraOutputTableHandle( connectorId, schemaName, @@ -330,6 +334,7 @@ private CassandraOutputTableHandle createTable(ConnectorSession session, Connect @Override public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) { + clearRollback(); return Optional.empty(); } @@ -365,4 +370,30 @@ public String normalizeIdentifier(ConnectorSession session, String identifier) { return caseSensitiveNameMatchingEnabled ? identifier : identifier.toLowerCase(ROOT); } + + public void rollback() + { + Runnable action = rollbackAction.getAndSet(null); + if (action == null) { + return; // nothing to roll back + } + + if (!allowDropTable) { + throw new PrestoException( + PERMISSION_DENIED, + "Table creation was aborted and requires rollback, but cleanup failed because DROP TABLE is disabled in this Cassandra catalog."); + } + + action.run(); + } + + private void setRollback(String schemaName, String tableName) + { + checkState(rollbackAction.compareAndSet(null, () -> cassandraSession.execute(String.format("DROP TABLE \"%s\".\"%s\"", schemaName, tableName))), "rollback action is already set"); + } + + private void clearRollback() + { + rollbackAction.set(null); + } } diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraTransactionHandle.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraTransactionHandle.java index 7a2eb23d4f162..4128e287135ef 100644 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraTransactionHandle.java +++ b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/CassandraTransactionHandle.java @@ -14,9 +14,61 @@ package com.facebook.presto.cassandra; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; -public enum CassandraTransactionHandle +import java.util.Objects; +import java.util.UUID; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CassandraTransactionHandle implements ConnectorTransactionHandle { - INSTANCE + private final UUID uuid; + + public CassandraTransactionHandle() + { + this(UUID.randomUUID()); + } + + @JsonCreator + public CassandraTransactionHandle(@JsonProperty("uuid") UUID uuid) + { + this.uuid = requireNonNull(uuid, "uuid is null"); + } + + @JsonProperty + public UUID getUuid() + { + return uuid; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + CassandraTransactionHandle other = (CassandraTransactionHandle) obj; + return Objects.equals(uuid, other.uuid); + } + + @Override + public int hashCode() + { + return Objects.hash(uuid); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("uuid", uuid) + .toString(); + } } diff --git a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java b/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java deleted file mode 100644 index 9525145e4e510..0000000000000 --- a/presto-cassandra/src/main/java/com/facebook/presto/cassandra/RebindSafeMBeanServer.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.cassandra; - -import com.facebook.airlift.log.Logger; -import com.google.errorprone.annotations.ThreadSafe; - -import javax.management.Attribute; -import javax.management.AttributeList; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceAlreadyExistsException; -import javax.management.InstanceNotFoundException; -import javax.management.IntrospectionException; -import javax.management.InvalidAttributeValueException; -import javax.management.ListenerNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanInfo; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.NotCompliantMBeanException; -import javax.management.NotificationFilter; -import javax.management.NotificationListener; -import javax.management.ObjectInstance; -import javax.management.ObjectName; -import javax.management.OperationsException; -import javax.management.QueryExp; -import javax.management.ReflectionException; -import javax.management.loading.ClassLoaderRepository; - -import java.io.ObjectInputStream; -import java.util.Set; - -/** - * MBeanServer wrapper that a ignores calls to registerMBean when there is already - * a MBean registered with the specified object name. - */ -@ThreadSafe -public class RebindSafeMBeanServer - implements MBeanServer -{ - private static final Logger log = Logger.get(RebindSafeMBeanServer.class); - - private final MBeanServer mbeanServer; - - public RebindSafeMBeanServer(MBeanServer mbeanServer) - { - this.mbeanServer = mbeanServer; - } - - /** - * Delegates to the wrapped mbean server, but if a mbean is already registered - * with the specified name, the existing instance is returned. - */ - @Override - public ObjectInstance registerMBean(Object object, ObjectName name) - throws MBeanRegistrationException, NotCompliantMBeanException - { - while (true) { - try { - // try to register the mbean - return mbeanServer.registerMBean(object, name); - } - catch (InstanceAlreadyExistsException ignored) { - } - - try { - // a mbean is already installed, try to return the already registered instance - ObjectInstance objectInstance = mbeanServer.getObjectInstance(name); - log.debug("%s already bound to %s", name, objectInstance); - return objectInstance; - } - catch (InstanceNotFoundException ignored) { - // the mbean was removed before we could get the reference - // start the whole process over again - } - } - } - - @Override - public void unregisterMBean(ObjectName name) - throws InstanceNotFoundException, MBeanRegistrationException - { - mbeanServer.unregisterMBean(name); - } - - @Override - public ObjectInstance getObjectInstance(ObjectName name) - throws InstanceNotFoundException - { - return mbeanServer.getObjectInstance(name); - } - - @Override - public Set queryMBeans(ObjectName name, QueryExp query) - { - return mbeanServer.queryMBeans(name, query); - } - - @Override - public Set queryNames(ObjectName name, QueryExp query) - { - return mbeanServer.queryNames(name, query); - } - - @Override - public boolean isRegistered(ObjectName name) - { - return mbeanServer.isRegistered(name); - } - - @Override - public Integer getMBeanCount() - { - return mbeanServer.getMBeanCount(); - } - - @Override - public Object getAttribute(ObjectName name, String attribute) - throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttribute(name, attribute); - } - - @Override - public AttributeList getAttributes(ObjectName name, String[] attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttributes(name, attributes); - } - - @Override - public void setAttribute(ObjectName name, Attribute attribute) - throws InstanceNotFoundException, AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException - { - mbeanServer.setAttribute(name, attribute); - } - - @Override - public AttributeList setAttributes(ObjectName name, AttributeList attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.setAttributes(name, attributes); - } - - @Override - public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) - throws InstanceNotFoundException, MBeanException, ReflectionException - { - return mbeanServer.invoke(name, operationName, params, signature); - } - - @Override - public String getDefaultDomain() - { - return mbeanServer.getDefaultDomain(); - } - - @Override - public String[] getDomains() - { - return mbeanServer.getDomains(); - } - - @Override - public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public MBeanInfo getMBeanInfo(ObjectName name) - throws InstanceNotFoundException, IntrospectionException, ReflectionException - { - return mbeanServer.getMBeanInfo(name); - } - - @Override - public boolean isInstanceOf(ObjectName name, String className) - throws InstanceNotFoundException - { - return mbeanServer.isInstanceOf(name, className); - } - - @Override - public Object instantiate(String className) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className); - } - - @Override - public Object instantiate(String className, ObjectName loaderName) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName); - } - - @Override - public Object instantiate(String className, Object[] params, String[] signature) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className, params, signature); - } - - @Override - public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName, params, signature); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(ObjectName name, byte[] data) - throws OperationsException - { - return mbeanServer.deserialize(name, data); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(String className, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, data); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, loaderName, data); - } - - @Override - public ClassLoader getClassLoaderFor(ObjectName mbeanName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoaderFor(mbeanName); - } - - @Override - public ClassLoader getClassLoader(ObjectName loaderName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoader(loaderName); - } - - @Override - public ClassLoaderRepository getClassLoaderRepository() - { - return mbeanServer.getClassLoaderRepository(); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name, params, signature); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName, params, signature); - } -} diff --git a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java index 49a8a2bdc247d..a3535c0e46b4f 100644 --- a/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java +++ b/presto-cassandra/src/test/java/com/facebook/presto/cassandra/TestCassandraConnector.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorOutputTableHandle; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; @@ -66,10 +67,12 @@ import static com.facebook.presto.common.type.Varchars.isVarcharType; import static com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING; import static com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED; +import static com.facebook.presto.spi.transaction.IsolationLevel.READ_UNCOMMITTED; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Locale.ENGLISH; import static java.util.Locale.ROOT; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -98,10 +101,11 @@ public class TestCassandraConnector protected SchemaTableName table; protected SchemaTableName tableUnpartitioned; protected SchemaTableName invalidTable; + protected SchemaTableName rollbackTable; private CassandraServer server; - private ConnectorMetadata metadata; private ConnectorSplitManager splitManager; private ConnectorRecordSetProvider recordSetProvider; + private Connector connector; @BeforeClass public void setup() @@ -115,14 +119,12 @@ public void setup() String connectorId = "cassandra-test"; CassandraConnectorFactory connectorFactory = new CassandraConnectorFactory(connectorId); - Connector connector = connectorFactory.create(connectorId, ImmutableMap.of( - "cassandra.contact-points", server.getHost(), - "cassandra.native-protocol-port", Integer.toString(server.getPort())), + connector = connectorFactory.create(connectorId, ImmutableMap.of( + "cassandra.contact-points", server.getHost(), + "cassandra.native-protocol-port", Integer.toString(server.getPort()), + "cassandra.allow-drop-table", "true"), new TestingConnectorContext()); - metadata = connector.getMetadata(CassandraTransactionHandle.INSTANCE); - assertInstanceOf(metadata, CassandraMetadata.class); - splitManager = connector.getSplitManager(); assertInstanceOf(splitManager, CassandraSplitManager.class); @@ -133,6 +135,7 @@ public void setup() table = new SchemaTableName(database, TABLE_ALL_TYPES.toLowerCase(ROOT)); tableUnpartitioned = new SchemaTableName(database, "presto_test_unpartitioned"); invalidTable = new SchemaTableName(database, "totally_invalid_table_name"); + rollbackTable = new SchemaTableName(database, "rollback_table"); } @Test @@ -149,6 +152,8 @@ public void tearDown() @Test public void testGetDatabaseNames() { + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); List databases = metadata.listSchemaNames(SESSION); assertTrue(databases.contains(database.toLowerCase(ROOT))); } @@ -156,6 +161,8 @@ public void testGetDatabaseNames() @Test public void testGetTableNames() { + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); List tables = metadata.listTables(SESSION, database); assertTrue(tables.contains(table)); } @@ -164,12 +171,16 @@ public void testGetTableNames() @Test(enabled = false, expectedExceptions = SchemaNotFoundException.class) public void testGetTableNamesException() { + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); metadata.listTables(SESSION, INVALID_DATABASE); } @Test public void testListUnknownSchema() { + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); assertNull(metadata.getTableHandle(SESSION, new SchemaTableName("totally_invalid_database_name", "dual"))); assertEquals(metadata.listTables(SESSION, "totally_invalid_database_name"), ImmutableList.of()); assertEquals(metadata.listTableColumns(SESSION, new SchemaTablePrefix("totally_invalid_database_name", "dual")), ImmutableMap.of()); @@ -178,23 +189,23 @@ public void testListUnknownSchema() @Test public void testGetRecords() { - ConnectorTableHandle tableHandle = getTableHandle(table); + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); + ConnectorTableHandle tableHandle = getTableHandle(table, metadata); ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(SESSION, tableHandle); List columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(SESSION, tableHandle).values()); Map columnIndex = indexColumns(columnHandles); - ConnectorTransactionHandle transaction = CassandraTransactionHandle.INSTANCE; - ConnectorTableLayoutResult layoutResult = metadata.getTableLayoutForConstraint(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty()); ConnectorTableLayoutHandle layout = layoutResult.getTableLayout().getHandle(); - List splits = getAllSplits(splitManager.getSplits(transaction, SESSION, layout, new SplitSchedulingContext(UNGROUPED_SCHEDULING, false, WarningCollector.NOOP))); + List splits = getAllSplits(splitManager.getSplits(transactionHandle, SESSION, layout, new SplitSchedulingContext(UNGROUPED_SCHEDULING, false, WarningCollector.NOOP))); long rowNumber = 0; for (ConnectorSplit split : splits) { CassandraSplit cassandraSplit = (CassandraSplit) split; long completedBytes = 0; - try (RecordCursor cursor = recordSetProvider.getRecordSet(transaction, SESSION, cassandraSplit, columnHandles).cursor()) { + try (RecordCursor cursor = recordSetProvider.getRecordSet(transactionHandle, SESSION, cassandraSplit, columnHandles).cursor()) { while (cursor.advanceNextPosition()) { try { assertReadFields(cursor, tableMetadata.getColumns()); @@ -231,6 +242,39 @@ public void testGetRecords() assertEquals(rowNumber, 9); } + @Test + public void testRollbackTables() + { + ConnectorTableMetadata connectorTableMetadata = new ConnectorTableMetadata( + rollbackTable, + ImmutableList.of( + ColumnMetadata.builder() + .setName("test_col") + .setType(BIGINT) + .build())); + + // start a transaction + ConnectorTransactionHandle transactionHandle = connector.beginTransaction(READ_UNCOMMITTED, true); + ConnectorMetadata metadata = connector.getMetadata(transactionHandle); + ConnectorOutputTableHandle handle = null; + + try { + // Begin table creation (STAGING only) + handle = metadata.beginCreateTable(SESSION, connectorTableMetadata, Optional.empty()); + // simulate a failure + throw new RuntimeException("Force failure before finish"); + } + catch (RuntimeException e) { + if (handle != null) { + // table should exist + assertTrue(metadata.listTables(SESSION, database).contains(rollbackTable)); + // rollback table + connector.rollback(transactionHandle); + } + } + assertFalse(metadata.listTables(SESSION, database).contains(rollbackTable)); + } + private static void assertReadFields(RecordCursor cursor, List schema) { for (int columnIndex = 0; columnIndex < schema.size(); columnIndex++) { @@ -270,7 +314,7 @@ else if (isVarcharType(type) || VARBINARY.equals(type)) { } } - private ConnectorTableHandle getTableHandle(SchemaTableName tableName) + private ConnectorTableHandle getTableHandle(SchemaTableName tableName, ConnectorMetadata metadata) { ConnectorTableHandle handle = metadata.getTableHandle(SESSION, tableName); checkArgument(handle != null, "table not found: %s", tableName); diff --git a/presto-clickhouse/pom.xml b/presto-clickhouse/pom.xml index f6ece72d07bd9..b143e209efa5c 100644 --- a/presto-clickhouse/pom.xml +++ b/presto-clickhouse/pom.xml @@ -196,13 +196,13 @@ org.testcontainers - clickhouse + testcontainers-clickhouse test org.testcontainers - jdbc + testcontainers-jdbc test diff --git a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/ClickHouseQueryRunner.java b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/ClickHouseQueryRunner.java index a0519c2f6855b..1f5847ba8efdc 100755 --- a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/ClickHouseQueryRunner.java +++ b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/ClickHouseQueryRunner.java @@ -60,6 +60,8 @@ public static DistributedQueryRunner createClickHouseQueryRunner( connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); connectorProperties.putIfAbsent("clickhouse.connection-url", server.getJdbcUrl()); + connectorProperties.putIfAbsent("clickhouse.connection-user", server.getClickHouseContainer().getUsername()); + connectorProperties.putIfAbsent("clickhouse.connection-password", server.getClickHouseContainer().getPassword()); connectorProperties.putIfAbsent("clickhouse.allow-drop-table", String.valueOf(true)); connectorProperties.putIfAbsent("clickhouse.map-string-as-varchar", String.valueOf(true)); diff --git a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestingClickHouseServer.java b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestingClickHouseServer.java index 87dbb916f1179..1a1c80c312782 100755 --- a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestingClickHouseServer.java +++ b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestingClickHouseServer.java @@ -13,16 +13,13 @@ */ package com.facebook.presto.plugin.clickhouse; -import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.clickhouse.ClickHouseContainer; import java.io.Closeable; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; -import static java.lang.String.format; -import static org.testcontainers.containers.ClickHouseContainer.HTTP_PORT; - public class TestingClickHouseServer implements Closeable { @@ -44,7 +41,10 @@ public ClickHouseContainer getClickHouseContainer() } public void execute(String sql) { - try (Connection connection = DriverManager.getConnection(getJdbcUrl()); + try (Connection connection = DriverManager.getConnection( + getJdbcUrl(), + dockerContainer.getUsername(), + dockerContainer.getPassword()); Statement statement = connection.createStatement()) { statement.execute(sql); } @@ -55,10 +55,7 @@ public void execute(String sql) public String getJdbcUrl() { - String s = format("jdbc:clickhouse://%s:%s/", dockerContainer.getContainerIpAddress(), - dockerContainer.getMappedPort(HTTP_PORT)); - return format("jdbc:clickhouse://%s:%s/", dockerContainer.getContainerIpAddress(), - dockerContainer.getMappedPort(HTTP_PORT)); + return dockerContainer.getJdbcUrl(); } @Override diff --git a/presto-common-arrow/pom.xml b/presto-common-arrow/pom.xml new file mode 100644 index 0000000000000..c2507c9f136d4 --- /dev/null +++ b/presto-common-arrow/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.297-SNAPSHOT + + + presto-common-arrow + presto-common-arrow + Presto - Common Arrow Utilities + + + ${project.parent.basedir} + + + + + org.apache.arrow + arrow-vector + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + + + + com.facebook.presto + presto-spi + + + + com.facebook.presto + presto-common + + + + io.airlift + slice + + + + com.google.guava + guava + + + + jakarta.inject + jakarta.inject-api + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + 1.2.1 + + + module-info + META-INF.versions.9.module-info + + + arrow-git.properties + about.html + + + + + + check + + + + + + + diff --git a/presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java b/presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java similarity index 92% rename from presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java rename to presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java index ed703f1b2444f..a6180a18fe8fb 100644 --- a/presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java +++ b/presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowBlockBuilder.java @@ -54,6 +54,7 @@ import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.arrow.vector.TimeStampMilliTZVector; import org.apache.arrow.vector.TimeStampMilliVector; @@ -62,6 +63,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; @@ -112,7 +114,7 @@ public Block buildBlockFromFieldVector(FieldVector vector, Type type, Dictionary return builder.build(); } - protected Type getPrestoTypeFromArrowField(Field field) + public Type getPrestoTypeFromArrowField(Field field) { switch (field.getType().getTypeID()) { case Int: @@ -139,7 +141,8 @@ protected Type getPrestoTypeFromArrowField(Field field) return BooleanType.BOOLEAN; case Time: return TimeType.TIME; - case List: { + case List: + case FixedSizeList: { List children = field.getChildren(); checkArgument(children.size() == 1, "Arrow List expected to have 1 child Field, got: " + children.size()); return new ArrayType(getPrestoTypeFromArrowField(field.getChildren().get(0))); @@ -292,6 +295,9 @@ else if (vector instanceof TimeStampSecVector) { else if (vector instanceof TimeMicroVector) { assignBlockFromTimeMicroVector((TimeMicroVector) vector, type, builder, startIndex, endIndex); } + else if (vector instanceof TimeStampMicroTZVector) { + assignBlockFromTimeStampMicroTZVector((TimeStampMicroTZVector) vector, type, builder, startIndex, endIndex); + } else if (vector instanceof TimeStampMilliTZVector) { assignBlockFromTimeMilliTZVector((TimeStampMilliTZVector) vector, type, builder, startIndex, endIndex); } @@ -299,6 +305,9 @@ else if (vector instanceof MapVector) { // NOTE: MapVector is also instanceof ListVector, so check for Map first assignBlockFromMapVector((MapVector) vector, type, builder, startIndex, endIndex); } + else if (vector instanceof FixedSizeListVector) { + assignBlockFromFixedSizeListVector((FixedSizeListVector) vector, type, builder, startIndex, endIndex); + } else if (vector instanceof ListVector) { assignBlockFromListVector((ListVector) vector, type, builder, startIndex, endIndex); } @@ -666,6 +675,49 @@ public void assignBlockFromListVector(ListVector vector, Type type, BlockBuilder } } + public void assignBlockFromFixedSizeListVector(FixedSizeListVector vector, Type type, BlockBuilder builder, int startIndex, int endIndex) + { + if (!(type instanceof ArrayType)) { + throw new IllegalArgumentException("Type must be an ArrayType for FixedSizeListVector"); + } + + ArrayType arrayType = (ArrayType) type; + Type elementType = arrayType.getElementType(); + int listSize = vector.getListSize(); + + for (int i = startIndex; i < endIndex; i++) { + if (vector.isNull(i)) { + builder.appendNull(); + } + else { + BlockBuilder elementBuilder = builder.beginBlockEntry(); + int elementStart = i * listSize; + int elementEnd = elementStart + listSize; + assignBlockFromValueVector( + vector.getDataVector(), elementType, elementBuilder, elementStart, elementEnd); + builder.closeEntry(); + } + } + } + + public void assignBlockFromTimeStampMicroTZVector(TimeStampMicroTZVector vector, Type type, BlockBuilder builder, int startIndex, int endIndex) + { + if (!(type instanceof TimestampType)) { + throw new IllegalArgumentException("Expected TimestampType but got " + type.getClass().getName()); + } + + for (int i = startIndex; i < endIndex; i++) { + if (vector.isNull(i)) { + builder.appendNull(); + } + else { + long micros = vector.get(i); + long millis = TimeUnit.MICROSECONDS.toMillis(micros); + type.writeLong(builder, millis); + } + } + } + public void assignBlockFromMapVector(MapVector vector, Type type, BlockBuilder builder, int startIndex, int endIndex) { if (!(type instanceof MapType)) { diff --git a/presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowErrorCode.java b/presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowErrorCode.java similarity index 100% rename from presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowErrorCode.java rename to presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowErrorCode.java diff --git a/presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowException.java b/presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowException.java similarity index 100% rename from presto-base-arrow-flight/src/main/java/com/facebook/plugin/arrow/ArrowException.java rename to presto-common-arrow/src/main/java/com/facebook/plugin/arrow/ArrowException.java diff --git a/presto-common/pom.xml b/presto-common/pom.xml index 07e80d39313e7..37db32216242d 100644 --- a/presto-common/pom.xml +++ b/presto-common/pom.xml @@ -62,6 +62,12 @@ jol-core + + com.facebook.airlift + log + provided + + com.facebook.presto @@ -78,7 +84,6 @@ com.google.guava guava - test diff --git a/presto-common/src/main/java/com/facebook/presto/common/Subfield.java b/presto-common/src/main/java/com/facebook/presto/common/Subfield.java index 5854fff8a8731..4d49c557fcaab 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/Subfield.java +++ b/presto-common/src/main/java/com/facebook/presto/common/Subfield.java @@ -81,6 +81,31 @@ public String toString() } } + public static final class StructureOnly + implements PathElement + { + private static final StructureOnly STRUCTURE_ONLY = new StructureOnly(); + + private StructureOnly() {} + + public static StructureOnly getInstance() + { + return STRUCTURE_ONLY; + } + + @Override + public boolean isSubscript() + { + return true; + } + + @Override + public String toString() + { + return "[$]"; + } + } + public static final class NestedField implements PathElement { @@ -238,6 +263,11 @@ public static PathElement noSubfield() return NoSubfield.getInstance(); } + public static PathElement structureOnly() + { + return StructureOnly.getInstance(); + } + @JsonCreator public Subfield(String path) { diff --git a/presto-common/src/main/java/com/facebook/presto/common/SubfieldTokenizer.java b/presto-common/src/main/java/com/facebook/presto/common/SubfieldTokenizer.java index fe61bf6e73eaf..562a99e1c151a 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/SubfieldTokenizer.java +++ b/presto-common/src/main/java/com/facebook/presto/common/SubfieldTokenizer.java @@ -106,7 +106,7 @@ private Subfield.PathElement computeNext() } if (tryMatch(OPEN_BRACKET)) { - Subfield.PathElement token = tryMatch(QUOTE) ? matchQuotedSubscript() : tryMatch(WILDCARD) ? matchWildcardSubscript() : matchUnquotedSubscript(); + Subfield.PathElement token = tryMatch(QUOTE) ? matchQuotedSubscript() : tryMatch(WILDCARD) ? matchWildcardSubscript() : tryMatch(DOLLAR) ? matchStructureOnlySubscript() : matchUnquotedSubscript(); match(CLOSE_BRACKET); firstSegment = false; @@ -151,6 +151,11 @@ private Subfield.PathElement matchDollarPathElement() return Subfield.noSubfield(); } + private Subfield.PathElement matchStructureOnlySubscript() + { + return Subfield.structureOnly(); + } + private static boolean isUnquotedPathCharacter(char c) { return c == ':' || c == '$' || c == '-' || c == '/' || c == '@' || c == '|' || c == '#' || c == ' ' || c == '<' || c == '>' || isUnquotedSubscriptCharacter(c); diff --git a/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java b/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java index 9df746537e007..eda4d7fc9f764 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java +++ b/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java @@ -18,9 +18,12 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; public class SqlFunctionProperties @@ -38,6 +41,7 @@ public class SqlFunctionProperties private final Map extraCredentials; private final boolean warnOnCommonNanPatterns; private final boolean canonicalizedJsonExtract; + private final Set tryCatchableErrorCodes; private SqlFunctionProperties( boolean parseDecimalLiteralAsDouble, @@ -52,7 +56,8 @@ private SqlFunctionProperties( boolean legacyJsonCast, Map extraCredentials, boolean warnOnCommonNanPatterns, - boolean canonicalizedJsonExtract) + boolean canonicalizedJsonExtract, + Set tryCatchableErrorCodes) { this.parseDecimalLiteralAsDouble = parseDecimalLiteralAsDouble; this.legacyRowFieldOrdinalAccessEnabled = legacyRowFieldOrdinalAccessEnabled; @@ -67,6 +72,7 @@ private SqlFunctionProperties( this.extraCredentials = requireNonNull(extraCredentials, "extraCredentials is null"); this.warnOnCommonNanPatterns = warnOnCommonNanPatterns; this.canonicalizedJsonExtract = canonicalizedJsonExtract; + this.tryCatchableErrorCodes = requireNonNull(tryCatchableErrorCodes, "tryCatchableErrorCodes is null"); } public boolean isParseDecimalLiteralAsDouble() @@ -133,6 +139,11 @@ public boolean shouldWarnOnCommonNanPatterns() public boolean isCanonicalizedJsonExtract() { return canonicalizedJsonExtract; } + public Set getTryCatchableErrorCodes() + { + return tryCatchableErrorCodes; + } + @Override public boolean equals(Object o) { @@ -153,7 +164,8 @@ public boolean equals(Object o) Objects.equals(sessionUser, that.sessionUser) && Objects.equals(extraCredentials, that.extraCredentials) && Objects.equals(legacyJsonCast, that.legacyJsonCast) && - Objects.equals(canonicalizedJsonExtract, that.canonicalizedJsonExtract); + Objects.equals(canonicalizedJsonExtract, that.canonicalizedJsonExtract) && + Objects.equals(tryCatchableErrorCodes, that.tryCatchableErrorCodes); } @Override @@ -161,7 +173,7 @@ public int hashCode() { return Objects.hash(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, legacyTimestamp, legacyMapSubscript, sessionStartTime, sessionLocale, sessionUser, - extraCredentials, legacyJsonCast, canonicalizedJsonExtract); + extraCredentials, legacyJsonCast, canonicalizedJsonExtract, tryCatchableErrorCodes); } public static Builder builder() @@ -184,6 +196,7 @@ public static class Builder private Map extraCredentials = emptyMap(); private boolean warnOnCommonNanPatterns; private boolean canonicalizedJsonExtract; + private Set tryCatchableErrorCodes = emptySet(); private Builder() {} @@ -265,6 +278,12 @@ public Builder setCanonicalizedJsonExtract(boolean canonicalizedJsonExtract) return this; } + public Builder setTryCatchableErrorCodes(Set tryCatchableErrorCodes) + { + this.tryCatchableErrorCodes = unmodifiableSet(tryCatchableErrorCodes); + return this; + } + public SqlFunctionProperties build() { return new SqlFunctionProperties( @@ -280,7 +299,8 @@ public SqlFunctionProperties build() legacyJsonCast, extraCredentials, warnOnCommonNanPatterns, - canonicalizedJsonExtract); + canonicalizedJsonExtract, + tryCatchableErrorCodes); } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java b/presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java rename to presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java index dbcc9d46ee587..5d2b1b11380dd 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpAddressType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/IpAddressType.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.type; +package com.facebook.presto.common.type; import com.facebook.presto.common.block.Block; import com.facebook.presto.common.block.BlockBuilder; @@ -19,9 +19,6 @@ import com.facebook.presto.common.block.Int128ArrayBlockBuilder; import com.facebook.presto.common.block.PageBuilderStatus; import com.facebook.presto.common.function.SqlFunctionProperties; -import com.facebook.presto.common.type.AbstractPrimitiveType; -import com.facebook.presto.common.type.FixedWidthType; -import com.facebook.presto.common.type.StandardTypes; import com.google.common.net.InetAddresses; import io.airlift.slice.Slice; import io.airlift.slice.Slices; diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java b/presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java similarity index 96% rename from presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java rename to presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java index 9381ff5c0ea34..dbf624672f903 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/IpPrefixType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/IpPrefixType.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.type; +package com.facebook.presto.common.type; import com.facebook.presto.common.block.Block; import com.facebook.presto.common.block.BlockBuilder; @@ -19,9 +19,6 @@ import com.facebook.presto.common.block.PageBuilderStatus; import com.facebook.presto.common.block.VariableWidthBlockBuilder; import com.facebook.presto.common.function.SqlFunctionProperties; -import com.facebook.presto.common.type.AbstractPrimitiveType; -import com.facebook.presto.common.type.FixedWidthType; -import com.facebook.presto.common.type.StandardTypes; import com.google.common.net.InetAddresses; import io.airlift.slice.Slice; import io.airlift.slice.XxHash64; diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java b/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java index 2291fd8dc80e4..f2c1f408969d8 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/StandardTypes.java @@ -34,6 +34,7 @@ public final class StandardTypes public static final String QDIGEST = "qdigest"; public static final String TDIGEST = "tdigest"; public static final String KLL_SKETCH = "kllsketch"; + public static final String K_HYPER_LOG_LOG = "KHyperLogLog"; public static final String P4_HYPER_LOG_LOG = "P4HyperLogLog"; public static final String INTERVAL_DAY_TO_SECOND = "interval day to second"; public static final String INTERVAL_YEAR_TO_MONTH = "interval year to month"; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java b/presto-common/src/main/java/com/facebook/presto/common/util/RebindSafeMBeanServer.java similarity index 98% rename from presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java rename to presto-common/src/main/java/com/facebook/presto/common/util/RebindSafeMBeanServer.java index c1af771e83253..c11ed61d23f7e 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RebindSafeMBeanServer.java +++ b/presto-common/src/main/java/com/facebook/presto/common/util/RebindSafeMBeanServer.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.hive; +package com.facebook.presto.common.util; import com.facebook.airlift.log.Logger; import com.google.errorprone.annotations.ThreadSafe; @@ -47,7 +47,6 @@ * MBeanServer wrapper that a ignores calls to registerMBean when there is already * a MBean registered with the specified object name. */ -@SuppressWarnings("deprecation") @ThreadSafe public class RebindSafeMBeanServer implements MBeanServer @@ -261,6 +260,7 @@ public Object instantiate(String className, ObjectName loaderName, Object[] para @Override @Deprecated + @SuppressWarnings("deprecation") public ObjectInputStream deserialize(ObjectName name, byte[] data) throws OperationsException { @@ -269,6 +269,7 @@ public ObjectInputStream deserialize(ObjectName name, byte[] data) @Override @Deprecated + @SuppressWarnings("deprecation") public ObjectInputStream deserialize(String className, byte[] data) throws OperationsException, ReflectionException { @@ -277,6 +278,7 @@ public ObjectInputStream deserialize(String className, byte[] data) @Override @Deprecated + @SuppressWarnings("deprecation") public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) throws OperationsException, ReflectionException { diff --git a/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties b/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties index 7d53bb63b6ecd..19615ee15f34a 100644 --- a/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties +++ b/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties @@ -2240,3 +2240,4 @@ 2231 Pacific/Kanton 2232 Europe/Kyiv 2233 America/Ciudad_Juarez +2234 America/Coyhaique diff --git a/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java b/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java index 6750e43a34937..263abb0e3c0c8 100644 --- a/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java +++ b/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java @@ -216,7 +216,7 @@ public int compare(TimeZoneKey left, TimeZoneKey right) hasher.putString(timeZoneKey.getId(), StandardCharsets.UTF_8); } // Zone file should not (normally) be changed, so let's make this more difficult - assertEquals(hasher.hash().asLong(), 4825838578917475630L, "zone-index.properties file contents changed!"); + assertEquals(hasher.hash().asLong(), 3765670086753811806L, "zone-index.properties file contents changed!"); } public void assertTimeZoneNotSupported(String zoneId) diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaMetadata.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaMetadata.java index 22e67e9d47f6e..8cdfde27b4f2f 100644 --- a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaMetadata.java +++ b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaMetadata.java @@ -46,6 +46,7 @@ import jakarta.inject.Inject; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -331,11 +332,19 @@ private ConnectorTableMetadata getTableMetadata(ConnectorSession session, Schema return null; } - List columnMetadata = tableHandle.getDeltaTable().getColumns().stream() + DeltaTable deltaTable = tableHandle.getDeltaTable(); + + // External location property + Map properties = new HashMap<>(1); + if (deltaTable.getTableLocation() != null) { + properties.put(DeltaTableProperties.EXTERNAL_LOCATION_PROPERTY, deltaTable.getTableLocation()); + } + + List columnMetadata = deltaTable.getColumns().stream() .map(column -> getColumnMetadata(session, column)) .collect(Collectors.toList()); - return new ConnectorTableMetadata(tableName, columnMetadata); + return new ConnectorTableMetadata(tableName, columnMetadata, properties); } @Override diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaModule.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaModule.java index 598eadcdd5bf7..cbbe73ef3ec92 100644 --- a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaModule.java +++ b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaModule.java @@ -42,6 +42,7 @@ import com.facebook.presto.hive.metastore.HivePartitionMutator; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; import com.facebook.presto.hive.metastore.InvalidateMetastoreCacheProcedure; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.MetastoreCacheStats; import com.facebook.presto.hive.metastore.MetastoreConfig; import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreConfig; @@ -105,6 +106,7 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(HiveClientConfig.class); configBinder(binder).bindConfig(MetastoreClientConfig.class); configBinder(binder).bindConfig(ThriftHiveMetastoreConfig.class); + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(MetastoreCacheStats.class).to(HiveMetastoreCacheStats.class).in(Scopes.SINGLETON); newExporter(binder).export(MetastoreCacheStats.class).as(generatedNameOf(MetastoreCacheStats.class, connectorId)); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); diff --git a/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java b/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java index da535ca4177b0..4c016e734126d 100644 --- a/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java +++ b/presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaIntegration.java @@ -338,4 +338,33 @@ private static void setCommitFileModificationTime(String tableLocation, long com Paths.get(URI.create(tableLocation)).resolve("_delta_log/").resolve(format("%020d.json", commitId)), FileTime.from(commitTimeMillis, TimeUnit.MILLISECONDS)); } + + @Test(dataProvider = "deltaReaderVersions") + public void testShowCreateTable(String deltaVersion) + { + String tableName = deltaVersion + "/data-reader-primitives"; + String fullTableName = format("%s.%s.\"%s\"", DELTA_CATALOG, DELTA_SCHEMA.toLowerCase(), tableName); + + String createTableQueryTemplate = "CREATE TABLE %s (\n" + + " \"as_int\" integer,\n" + + " \"as_long\" bigint,\n" + + " \"as_byte\" tinyint,\n" + + " \"as_short\" smallint,\n" + + " \"as_boolean\" boolean,\n" + + " \"as_float\" real,\n" + + " \"as_double\" double,\n" + + " \"as_string\" varchar,\n" + + " \"as_binary\" varbinary,\n" + + " \"as_big_decimal\" decimal(1,0)\n" + + ")\n" + + "WITH (\n" + + " external_location = '%s'\n" + + ")"; + + String expectedSqlCommand = format(createTableQueryTemplate, fullTableName, goldenTablePath(tableName)); + + String showCreateTableCommandResult = (String) computeActual("SHOW CREATE TABLE " + fullTableName).getOnlyValue(); + + assertEquals(showCreateTableCommandResult, expectedSqlCommand); + } } diff --git a/presto-docs/requirements.txt b/presto-docs/requirements.txt index f5ee358d6d1e4..3203e335054ca 100644 --- a/presto-docs/requirements.txt +++ b/presto-docs/requirements.txt @@ -1,3 +1,2 @@ sphinx==8.2.1 sphinx-immaterial==0.13.0 -sphinx-copybutton==0.5.2 diff --git a/presto-docs/src/main/sphinx/admin.rst b/presto-docs/src/main/sphinx/admin.rst index 9c3bc6f547b5c..38b4e74adf3d2 100644 --- a/presto-docs/src/main/sphinx/admin.rst +++ b/presto-docs/src/main/sphinx/admin.rst @@ -22,3 +22,4 @@ Administration admin/verifier admin/grafana-cloud admin/version-support + admin/jmx-metrics diff --git a/presto-docs/src/main/sphinx/admin/jmx-metrics.rst b/presto-docs/src/main/sphinx/admin/jmx-metrics.rst new file mode 100644 index 0000000000000..983e0fe4dcf17 --- /dev/null +++ b/presto-docs/src/main/sphinx/admin/jmx-metrics.rst @@ -0,0 +1,208 @@ +===================== +JMX Metrics Reference +===================== + +Presto exposes comprehensive metrics via Java Management Extensions (JMX) for monitoring +cluster health, query performance, and system behavior. This page documents some +important JMX metrics available for production monitoring. + +Overview +-------- + +JMX metrics can be accessed through: + +* **JMX clients**: JConsole, VisualVM, or jmxterm +* **SQL queries**: Using the :doc:`/connector/jmx` connector +* **Monitoring systems**: Prometheus, Grafana, or other JMX exporters + +Querying Metrics via SQL +------------------------- + +Once configured, you can query metrics using SQL: + +.. code-block:: sql + + -- List all available metrics + SHOW TABLES FROM jmx.current; + + -- Query specific metrics + SELECT * FROM jmx.current."com.facebook.presto.metadata:name=metadatamanagerstats"; + +Metadata Operation Metrics +--------------------------- + +**JMX Table Name:** ``com.facebook.presto.metadata:name=metadatamanagerstats`` + +Tracks performance and usage of all metadata operations including schema discovery, +table lookups, and column information retrieval. + +Key Metrics +^^^^^^^^^^^ + +For each metadata operation such as ``listSchemaNames``, ``listTables``, or ``getTableHandle``: + +**Call Counters** + +* ``Calls``: Total number of times the operation was called +* Example: ``listSchemaNamesCalls``, ``listTablesCalls`` + +**Timing Statistics** + +All timing values are in nanoseconds: + +* ``time.alltime.avg``: Average execution time across all calls +* ``time.alltime.min``: Fastest execution time +* ``time.alltime.max``: Slowest execution time +* ``time.alltime.count``: Number of samples collected +* ``time.alltime.p50``: Median (50th percentile) +* ``time.alltime.p75``: 75th percentile +* ``time.alltime.p90``: 90th percentile +* ``time.alltime.p95``: 95th percentile +* ``time.alltime.p99``: 99th percentile + +**Time Windows** + +Statistics are also available for recent time windows: + +* ``time.oneminute.*``: Last 1 minute +* ``time.fiveminutes.*``: Last 5 minutes +* ``time.fifteenminutes.*``: Last 15 minutes + +Common Operations +^^^^^^^^^^^^^^^^^ + +**Schema Operations** + +* ``listSchemaNames``: List all schemas in a catalog +* ``getSchemaProperties``: Get schema-level properties + +**Table Operations** + +* ``listTables``: List tables in a schema +* ``getTableHandle``: Get table metadata handle +* ``getTableMetadata``: Get detailed table information +* ``getTableStatistics``: Get table statistics + +**Column Operations** + +* ``getColumnHandles``: Get column information +* ``getColumnMetadata``: Get detailed column metadata + +**View Operations** + +* ``listViews``: List views in a schema +* ``getView``: Get view definition + +Example Queries +^^^^^^^^^^^^^^^ + +**Query Lifecycle Metrics** + +Track query begin and completion times: + +.. code-block:: sql + + -- Query begin operation metrics + SELECT + "beginquerytime.alltime.count" as total_queries, + "beginquerytime.alltime.avg" / 1000.0 as avg_microseconds, + "beginquerytime.alltime.min" / 1000.0 as min_microseconds, + "beginquerytime.alltime.max" / 1000.0 as max_microseconds + FROM jmx.current."com.facebook.presto.metadata:name=metadatamanagerstats"; + + -- Example output: + -- total_queries | avg_microseconds | min_microseconds | max_microseconds + -- 3.0 | 49.42 | 28.63 | 75.38 + +**Insert Operation Metrics** + +Track data insertion performance: + +.. code-block:: sql + + -- Begin insert operation metrics + SELECT + "begininserttime.alltime.count" as insert_operations, + "begininserttime.alltime.avg" / 1000000000.0 as avg_seconds, + "begininserttime.alltime.min" / 1000000000.0 as min_seconds, + "begininserttime.alltime.max" / 1000000000.0 as max_seconds + FROM jmx.current."com.facebook.presto.metadata:name=metadatamanagerstats"; + + -- Example output: + -- insert_operations | avg_seconds | min_seconds | max_seconds + -- 1.0 | 0.82 | 0.82 | 0.82 + + -- Finish insert operation metrics + SELECT + "finishinserttime.alltime.count" as completed_inserts, + "finishinserttime.alltime.avg" / 1000000000.0 as avg_seconds, + "finishinserttime.alltime.min" / 1000000000.0 as min_seconds, + "finishinserttime.alltime.max" / 1000000000.0 as max_seconds + FROM jmx.current."com.facebook.presto.metadata:name=metadatamanagerstats"; + + -- Example output: + -- completed_inserts | avg_seconds | min_seconds | max_seconds + -- 1.0 | 11.47 | 11.47 | 11.47 + +System Access Control Metrics +------------------------------ + +**JMX Table Name:** ``com.facebook.presto.security:name=accesscontrolmanager`` + +Tracks performance of access control checks. + +Key Metrics +^^^^^^^^^^^ + +Similar structure to metadata metrics, tracking operations like: + +* ``checkCanSetUser``: User impersonation checks +* ``checkCanAccessCatalog``: Catalog access checks +* ``checkCanSelectFromColumns``: Column-level access checks +* ``checkCanCreateTable``: Table creation permission checks + +Query Execution Metrics +----------------------- + +**Task Metrics** + +* ``com.facebook.presto.execution:name=taskmanager``: Task execution statistics +* ``com.facebook.presto.execution.executor:name=taskexecutor``: Task executor pool metrics + +**Memory Metrics** + +* ``com.facebook.presto.memory:name=general,type=memorypool``: General memory pool usage +* ``com.facebook.presto.memory:name=reserved,type=memorypool``: Reserved memory pool usage + +**Query Manager Metrics** + +* ``com.facebook.presto.dispatcher:name=dispatchmanager``: Query dispatch statistics +* ``com.facebook.presto.execution:name=querymanager``: Query execution statistics + +Connector-Specific Metrics +--------------------------- + +Hive Connector +^^^^^^^^^^^^^^ + +* ``com.facebook.presto.hive:name=*``: Hive metastore and file system metrics +Example - +* com.facebook.presto.hive:name=hive,type=cachingdirectorylister + +Iceberg Connector +^^^^^^^^^^^^^^^^^ + +* ``com.facebook.presto.iceberg:name=*``: Iceberg-specific caching and I/O metrics + +Examples: + +* com.facebook.presto.iceberg:name=iceberg,type=icebergsplitmanager +* com.facebook.presto.iceberg:name=iceberg,type=manifestfilecache +* com.facebook.presto.iceberg:name=icebergfilewriterfactory + +See Also +-------- + +* :doc:`/connector/jmx` - JMX Connector documentation +* :doc:`web-interface` - Web UI monitoring +* :doc:`tuning` - Performance tuning guide diff --git a/presto-docs/src/main/sphinx/admin/materialized-views.rst b/presto-docs/src/main/sphinx/admin/materialized-views.rst index 9e330c5509511..318397b481057 100644 --- a/presto-docs/src/main/sphinx/admin/materialized-views.rst +++ b/presto-docs/src/main/sphinx/admin/materialized-views.rst @@ -91,6 +91,250 @@ The following permissions are required for materialized view operations when non-owners query the view to prevent privilege escalation. * For INVOKER mode: User needs ``SELECT`` permission on all underlying base tables +Data Consistency Modes +---------------------- + +Materialized views support three data consistency modes that control how queries are optimized +when the view's data may be stale: + +**USE_STITCHING** (default) + Reads fresh data from storage, recomputes stale data from base tables, + and combines results via UNION. + +**FAIL** + Fails the query if the materialized view is stale. + +**USE_VIEW_QUERY** + Executes the view query against base tables. Always fresh but highest cost. + +Set via session property:: + + SET SESSION materialized_view_skip_storage = 'USE_STITCHING'; + +Predicate Stitching (USE_STITCHING Mode) +---------------------------------------- + +Overview +^^^^^^^^ + +Predicate stitching recomputes only stale data rather than the entire view. When base +tables change, Presto identifies which data is affected and generates a UNION query +that combines: + +* **Storage scan**: Reads unchanged (fresh) data from the materialized view's storage +* **Recompute branch**: Recomputes changed (stale) data from base tables using the view's + defining query + +This avoids full recomputation when only a subset of data is stale, though there is +overhead from the UNION operation and predicate-based filtering. + +How It Works +^^^^^^^^^^^^ + +**Staleness Detection** + +For each base table referenced in the materialized view, a connector may track which data +has changed since the last refresh and return predicates identifying the stale data. The +specific mechanism depends on the connector: + +1. At refresh time, metadata is recorded (implementation varies by connector) +2. When the view is queried, the current state is compared with the recorded state +3. Predicates are built that identify exactly which data is stale + +See the connector-specific documentation for details on how staleness is tracked. +For Iceberg tables, see :ref:`connector/iceberg:materialized views`. + +**Query Rewriting** + +When a query uses a materialized view with stale data, the optimizer rewrites the query +to use UNION:: + + -- Original query + SELECT * FROM my_materialized_view WHERE order_date >= '2024-01-01' + + -- Rewritten with predicate stitching (example using partition predicates) + SELECT * FROM ( + -- Fresh partitions from storage + SELECT * FROM my_materialized_view_storage + WHERE order_date >= '2024-01-01' + AND order_date NOT IN ('2024-01-15', '2024-01-16') -- Exclude stale + UNION ALL + -- Stale partitions recomputed + SELECT o.order_id, c.customer_name, o.order_date + FROM orders o + JOIN customers c ON o.customer_id = c.customer_id + AND o.order_date = c.reg_date + WHERE o.order_date IN ('2024-01-15', '2024-01-16') -- Stale partition filter + AND c.reg_date IN ('2024-01-15', '2024-01-16') -- Propagated via equivalence + AND o.order_date >= '2024-01-01' -- Original filter preserved + ) + +The partition predicate is propagated to equivalent columns in joined tables (in this case, +``c.reg_date``), allowing partition pruning on the ``customers`` table as well. + +Requirements +^^^^^^^^^^^^ + +For predicate stitching to work effectively, the following requirements must be met: + +**Predicate Mapping Requirement** + +The connector must be able to express staleness as predicates that can be mapped to the +materialized view's columns. The specific requirements depend on the connector implementation. +For partition-based connectors (like Iceberg), this typically means: + +* Base table partition columns must appear in the SELECT list or be equivalent to columns that do +* The materialized view should be partitioned on the same or equivalent columns +* Partition columns must use compatible data types + +See connector-specific documentation for details on staleness tracking requirements. + +**Unsupported Query Patterns** + +Predicate stitching does not work with: + +* **Outer joins**: LEFT, RIGHT, and FULL OUTER joins +* **Non-deterministic functions**: ``RANDOM()``, ``NOW()``, ``UUID()``, etc. + +**Security Constraints** + +For SECURITY INVOKER materialized views, predicate stitching requires that: + +* No column masks are defined on base tables (or the view is treated as fully stale) +* No row filters are defined on base tables (or the view is treated as fully stale) + +This is because column masks and row filters can vary by user, making it impossible to +determine staleness in a user-independent way. + +Column Equivalences and Passthrough Columns +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Predicate stitching supports **passthrough columns** through **column equivalences**, +which allows tracking staleness even when predicate columns from base tables +are not directly in the materialized view's output. + +**Column Equivalence** + +When tables are joined with equality predicates, those columns become equivalent for +predicate propagation purposes. This applies to any type of staleness predicate +(partition-based, snapshot-based, etc.). For example with partition predicates:: + + CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, order_date VARCHAR) + WITH (partitioning = ARRAY['order_date']); + + CREATE TABLE customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) + WITH (partitioning = ARRAY['reg_date']); + + -- Materialized view with equivalence: order_date = reg_date + CREATE MATERIALIZED VIEW order_summary + WITH (partitioning = ARRAY['order_date']) + AS + SELECT o.order_id, c.name, o.order_date + FROM orders o + JOIN customers c ON o.customer_id = c.customer_id + AND o.order_date = c.reg_date; -- Creates equivalence + +In this example: + +* ``orders.order_date`` and ``customers.reg_date`` are equivalent due to the equality join condition +* Even though ``reg_date`` is not in the SELECT list, staleness can be tracked through the equivalence to ``order_date`` +* When ``customers`` table changes in partition ``reg_date='2024-01-15'``, this maps to ``order_date='2024-01-15'`` for recomputation + +**How Passthrough Mapping Works** + +1. **Equivalence Extraction**: During materialized view creation, Presto analyzes JOIN conditions to identify + column equivalences + +2. **Staleness Detection**: When a base table changes: + + * The connector detects which data changed in the base table and returns predicates + * For passthrough columns, predicates are mapped through equivalences + * Example: ``customers.reg_date='2024-01-15'`` → ``orders.order_date='2024-01-15'`` + +3. **Predicate Application**: The mapped predicates are used in: + + * Storage scan: Exclude data where equivalent columns match stale values + * Recompute branch: Filter the stale table using the staleness predicate + * Joined tables: Propagate the predicate to equivalent columns in joined + tables, enabling pruning on those tables as well + +**Requirements for Passthrough Columns** + +* Join must be an INNER JOIN (not LEFT, RIGHT, or FULL OUTER) +* Equality must be direct (``col1 = col2``), not through expressions like ``col1 = col2 + 1`` +* At least one column in the equivalence class must be in the materialized view's output +* Data types must be compatible + +**Transitive Equivalences** + +Multiple equivalences can be chained together. If ``A.x = B.y`` and ``B.y = C.z``, then +``A.x``, ``B.y``, and ``C.z`` are all equivalent for predicate propagation. + +Unsupported Patterns +^^^^^^^^^^^^^^^^^^^^ + +Predicate stitching is **not** applied in the following cases: + +* **No staleness predicates available**: If the connector cannot provide staleness predicates +* **Predicate columns not preserved**: If predicate columns are transformed or not mappable to the materialized view's output +* **Outer joins with passthrough**: LEFT, RIGHT, and FULL OUTER joins invalidate passthrough equivalences due to null handling +* **Expression-based equivalences**: ``CAST(col1 AS DATE) = col2`` or ``col1 = col2 + 1`` + +When predicate stitching cannot be applied, the behavior falls back to the configured consistency mode: + +* If ``USE_STITCHING`` is set but stitching is not possible, the query falls back to full + recompute (equivalent to ``USE_VIEW_QUERY``) +* A warning may be logged indicating why stitching was not possible + +Performance Considerations +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**When Stitching is Most Effective** + +* **Large materialized views**: More benefit from avoiding full recomputation +* **Localized changes**: When only a small fraction of data is stale +* **Frequently refreshed**: When most data remains fresh between queries +* **Well-structured data**: When staleness predicates align with data modification patterns + +**Cost Trade-offs** + +Predicate stitching introduces a UNION operation, which has overhead: + +* **Storage scan overhead**: Reading from storage + filtering fresh data +* **Recompute overhead**: Querying base tables + filtering stale data +* **Union overhead**: Combining results from both branches + +However, this is typically much cheaper than: + +* **Full recompute**: Reading all base table data +* **Stale data**: Returning incorrect results + +**Optimization Tips** + +1. **Predicate granularity**: For partition-based connectors, choose partition columns that align + with data modification patterns + + * Too coarse (e.g., partitioning by year): Recomputes too much data + * Too fine (e.g., partitioning by second): Too many partitions to manage + +2. **Refresh frequency**: Balance freshness needs with refresh costs + + * More frequent refreshes: Less recomputation per query, but higher refresh costs + * Less frequent refreshes: More recomputation per query, but lower refresh costs + +3. **Query filters**: Include predicate columns in query filters when possible:: + + -- Good: Limits scan to relevant data + SELECT * FROM mv WHERE order_date >= '2024-01-01' + + -- Less optimal: Scans all data + SELECT * FROM mv WHERE customer_id = 12345 + +4. **Monitor metrics**: Track the ratio of storage scan vs recompute: + + * High recompute ratio: Consider more frequent refreshes or better staleness granularity + * High storage scan ratio: Stitching is working efficiently + See Also -------- diff --git a/presto-docs/src/main/sphinx/admin/properties-session.rst b/presto-docs/src/main/sphinx/admin/properties-session.rst index f8a46b4063e30..dad07f4e654eb 100644 --- a/presto-docs/src/main/sphinx/admin/properties-session.rst +++ b/presto-docs/src/main/sphinx/admin/properties-session.rst @@ -50,7 +50,7 @@ The corresponding configuration property is :ref:`admin/properties:\`\`join-dist ^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** ``boolean`` -* **Default value:** ``true`` +* **Default value:** ``false`` This property enables redistribution of data before writing. This can eliminate the performance impact of data skew when writing by hashing it @@ -58,8 +58,27 @@ across nodes in the cluster. It can be disabled when it is known that the output data set is not skewed in order to avoid the overhead of hashing and redistributing all the data across the network. +When both ``scale_writers`` and ``redistribute_writes`` are set to ``true``, +``scale_writers`` takes precedence. + The corresponding configuration property is :ref:`admin/properties:\`\`redistribute-writes\`\``. +``scale_writers`` +^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``true`` + +This property enables dynamic scaling of writer tasks based on throughput. When enabled, +Presto automatically adjusts the number of writer tasks to use the minimum necessary +for optimal performance. This can improve resource utilization by scaling out writers +only when needed based on data throughput. + +When both ``scale_writers`` and ``redistribute_writes`` are set to ``true``, +``scale_writers`` takes precedence. + +The corresponding configuration property is :ref:`admin/properties:\`\`scale-writers\`\``. + ``task_writer_count`` ^^^^^^^^^^^^^^^^^^^^^ @@ -148,6 +167,27 @@ If it’s below the limit, the generated prefixes are used. The corresponding configuration property is :ref:`admin/properties:\`\`max-prefixes-count\`\``. +``try_function_catchable_errors`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``""`` (empty string) + +A comma-separated list of error code names that the ``TRY()`` function should catch +and return ``NULL`` for, in addition to the default catchable errors (such as +``DIVISION_BY_ZERO``, ``INVALID_CAST_ARGUMENT``, ``INVALID_FUNCTION_ARGUMENT``, +and ``NUMERIC_VALUE_OUT_OF_RANGE``). + +This allows users to specify exactly which additional errors ``TRY()`` should suppress. +Error codes are matched by their name (such as ``GENERIC_INTERNAL_ERROR``, ``INVALID_ARGUMENTS``). + +Example usage:: + + SET SESSION try_function_catchable_errors = 'GENERIC_INTERNAL_ERROR,INVALID_ARGUMENTS'; + SELECT TRY(my_function(x)) FROM table; + +The corresponding configuration property is :ref:`admin/properties:\`\`try-function-catchable-errors\`\``. + Spilling Properties ------------------- @@ -160,9 +200,8 @@ Spilling Properties Try spilling memory to disk to avoid exceeding memory limits for the query. Spilling works by offloading memory to disk. This process can allow a query with a large memory -footprint to pass at the cost of slower execution times. Currently, spilling is supported only for -aggregations and joins (inner and outer), so this property will not reduce memory usage required for -window functions, sorting and other join types. +footprint to pass at the cost of slower execution times. See :ref:`spill-operations` +for a list of operations that support spilling. Be aware that this is an experimental feature and should be used with care. @@ -331,6 +370,19 @@ queries that have very selective joins. The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.push-aggregation-through-join\`\``. +``push_partial_aggregation_through_join`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +When a partial aggregation is above an inner join and all aggregation inputs come from +only one side of the join, the partial aggregation is pushed below the join to that side. +This reduces the amount of data flowing into the join operator, which can improve +performance by allowing the aggregation to pre-reduce data before the join is performed. + +The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.push-partial-aggregation-through-join\`\``. + ``push_table_write_through_union`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -447,6 +499,17 @@ Use this to optimize the ``map_filter()`` and ``map_subset()`` function. It controls if subfields access is executed at the data source or not. +``pushdown_subfields_for_cardinality`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* **Type:** ``boolean`` +* **Default value:** ``false`` + +Enable subfield pruning for the ``cardinality()`` function to skip reading keys and values. + +When enabled, the query optimizer can push down subfield pruning for cardinality operations, +allowing the data source to skip reading the actual keys and values when only the cardinality +(count of elements) is needed. + ``schedule_splits_based_on_task_load`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** ``boolean`` @@ -486,6 +549,59 @@ parallelism factor is below the ``table_scan_shuffle_parallelism_threshold``. The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.table-scan-shuffle-strategy\`\``. +``remote_function_names_for_fixed_parallelism`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``""`` (empty string, disabled) + +A regular expression pattern to match fully qualified remote function names, such as ``catalog.schema.function_name``, +that should use fixed parallelism. When a remote function matches this pattern, the optimizer inserts +round-robin shuffle exchanges before and after the projection containing the remote function call. +This ensures that the remote function executes with a fixed degree of parallelism, which can be useful +for controlling resource usage when calling external services. + +This property only applies to external/remote functions (functions where ``isExternalExecution()`` returns ``true``, +such as functions using THRIFT, GRPC, or REST implementation types). + +Example patterns: + +* ``myschema.myfunction`` - matches an exact function name +* ``catalog.schema.remote_.*`` - matches all functions starting with ``remote_`` in the specified catalog and schema +* ``.*remote.*`` - matches any function containing ``remote`` in its fully qualified name + +The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.remote-function-names-for-fixed-parallelism\`\``. + +``remote_function_fixed_parallelism_task_count`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Default value:** ``null`` (uses the default hash partition count) + +The number of tasks to use for remote functions matching the ``remote_function_names_for_fixed_parallelism`` pattern. +When set, this value determines the degree of parallelism for the round-robin shuffle exchanges inserted +around matching remote function projections. If not set, the default hash partition count will be used. + +This property is only effective when ``remote_function_names_for_fixed_parallelism`` is set to a non-empty pattern. + +The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.remote-function-fixed-parallelism-task-count\`\``. + +``local_exchange_parent_preference_strategy`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Allowed values:** ``ALWAYS``, ``NEVER``, ``AUTOMATIC`` +* **Default value:** ``ALWAYS`` + +Strategy to consider parent preferences when adding local exchange partitioning for aggregations. +When set to ``ALWAYS``, the optimizer always uses parent preferences for local exchange partitioning. +When set to ``NEVER``, it never uses parent preferences and instead uses the aggregation's own +grouping keys. When set to ``AUTOMATIC``, the optimizer makes a cost-based decision, using parent +preferences only when the estimated partition cardinality is greater than or equal to the task +concurrency. + +The corresponding configuration property is :ref:`admin/properties:\`\`optimizer.local-exchange-parent-preference-strategy\`\``. + JDBC Properties --------------- diff --git a/presto-docs/src/main/sphinx/admin/properties.rst b/presto-docs/src/main/sphinx/admin/properties.rst index bb84818e9645f..4a89a9c229ed3 100644 --- a/presto-docs/src/main/sphinx/admin/properties.rst +++ b/presto-docs/src/main/sphinx/admin/properties.rst @@ -2,12 +2,12 @@ Presto Configuration Properties =============================== -This section describes configuration properties that may be used to tune +This section describes configuration properties that may be used to tune Presto or alter its behavior when required. -The following is not a complete list of all configuration properties +The following is not a complete list of all configuration properties available in Presto, and does not include any connector-specific -catalog configuration properties. +catalog configuration properties. For information on catalog configuration properties, see the :doc:`connector documentation `. @@ -40,25 +40,44 @@ only need to fit in distributed memory across all nodes. When set to ``AUTOMATIC Presto will make a cost based decision as to which distribution type is optimal. It will also consider switching the left and right inputs to the join. In ``AUTOMATIC`` mode, Presto will default to hash distributed joins if no cost could be computed, such as if -the tables do not have statistics. +the tables do not have statistics. -The corresponding session property is :ref:`admin/properties-session:\`\`join_distribution_type\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`join_distribution_type\`\``. ``redistribute-writes`` ^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** ``boolean`` -* **Default value:** ``true`` +* **Default value:** ``false`` This property enables redistribution of data before writing. This can eliminate the performance impact of data skew when writing by hashing it across nodes in the cluster. It can be disabled when it is known that the output data set is not skewed in order to avoid the overhead of hashing and -redistributing all the data across the network. +redistributing all the data across the network. + +When both ``scale-writers`` and ``redistribute-writes`` are set to ``true``, +``scale-writers`` takes precedence. The corresponding session property is :ref:`admin/properties-session:\`\`redistribute_writes\`\``. +``scale-writers`` +^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``true`` + +This property enables dynamic scaling of writer tasks based on throughput. When enabled, +Presto automatically adjusts the number of writer tasks to use the minimum necessary +for optimal performance. This can improve resource utilization by scaling out writers +only when needed based on data throughput. + +When both ``scale-writers`` and ``redistribute-writes`` are set to ``true``, +``scale-writers`` takes precedence. + +The corresponding session property is :ref:`admin/properties-session:\`\`scale_writers\`\``. + ``check-access-control-on-utilized-columns-only`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -124,8 +143,8 @@ session properties are included. * **Minimum value:** ``0`` * **Default value:** ``0`` -The number of times that a query is automatically retried in the case of a transient query or communications failure. -The default value ``0`` means that retries are disabled. +The number of times that a query is automatically retried in the case of a transient query or communications failure. +The default value ``0`` means that retries are disabled. ``http-server.max-request-header-size`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -133,10 +152,10 @@ The default value ``0`` means that retries are disabled. * **Type:** ``data size`` * **Default value:** ``8 kB`` -The maximum size of the request header from the HTTP server. +The maximum size of the request header from the HTTP server. -Note: The default value can cause errors when large session properties -or other large session information is involved. +Note: The default value can cause errors when large session properties +or other large session information is involved. See :ref:`troubleshoot/query:\`\`Request Header Fields Too Large\`\``. ``offset-clause-enabled`` @@ -147,7 +166,7 @@ See :ref:`troubleshoot/query:\`\`Request Header Fields Too Large\`\``. To enable the ``OFFSET`` clause in SQL query expressions, set this property to ``true``. -The corresponding session property is :ref:`admin/properties-session:\`\`offset_clause_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`offset_clause_enabled\`\``. ``max-serializable-object-size`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -178,9 +197,26 @@ The corresponding session property is :ref:`admin/properties-session:\`\`max_pre * **Type:** ``string`` * **Default value:** (none) -An optional identifier for the cluster. When set, this tag is included in the response from the +An optional identifier for the cluster. When set, this tag is included in the response from the ``/v1/cluster`` REST API endpoint, allowing clients to identify which cluster provided the response. +``try-function-catchable-errors`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``""`` (empty string) + +A comma-separated list of error code names that the ``TRY()`` function should catch +and return ``NULL`` for, in addition to the default catchable errors (such as +``DIVISION_BY_ZERO``, ``INVALID_CAST_ARGUMENT``, ``INVALID_FUNCTION_ARGUMENT``, +and ``NUMERIC_VALUE_OUT_OF_RANGE``). + +This allows administrators to configure which additional errors ``TRY()`` should suppress +at the server level. Error codes are matched by their name (such as ``GENERIC_INTERNAL_ERROR``, +``INVALID_ARGUMENTS``). + +The corresponding session property is :ref:`admin/properties-session:\`\`try_function_catchable_errors\`\``. + Memory Management Properties ---------------------------- @@ -274,13 +310,12 @@ Spilling Properties Try spilling memory to disk to avoid exceeding memory limits for the query. Spilling works by offloading memory to disk. This process can allow a query with a large memory -footprint to pass at the cost of slower execution times. Currently, spilling is supported only for -aggregations and joins (inner and outer), so this property will not reduce memory usage required for -window functions, sorting and other join types. +footprint to pass at the cost of slower execution times. See :ref:`spill-operations` +for a list of operations that support spilling. Be aware that this is an experimental feature and should be used with care. -The corresponding session property is :ref:`admin/properties-session:\`\`spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`spill_enabled\`\``. ``experimental.join-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -291,7 +326,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`spill_e When ``spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for joins to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`join_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`join_spill_enabled\`\``. ``experimental.aggregation-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -302,7 +337,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`join_sp When ``spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for aggregations to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`aggregation_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`aggregation_spill_enabled\`\``. ``experimental.distinct-aggregation-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -313,7 +348,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`aggrega When ``aggregation_spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for distinct aggregations to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`distinct_aggregation_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`distinct_aggregation_spill_enabled\`\``. ``experimental.order-by-aggregation-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -324,7 +359,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`distinc When ``aggregation_spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for order by aggregations to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`order_by_aggregation_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`order_by_aggregation_spill_enabled\`\``. ``experimental.window-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -335,7 +370,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`order_b When ``spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for window functions to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`window_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`window_spill_enabled\`\``. ``experimental.order-by-spill-enabled`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -346,7 +381,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`window_ When ``spill_enabled`` is ``true``, this determines whether Presto will try spilling memory to disk for order by to avoid exceeding memory limits for the query. -The corresponding session property is :ref:`admin/properties-session:\`\`order_by_spill_enabled\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`order_by_spill_enabled\`\``. ``experimental.spiller.task-spilling-strategy`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -472,7 +507,7 @@ Max spill space to be used by a single query on a single node. Limit for memory used for unspilling a single aggregation operator instance. -The corresponding session property is :ref:`admin/properties-session:\`\`aggregation_operator_unspill_memory_limit\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`aggregation_operator_unspill_memory_limit\`\``. ``experimental.spill-compression-codec`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -599,16 +634,16 @@ network has high latency or if there are many nodes in the cluster. * **Type:** ``boolean`` * **Default value:** ``false`` -Enables the use of custom connector-provided serialization codecs for handles. +Enables the use of custom connector-provided serialization codecs for handles. This feature allows connectors to use their own serialization format for handle objects (such as table handles, column handles, and splits) instead of standard JSON serialization. -When enabled, connectors that provide a ``ConnectorCodecProvider`` with -appropriate codecs will have their handles serialized using custom binary -formats, which are then Base64-encoded for transport. Connectors without -codec support automatically fall back to standard JSON serialization. -Internal Presto handles (prefixed with ``$``) always use JSON serialization +When enabled, connectors that provide a ``ConnectorCodecProvider`` with +appropriate codecs will have their handles serialized using custom binary +formats, which are then Base64-encoded for transport. Connectors without +codec support automatically fall back to standard JSON serialization. +Internal Presto handles (prefixed with ``$``) always use JSON serialization regardless of this setting. .. _task-properties: @@ -629,9 +664,9 @@ resource utilization. Lower values are better for clusters that run many queries concurrently because the cluster will already be utilized by all the running queries, so adding more concurrency will result in slow downs due to context switching and other overhead. Higher values are better for clusters that only run -one or a few queries at a time. +one or a few queries at a time. -The corresponding session property is :ref:`admin/properties-session:\`\`task_concurrency\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`task_concurrency\`\``. ``task.http-response-threads`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -691,7 +726,7 @@ can improve throughput if worker CPU utilization is low and all the threads are but will cause increased heap space usage. Setting the value too high may cause a drop in performance due to a context switching. The number of active threads is available via the ``RunningSplits`` property of the -``com.facebook.presto.execution.executor:name=TaskExecutor.RunningSplits`` JXM object. +``com.facebook.presto.execution.executor:name=TaskExecutor.RunningSplits`` JMX object. The number of threads can be configured using either an absolute value (for example, ``10``) or a value relative to the number of available CPU cores (for example, ``1.5C``). When @@ -722,9 +757,9 @@ The number of concurrent writer threads per worker per query. Increasing this va increase write speed, especially when a query is not I/O bound and can take advantage of additional CPU for parallel writes (some connectors can be bottlenecked on CPU when writing due to compression or other factors). Setting this too high may cause the cluster -to become overloaded due to excessive resource utilization. +to become overloaded due to excessive resource utilization. -The corresponding session property is :ref:`admin/properties-session:\`\`task_writer_count\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`task_writer_count\`\``. ``task.interrupt-runaway-splits-timeout`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -840,9 +875,9 @@ Optimizer Properties * **Type:** ``boolean`` * **Default value:** ``false`` -Enables optimization for aggregations on dictionaries. +Enables optimization for aggregations on dictionaries. -The corresponding session property is :ref:`admin/properties-session:\`\`dictionary_aggregation\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`dictionary_aggregation\`\``. ``optimizer.optimize-hash-generation`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -854,12 +889,12 @@ Compute hash codes for distribution, joins, and aggregations early during execut allowing result to be shared between operations later in the query. This can reduce CPU usage by avoiding computing the same hash multiple times, but at the cost of additional network transfer for the hashes. In most cases it will decrease overall -query processing time. +query processing time. It is often helpful to disable this property when using :doc:`/sql/explain` in order to make the query plan easier to read. -The corresponding session property is :ref:`admin/properties-session:\`\`optimize_hash_generation\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`optimize_hash_generation\`\``. ``optimizer.optimize-metadata-queries`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -905,9 +940,22 @@ over an outer join. For example:: Enabling this optimization can substantially speed up queries by reducing the amount of data that needs to be processed by the join. However, it may slow down some -queries that have very selective joins. +queries that have very selective joins. + +The corresponding session property is :ref:`admin/properties-session:\`\`push_aggregation_through_join\`\``. -The corresponding session property is :ref:`admin/properties-session:\`\`push_aggregation_through_join\`\``. +``optimizer.push-partial-aggregation-through-join`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +When a partial aggregation is above an inner join and all aggregation inputs come from +only one side of the join, the partial aggregation is pushed below the join to that side. +This reduces the amount of data flowing into the join operator, which can improve +performance by allowing the aggregation to pre-reduce data before the join is performed. + +The corresponding session property is :ref:`admin/properties-session:\`\`push_partial_aggregation_through_join\`\``. ``optimizer.push-table-write-through-union`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -919,9 +967,9 @@ Parallelize writes when using ``UNION ALL`` in queries that write data. This imp speed of writing output tables in ``UNION ALL`` queries because these writes do not require additional synchronization when collecting results. Enabling this optimization can improve ``UNION ALL`` speed when write speed is not yet saturated. However, it may slow down queries -in an already heavily loaded system. +in an already heavily loaded system. -The corresponding session property is :ref:`admin/properties-session:\`\`push_table_write_through_union\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`push_table_write_through_union\`\``. ``optimizer.join-reordering-strategy`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -935,9 +983,9 @@ query. ``ELIMINATE_CROSS_JOINS`` reorders joins to eliminate cross joins where otherwise maintains the original query order. When reordering joins it also strives to maintain the original table order as much as possible. ``AUTOMATIC`` enumerates possible orders and uses statistics-based cost estimation to determine the least cost order. If stats are not available or if -for any reason a cost could not be computed, the ``ELIMINATE_CROSS_JOINS`` strategy is used. +for any reason a cost could not be computed, the ``ELIMINATE_CROSS_JOINS`` strategy is used. -The corresponding session property is :ref:`admin/properties-session:\`\`join_reordering_strategy\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`join_reordering_strategy\`\``. ``optimizer.max-reordered-joins`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1020,7 +1068,7 @@ Enable broadcasting based on the confidence of the statistics that are being use broadcasting the side of a joinNode which has the highest (``HIGH`` or ``FACT``) confidence statistics. If both sides have the same confidence statistics, then the original behavior will be followed. -The corresponding session property is :ref:`admin/properties-session:\`\`confidence_based_broadcast\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`confidence_based_broadcast\`\``. ``optimizer.treat-low-confidence-zero-estimation-as-unknown`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1028,9 +1076,9 @@ The corresponding session property is :ref:`admin/properties-session:\`\`confide * **Type:** ``boolean`` * **Default value:** ``false`` -Enable treating ``LOW`` confidence, zero estimations as ``UNKNOWN`` during joins. +Enable treating ``LOW`` confidence, zero estimations as ``UNKNOWN`` during joins. -The corresponding session property is :ref:`admin/properties-session:\`\`treat-low-confidence-zero-estimation-as-unknown\`\``. +The corresponding session property is :ref:`admin/properties-session:\`\`treat-low-confidence-zero-estimation-as-unknown\`\``. ``optimizer.retry-query-with-history-based-optimization`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1038,7 +1086,7 @@ The corresponding session property is :ref:`admin/properties-session:\`\`treat-l * **Type:** ``boolean`` * **Default value:** ``false`` -Enable retry for failed queries who can potentially be helped by HBO. +Enable retry for failed queries who can potentially be helped by HBO. The corresponding session property is :ref:`admin/properties-session:\`\`retry-query-with-history-based-optimization\`\``. @@ -1100,6 +1148,59 @@ parallelism factor is below the ``optimizer.table-scan-shuffle-parallelism-thres The corresponding session property is :ref:`admin/properties-session:\`\`table_scan_shuffle_strategy\`\``. +``optimizer.remote-function-names-for-fixed-parallelism`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``""`` (empty string, disabled) + +A regular expression pattern to match fully qualified remote function names, such as ``catalog.schema.function_name``, +that should use fixed parallelism. When a remote function matches this pattern, the optimizer inserts +round-robin shuffle exchanges before and after the projection containing the remote function call. +This ensures that the remote function executes with a fixed degree of parallelism, which can be useful +for controlling resource usage when calling external services. + +This property only applies to external/remote functions (functions where ``isExternalExecution()`` returns ``true``, +such as functions using THRIFT, GRPC, or REST implementation types). + +Example patterns: + +* ``myschema.myfunction`` - matches an exact function name +* ``catalog.schema.remote_.*`` - matches all functions starting with ``remote_`` in the specified catalog and schema +* ``.*remote.*`` - matches any function containing ``remote`` in its fully qualified name + +The corresponding session property is :ref:`admin/properties-session:\`\`remote_function_names_for_fixed_parallelism\`\``. + +``optimizer.remote-function-fixed-parallelism-task-count`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Default value:** ``null`` (uses the default hash partition count) + +The number of tasks to use for remote functions matching the ``optimizer.remote-function-names-for-fixed-parallelism`` pattern. +When set, this value determines the degree of parallelism for the round-robin shuffle exchanges inserted +around matching remote function projections. If not set, the default hash partition count will be used. + +This property is only effective when ``optimizer.remote-function-names-for-fixed-parallelism`` is set to a non-empty pattern. + +The corresponding session property is :ref:`admin/properties-session:\`\`remote_function_fixed_parallelism_task_count\`\``. + +``optimizer.local-exchange-parent-preference-strategy`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Allowed values:** ``ALWAYS``, ``NEVER``, ``AUTOMATIC`` +* **Default value:** ``ALWAYS`` + +Strategy to consider parent preferences when adding local exchange partitioning for aggregations. +When set to ``ALWAYS``, the optimizer always uses parent preferences for local exchange partitioning. +When set to ``NEVER``, it never uses parent preferences and instead uses the aggregation's own +grouping keys. When set to ``AUTOMATIC``, the optimizer makes a cost-based decision, using parent +preferences only when the estimated partition cardinality is greater than or equal to the task +concurrency. + +The corresponding session property is :ref:`admin/properties-session:\`\`local_exchange_parent_preference_strategy\`\``. + Planner Properties ------------------ @@ -1248,6 +1349,72 @@ Use to configure how long a query can be queued before it is terminated. The corresponding session property is :ref:`admin/properties-session:\`\`query_max_queued_time\`\``. +``query-manager.query-pacing.max-queries-per-second`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Minimum value:** ``1`` +* **Default value:** ``2147483647`` (unlimited) + +Maximum number of queries that can be admitted per second globally across +all resource groups. This property enables query admission pacing to prevent +worker overload when many queries start simultaneously. Pacing only activates +when the number of running queries exceeds the threshold configured by +``query-manager.query-pacing.min-running-queries``. + +Set to a lower value such as ``10`` to limit query admission rate during +periods of high cluster load. The default value effectively disables pacing. + +``query-manager.query-pacing.min-running-queries`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Minimum value:** ``0`` +* **Default value:** ``30`` + +Minimum number of running queries required before query admission pacing +is applied. When the total number of running queries is below this threshold, +queries are admitted immediately without rate limiting, regardless of the +``query-manager.query-pacing.max-queries-per-second`` setting. + +This allows the cluster to quickly ramp up when idle while still providing +protection against overload when the cluster is busy. Set to ``0`` to always +apply pacing when ``max-queries-per-second`` is configured. + +``max-total-running-task-count-to-not-execute-new-query`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Minimum value:** ``1`` +* **Default value:** ``2147483647`` (unlimited) + +Maximum total running task count across all queries on the coordinator. When +this threshold is exceeded, new queries are held in the queue rather than +being scheduled for execution. This helps prevent coordinator overload by +limiting the number of concurrent tasks being managed. + +Unlike ``max-total-running-task-count-to-kill-query`` which kills queries when +the limit is exceeded, this property proactively prevents new queries from +starting while allowing existing queries to complete normally. + +This property works in conjunction with query admission pacing +(``query-manager.query-pacing.max-queries-per-second``) to provide +comprehensive coordinator load management. When both are configured: + +1. Pacing controls the rate at which queries are admitted +2. This property provides a hard cap on total concurrent tasks + +Without query-pacing, the cluster can admit multiple queries at once, which +can lead to significantly more concurrent tasks than expected over this limit. + +Set to a lower value (e.g., ``50000``) to limit coordinator task management +overhead. The default value effectively disables this feature. + +.. note:: + + For backwards compatibility, this property can also be configured using the + legacy name ``experimental.max-total-running-task-count-to-not-execute-new-query``. + Query Retry Properties ---------------------- diff --git a/presto-docs/src/main/sphinx/admin/spill.rst b/presto-docs/src/main/sphinx/admin/spill.rst index f0c51e751dd21..3af869c61f877 100644 --- a/presto-docs/src/main/sphinx/admin/spill.rst +++ b/presto-docs/src/main/sphinx/admin/spill.rst @@ -2,11 +2,6 @@ Spill to Disk ============= -.. contents:: - :local: - :backlinks: none - :depth: 1 - Overview -------- @@ -30,7 +25,7 @@ of memory to queries and prevents deadlock caused by memory allocation. It is efficient when there are a lot of small queries in the cluster, but leads to killing large queries that don't stay within the limits. -To overcome this inefficiency, the concept of revocable memory was introduced. A +To overcome this limitation, the concept of revocable memory was introduced. A query can request memory that does not count toward the limits, but this memory can be revoked by the memory manager at any time. When memory is revoked, the query runner spills intermediate data from memory to disk and continues to @@ -107,6 +102,8 @@ When spill encryption is enabled (``spill-encryption-enabled`` property in (per spill file) secret key. Enabling this will decrease the performance of spilling to disk but can protect spilled data from being recovered from the files written to disk. +.. _spill-operations: + Supported Operations -------------------- diff --git a/presto-docs/src/main/sphinx/cache/local.rst b/presto-docs/src/main/sphinx/cache/local.rst index 5b8efba0f8cb1..98a458d3edd32 100644 --- a/presto-docs/src/main/sphinx/cache/local.rst +++ b/presto-docs/src/main/sphinx/cache/local.rst @@ -2,11 +2,6 @@ Alluxio SDK Cache ================= -.. contents:: - :local: - :backlinks: none - :depth: 1 - Overview -------- diff --git a/presto-docs/src/main/sphinx/cache/service.rst b/presto-docs/src/main/sphinx/cache/service.rst index 4b11cee30e703..4313d1fb2a6f0 100644 --- a/presto-docs/src/main/sphinx/cache/service.rst +++ b/presto-docs/src/main/sphinx/cache/service.rst @@ -2,11 +2,6 @@ Alluxio Cache Service ===================== -.. contents:: - :local: - :backlinks: none - :depth: 1 - Overview -------- diff --git a/presto-docs/src/main/sphinx/conf.py b/presto-docs/src/main/sphinx/conf.py index ea71636098d47..0ba35583db294 100644 --- a/presto-docs/src/main/sphinx/conf.py +++ b/presto-docs/src/main/sphinx/conf.py @@ -64,7 +64,7 @@ def get_version(): needs_sphinx = '8.2.1' extensions = [ - 'sphinx_immaterial', 'sphinx_copybutton', 'download', 'issue', 'pr', 'sphinx.ext.autosectionlabel' + 'sphinx_immaterial', 'download', 'issue', 'pr', 'sphinx.ext.autosectionlabel' ] copyright = 'The Presto Foundation. All rights reserved. Presto is a registered trademark of LF Projects, LLC' @@ -106,13 +106,8 @@ def get_version(): html_logo = 'images/logo.png' html_favicon = 'images/favicon.ico' -# doesn't seem to do anything -# html_baseurl = 'overview.html' - html_static_path = ['.'] -templates_path = ['_templates'] - # Set the primary domain to js because if left as the default python # the theme errors when functions aren't available in a python module primary_domain = 'js' diff --git a/presto-docs/src/main/sphinx/connector.rst b/presto-docs/src/main/sphinx/connector.rst index d337fe4ed12d1..00221c91e86a7 100644 --- a/presto-docs/src/main/sphinx/connector.rst +++ b/presto-docs/src/main/sphinx/connector.rst @@ -27,6 +27,7 @@ from different data sources. connector/kafka connector/kafka-tutorial connector/kudu + connector/lance connector/larksheets connector/localfile connector/memory diff --git a/presto-docs/src/main/sphinx/connector/hive.rst b/presto-docs/src/main/sphinx/connector/hive.rst index e3c0c22a7576d..76daed9d6f773 100644 --- a/presto-docs/src/main/sphinx/connector/hive.rst +++ b/presto-docs/src/main/sphinx/connector/hive.rst @@ -164,9 +164,18 @@ Property Name Description absolutely necessary to access HDFS. Example: ``/etc/hdfs-site.xml`` -``hive.storage-format`` The default file format used when creating new tables. ``ORC`` - -``hive.compression-codec`` The compression codec to use when writing files. ``GZIP`` +``hive.storage-format`` The default file format used when creating new tables. The ``ORC`` + available values are ``ORC``, ``PARQUET``, ``AVRO``, + ``RCBINARY``, ``RCTEXT``, ``SEQUENCEFILE``, ``JSON``, + and ``TEXTFILE``. + +``hive.compression-codec`` The compression codec to use when writing files. The ``GZIP`` + available values are ``NONE``, ``SNAPPY``, ``GZIP``, + ``LZ4``, and ``ZSTD``. + + Note: ``LZ4`` is only available when + ``hive.storage-format=ORC``. ``ZSTD`` is available + for both ``ORC`` and ``PARQUET`` formats. ``hive.force-local-scheduling`` Force splits to be scheduled on the same node as the Hadoop ``false`` DataNode process serving the split data. This is useful for @@ -236,6 +245,18 @@ Property Name Description .. _constructor: https://github.com/apache/hadoop/blob/02a9190af5f8264e25966a80c8f9ea9bb6677899/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java#L844-L875 +Hive Session Properties +----------------------- + +======================================================== ============================================================ ============ +Property Name Description Default +======================================================== ============================================================ ============ +``native_max_target_file_size`` Native Execution only. Maximum target file size. When a ``0B`` + file exceeds this size during writing, the writer will + close the current file and start writing to a new file. + Zero means no limit. +======================================================== ============================================================ ============ + Avro Configuration Properties ----------------------------- @@ -279,6 +300,33 @@ Add the ``metastore.storage.schema.reader.impl`` property to ``hive-site.xml`` w You must restart the metastore service for this configuration to take effect. This setting allows the metastore to read storage schemas for Avro tables and avoids ``Storage schema reading not supported`` errors. +Textfile Configuration Properties +--------------------------------- + +Table Properties +^^^^^^^^^^^^^^^^ + +These properties can be used when creating TEXTFILE tables in Presto: + +======================================================== ============================================================================== ============================= +Property Name Description Default +======================================================== ============================================================================== ============================= +``textfile_field_delim`` A custom single-character delimiter to separate fields. NONE + +``textfile_escape_delim`` A custom single-character delimiter to escape characters. NONE + +``textfile_collection_delim`` A custom single-character delimiter to separate collection elements. NONE + +``textfile_mapkey_delim`` A custom single-character delimiter to separate map keys. NONE + +======================================================== ============================================================================== ============================= + +.. note:: +These properties are mapped to the corresponding properties in Hive ``LazySerDeParameters`` during serialization and +follow the same behaviors with ``LazySimpleSerDe``. +If they are not defined, the Hive defaults are used, which are typically ``\001`` for field delimiter, ``\002`` for +collection delimiter, ``\003`` for map key delimiter, and escape character is disabled. + Metastore Configuration Properties ---------------------------------- @@ -289,15 +337,29 @@ Property Name Descriptio ======================================================== ============================================================= ============ ``hive.metastore-timeout`` Timeout for Hive metastore requests. ``10s`` -``hive.metastore-cache-ttl`` Duration how long cached metastore data should be considered ``0s`` +``hive.metastore.cache.enabled-caches`` Comma-separated list of metastore cache types to enable. NONE + The value should be a valid . + +``hive.metastore.cache.disabled-caches`` Comma-separated list of metastore cache types to disable. NONE + The value should be a valid . + +``hive.metastore.cache.ttl.default`` Duration how long cached metastore data should be considered ``0s`` valid. +``hive.metastore.cache.ttl-by-type`` Per-cache time-to-live (TTL) overrides for Hive metastore NONE + caches. The value is a comma-separated list of + : pairs. + ``hive.metastore-cache-maximum-size`` Hive metastore cache maximum size. 10000 -``hive.metastore-refresh-interval`` Asynchronously refresh cached metastore data after access ``0s`` +``hive.metastore.cache.refresh-interval.default`` Asynchronously refresh cached metastore data after access ``0s`` if it is older than this but is not yet expired, allowing subsequent accesses to see fresh data. +``hive.metastore.cache.refresh-interval-by-type`` Per-cache refresh interval overrides for Hive metastore NONE + caches. The value is a comma-separated list of + : pairs. + ``hive.metastore-refresh-max-threads`` Maximum threads used to refresh cached metastore data. 100 ``hive.invalidate-metastore-cache-procedure-enabled`` When enabled, users will be able to invalidate metastore false @@ -315,6 +377,26 @@ Property Name Descriptio ======================================================== ============================================================= ============ +.. note:: + + The supported values for ``CACHE_TYPE`` when enabling Hive Metastore Cache are: + + * ``ALL``: Represents all supported Hive metastore cache types. + * ``DATABASE``: Caches metadata for individual Hive databases. + * ``DATABASE_NAMES``: Caches the list of all database names in the metastore. + * ``TABLE``: Caches metadata for individual Hive tables. + * ``TABLE_NAMES``: Caches the list of table names within a database. + * ``TABLE_STATISTICS``: Caches column-level statistics for Hive tables. + * ``TABLE_CONSTRAINTS``: Caches table constraint metadata such as primary and unique keys. + * ``PARTITION``: Caches metadata for individual Hive partitions. + * ``PARTITION_STATISTICS``: Caches column-level statistics for individual partitions. + * ``PARTITION_FILTER``: Caches partition name lookups based on partition filter predicates. + * ``PARTITION_NAMES``: Caches the list of partition names for a table. + * ``VIEW_NAMES``: Caches the list of view names within a database. + * ``TABLE_PRIVILEGES``: Caches table-level privilege information for users and roles. + * ``ROLES``: Caches the list of available Hive roles. + * ``ROLE_GRANTS``: Caches role grant mappings for principals. + AWS Glue Catalog Configuration Properties ----------------------------------------- @@ -1262,4 +1344,4 @@ Example:: CAST(id AS BIGINT) AS id, CAST(value AS INT) AS value, CAST(date_col AS DATE) AS date_col - FROM hive.csv.csv_data; \ No newline at end of file + FROM hive.csv.csv_data; diff --git a/presto-docs/src/main/sphinx/connector/hudi.rst b/presto-docs/src/main/sphinx/connector/hudi.rst index 6d6ef20b17187..4d928ad6832ef 100644 --- a/presto-docs/src/main/sphinx/connector/hudi.rst +++ b/presto-docs/src/main/sphinx/connector/hudi.rst @@ -6,9 +6,9 @@ Overview -------- The Hudi connector enables querying `Hudi `_ tables -synced to Hive metastore. The connector usesthe metastore only to track partition locations. +synced to Hive metastore. The connector uses the metastore only to track partition locations. It makes use of the underlying Hudi filesystem and input formats to list data files. To learn -more about the design of the connector, please check out `RFC-40 `_. Requirements diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index 5783c3e4c7cad..391b3dba70a73 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -557,6 +557,8 @@ Property Name Description names. Default: ``__mv_storage__`` ``materialized_view_missing_base_table_behavior`` Behavior when a base table referenced by a materialized view is Yes No missing. Valid values: ``FAIL``, ``IGNORE``. Default: ``FAIL`` +``max_partitions_per_writer`` Overrides the behavior of the connector property Yes No + ``iceberg.max-partitions-per-writer`` in the current session. ===================================================== ======================================================================= =================== ============================================= Caching Support @@ -679,7 +681,21 @@ File and stripe footer cache is not applicable for Presto C++. Metastore Cache ^^^^^^^^^^^^^^^ -Iceberg Connector does not support Metastore Caching. +Iceberg Connector supports Metastore Caching with some exceptions. Iceberg Connector does not allow enabling TABLE cache. +Metastore Caching is only supported when ``iceberg.catalog.type`` is ``HIVE``. + +The Iceberg connector supports the same configuration properties for +`Hive Metastore Caching `_ +as a Hive connector. + +The following configuration properties are the minimum set of configurations required to be added in the Iceberg catalog file ``catalog/iceberg.properties``: + +.. code-block:: none + + # Hive Metastore Cache + hive.metastore.cache.disabled-caches=TABLE + hive.metastore.cache.ttl.default=10m + hive.metastore.cache.refresh-interval.default=5m Extra Hidden Metadata Columns ----------------------------- @@ -968,7 +984,7 @@ Register Table Iceberg tables for which table data and metadata already exist in the file system can be registered with the catalog. Use the ``register_table`` procedure on the catalog's ``system`` schema to register a table which -already exists but does not known by the catalog. +already exists but is not known by the catalog. The following arguments are available: @@ -1589,6 +1605,42 @@ Alter table operations are supported in the Iceberg connector:: ALTER TABLE iceberg.web.page_views DROP TAG 'tag1'; + ALTER TABLE iceberg.default.mytable CREATE BRANCH 'audit-branch'; + + ALTER TABLE iceberg.default.mytable CREATE BRANCH IF NOT EXISTS 'audit-branch'; + + ALTER TABLE iceberg.default.mytable CREATE OR REPLACE BRANCH 'audit-branch'; + + ALTER TABLE iceberg.default.mytable CREATE BRANCH 'audit-branch-system' FOR SYSTEM_VERSION AS OF 4176642711908913940; + + ALTER TABLE iceberg.default.mytable CREATE BRANCH IF NOT EXISTS 'audit-branch-system' FOR SYSTEM_VERSION AS OF 4176642711908913940; + + ALTER TABLE iceberg.default.mytable CREATE BRANCH 'audit-branch-retain' FOR SYSTEM_VERSION AS OF 4176642711908913940 RETAIN 7 DAYS; + + ALTER TABLE iceberg.default.mytable CREATE BRANCH 'audit-branch-snap-retain' FOR SYSTEM_VERSION AS OF 4176642711908913940 RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 2 DAYS; + + ALTER TABLE iceberg.default.mytable CREATE OR REPLACE BRANCH 'audit-branch-time' FOR SYSTEM_TIME AS OF TIMESTAMP '2026-01-02 17:30:35.247 Asia/Kolkata'; + + ALTER TABLE iceberg.default.mytable CREATE TAG 'audit-tag'; + + ALTER TABLE iceberg.default.mytable CREATE TAG IF NOT EXISTS 'audit-tag'; + + ALTER TABLE iceberg.default.mytable CREATE OR REPLACE TAG 'audit-tag'; + + ALTER TABLE iceberg.default.mytable CREATE TAG 'audit-tag-system' FOR SYSTEM_VERSION AS OF 4176642711908913940; + + ALTER TABLE iceberg.default.mytable CREATE TAG IF NOT EXISTS 'audit-tag-system' FOR SYSTEM_VERSION AS OF 4176642711908913940; + + ALTER TABLE iceberg.default.mytable CREATE TAG 'audit-tag-retain' FOR SYSTEM_VERSION AS OF 4176642711908913940 RETAIN 7 DAYS; + + ALTER TABLE iceberg.default.mytable CREATE TAG 'audit-tag-snap-retain' FOR SYSTEM_VERSION AS OF 4176642711908913940 RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 2 DAYS; + + ALTER TABLE iceberg.default.mytable CREATE OR REPLACE TAG 'audit-tag-time' FOR SYSTEM_TIME AS OF TIMESTAMP '2026-01-02 17:30:35.247 Asia/Kolkata'; + +**Presto C++ Support** + +Creating and dropping tags and branches with ``ALTER TABLE`` statements is fully supported in Presto C++. + To add a new column as a partition column, identify the transform functions for the column. The table is partitioned by the transformed value of the column:: @@ -1874,6 +1926,73 @@ Iceberg tables do not support running multiple :doc:`../sql/merge` statements on Failed to commit Iceberg update to table: Found conflicting files that can contain records matching true +Transaction support +^^^^^^^^^^^^^^^^^^^ + +The Iceberg connector supports explicit multi-statement transactions with writes +to a single Iceberg table. To run transaction statements, use +:doc:`/sql/start-transaction` with :doc:`/sql/commit` or :doc:`/sql/rollback`. + +The Iceberg connector provides snapshot isolation at ``REPEATABLE READ`` level. +This also satisfies ``READ COMMITTED`` and ``READ UNCOMMITTED``, so these +isolation levels are supported as well. For snapshot semantics, use +``REPEATABLE READ``. + +Within a transaction, reads can access multiple tables, while write operations are +restricted to a single Iceberg table. All operations execute under snapshot isolation. +The transaction therefore behaves as a **multi-table read, single-table write** transaction:: + + START TRANSACTION ISOLATION LEVEL REPEATABLE READ; + INSERT INTO iceberg.default.test_table + SELECT id, status + FROM iceberg.source.source_table1 + WHERE status = 'pending'; + INSERT INTO iceberg.default.test_table + SELECT * FROM iceberg.source.source_table2; + INSERT INTO iceberg.default.test_table (id, status) VALUES (1, 'pending'); + UPDATE iceberg.default.test_table + SET status = 'committed' + WHERE id < 100 and status = 'pending'; + COMMIT; + +Statements executed within the same transaction follow **read-your-writes** +semantics. This behavior is important for standard SQL interactive transactions. +Data modifications performed earlier in the transaction are visible to subsequent +statements before the transaction is committed:: + + START TRANSACTION; + INSERT INTO iceberg.default.test_table (id, status) VALUES (1, 'pending'), (2, 'pending'); + UPDATE iceberg.default.test_table SET status = 'committed' WHERE id = 1; + SELECT * FROM iceberg.default.test_table; -- (1, 'committed'), (2, 'pending') + + DELETE FROM iceberg.default.test_table WHERE status = 'pending'; + SELECT * FROM iceberg.default.test_table; -- (1, 'committed') + COMMIT; + +Limitations: + +* Writes in the same transaction can target only one Iceberg table. Attempts + to write to another table fail with ``Not allowed to open write transactions on multiple tables``. +* ``SERIALIZABLE`` isolation is not supported by the Iceberg connector. +* The following statements are only supported in autocommit mode: + ``MERGE INTO``, ``CREATE/DROP/RENAME TABLE``, + ``CREATE/DROP/RENAME SCHEMA``, ``CREATE/DROP/RENAME VIEW``, + ``CREATE/DROP/REFRESH MATERIALIZED VIEW``, ``TRUNCATE TABLE``, and + ``ANALYZE``. +* ``CALL`` statements are only supported in autocommit mode. +* If concurrent transactions change table metadata, commit may fail and require + retrying the transaction (for example, ``Table metadata refresh is required``). + +.. _iceberg_analyze: + +Collecting table and column statistics +-------------------------------------- + +The Iceberg connector supports collection of table and column statistics +with the :doc:`/sql/analyze` statement:: + + ANALYZE iceberg.tpch.orders; + Schema Evolution ---------------- @@ -2176,7 +2295,7 @@ Querying branches and tags Iceberg supports branches and tags which are named references to snapshots. -Query Iceberg table by specifying the branch name: +Query Iceberg table by specifying the branch name using ``FOR SYSTEM_VERSION AS OF``: .. code-block:: sql @@ -2191,6 +2310,21 @@ Query Iceberg table by specifying the branch name: 30 | mexico | 3 | comment (3 rows) +Alternatively, you can query a branch using the dot notation syntax with quoted identifiers: + +.. code-block:: sql + + SELECT * FROM "nation.branch_testBranch"; + +.. code-block:: text + + nationkey | name | regionkey | comment + -----------+---------------+-----------+--------- + 10 | united states | 1 | comment + 20 | canada | 2 | comment + 30 | mexico | 3 | comment + (3 rows) + Query Iceberg table by specifying the tag name: .. code-block:: sql @@ -2205,6 +2339,110 @@ Query Iceberg table by specifying the tag name: 20 | canada | 2 | comment (3 rows) +**Note:** The dot notation syntax ``"
.branch_"`` requires double quotes to prevent the SQL parser from interpreting the dot as a schema.table separator. This syntax works for both querying (SELECT) and mutating (INSERT, UPDATE, DELETE, MERGE) branch data. + +**Presto C++ Support** + +Querying tags and branches is fully supported in Presto C++. + +Mutating Iceberg Branches +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Iceberg supports performing INSERT, UPDATE, DELETE, and MERGE operations directly on branches, +allowing you to make changes to a branch without affecting the main table or other branches. + +To perform mutations on a branch, use the quoted identifier syntax ``"
.branch_"`` (for example, ``"orders.branch_audit_branch"``). +The quotes are required to prevent the SQL parser from interpreting the dot as a schema.table separator. + +**Insert into a branch:** + +.. code-block:: sql + + -- Create a branch first + ALTER TABLE orders CREATE BRANCH 'audit_branch'; + + -- Insert data into the branch + INSERT INTO "orders.branch_audit_branch" VALUES (1, 'Product A', 100.00); + INSERT INTO "orders.branch_audit_branch" VALUES (2, 'Product B', 200.00); + +**Update data in a branch:** + +.. code-block:: sql + + -- Update specific rows in the branch + UPDATE "orders.branch_audit_branch" SET price = 120.00 WHERE id = 1; + + -- Update with complex expressions + UPDATE "orders.branch_audit_branch" + SET price = price * 1.1 + WHERE category = 'electronics'; + +**Delete from a branch:** + +.. code-block:: sql + + -- Delete specific rows from the branch + DELETE FROM "orders.branch_audit_branch" WHERE id = 2; + + -- Delete with complex predicates + DELETE FROM "orders.branch_audit_branch" + WHERE created_date < DATE '2024-01-01'; + +**Merge into a branch:** + +.. code-block:: sql + + -- Merge data from source table into branch + MERGE INTO "orders.branch_audit_branch" t + USING source_table s + ON t.id = s.id + WHEN MATCHED THEN UPDATE SET price = s.price + WHEN NOT MATCHED THEN INSERT (id, product, price) VALUES (s.id, s.product, s.price); + +**Verify branch isolation:** + +After performing mutations on a branch, you can verify that the main table remains unchanged: + +.. code-block:: sql + + -- Query the branch to see changes + SELECT * FROM orders FOR SYSTEM_VERSION AS OF 'audit_branch'; + + -- Query the main table (unchanged) + SELECT * FROM orders; + +**Supported operations:** + +The following DML operations are supported with branch-specific table names: + +* ``INSERT`` - Add new rows to a branch +* ``UPDATE`` - Modify existing rows in a branch +* ``DELETE`` - Remove rows from a branch (including metadata delete optimization) +* ``MERGE`` - Conditionally insert, update, or delete rows in a branch +* ``TRUNCATE TABLE`` - Remove all rows from a branch +* ``SELECT`` - Query branch data using ``FOR SYSTEM_VERSION AS OF 'branch_name'`` + +**Unsupported operations:** + +The following operations are **not supported** with branch-specific table names and will result in an error: + +* ``ALTER TABLE`` DDL operations (``ADD COLUMN``, ``DROP COLUMN``, ``RENAME COLUMN``, ``SET PROPERTIES``) - Schema changes must be applied to the main table +* ``CREATE VIEW`` / ``CREATE MATERIALIZED VIEW`` - Views cannot be created from branch-specific tables + +**Important notes:** + +* Branch mutations require quoted identifiers (double quotes) around the table name with branch suffix +* The branch must exist before performing mutations (create it with ``ALTER TABLE ... CREATE BRANCH``) +* Changes are isolated to the specified branch and do not affect the main table or other branches +* All standard SQL features work with branch mutations such as WHERE clauses, column lists, INSERT from SELECT, and others +* For MERGE operations, the table must have format version 2 or higher and update mode set to ``merge-on-read`` + +**Presto C++ Support** + +Branch mutations are partially supported in Presto C++. + +* **Supported:** ``INSERT``, ``TRUNCATE TABLE`` + Presto C++ Support ^^^^^^^^^^^^^^^^^^ @@ -2427,27 +2665,36 @@ The storage table inherits standard Iceberg table properties for partitioning, s Freshness and Refresh ^^^^^^^^^^^^^^^^^^^^^ -Materialized views track the snapshot IDs of their base tables to determine staleness. When base tables are modified, the materialized view becomes stale and returns results by querying the base tables directly. After running ``REFRESH MATERIALIZED VIEW``, queries read from the pre-computed storage table. - -The refresh operation uses a full refresh strategy, replacing all data in the storage table with the current query results. +After running ``REFRESH MATERIALIZED VIEW``, queries read from the pre-computed storage table. The refresh operation uses a full refresh strategy, replacing all data in the storage table with the current query results and recording the new snapshot IDs for all base tables. .. _iceberg-stale-data-handling: Stale Data Handling ^^^^^^^^^^^^^^^^^^^ +The Iceberg connector automatically detects staleness by comparing current base table +snapshots against the snapshots recorded at the last refresh. A materialized view is +considered stale if base tables have changed AND the time since the last base table +modification exceeds the configured staleness window. + By default, when no staleness properties are configured, queries against a stale materialized view will fall back to executing the underlying view query against the base tables. You can change this default using the ``materialized_view_stale_read_behavior`` session property. To configure staleness handling per view, set both of these properties together: -- ``stale_read_behavior``: What to do when reading stale data (``FAIL`` or ``USE_VIEW_QUERY``) +- ``stale_read_behavior``: What to do when reading stale data (``FAIL``, ``USE_VIEW_QUERY``, or ``USE_STITCHING``) - ``staleness_window``: How much staleness to tolerate (e.g., ``1h``, ``30m``, ``0s``) -The Iceberg connector automatically detects staleness based on base table modifications. -A materialized view is considered stale if base tables have changed AND the time since -the last base table modification exceeds the staleness window. +When ``USE_STITCHING`` is configured, the Iceberg connector tracks staleness at the +partition level, enabling predicate stitching to recompute only affected partitions +rather than the entire view. See :doc:`/admin/materialized-views` for details on how +predicate stitching works. + +.. note:: + Partition-level staleness detection only works for append-only changes (INSERT). + DELETE or UPDATE operations on base tables cause the entire view to be treated + as stale, requiring full recomputation. Example with staleness handling: @@ -2464,8 +2711,8 @@ Example with staleness handling: Limitations ^^^^^^^^^^^ -- All refreshes recompute the entire result set -- REFRESH does not provide snapshot isolation across multiple base tables +- All refreshes recompute the entire result set (incremental refresh not supported) +- REFRESH does not provide snapshot isolation across multiple base tables (each base table's current snapshot is used independently) - Querying materialized views at specific snapshots or timestamps is not supported Example diff --git a/presto-docs/src/main/sphinx/connector/lance.rst b/presto-docs/src/main/sphinx/connector/lance.rst new file mode 100644 index 0000000000000..266a26a7a3997 --- /dev/null +++ b/presto-docs/src/main/sphinx/connector/lance.rst @@ -0,0 +1,236 @@ +=============== +Lance Connector +=============== + +Overview +-------- + +The Lance connector allows querying and writing data stored in +`Lance `_ format from Presto. Lance is a modern columnar +data format optimized for machine learning workloads and fast random access. + +The connector uses the Lance Java SDK to read and write Lance datasets. +Each Lance dataset is organized into **fragments**, and the connector maps each fragment to a +Presto split for parallel processing across workers. + +Configuration +------------- + +To configure the Lance connector, create a catalog properties file +``etc/catalog/lance.properties`` with the following contents, +replacing the properties as appropriate: + +.. code-block:: none + + connector.name=lance + lance.root-url=/path/to/lance/data + +Configuration Properties +------------------------ + +The following configuration properties are available: + +=============================== ============================================================= =============== +Property Name Description Default +=============================== ============================================================= =============== +``lance.impl`` Namespace implementation: ``dir`` ``dir`` +``lance.root-url`` Root storage path for Lance datasets. ``""`` +``lance.single-level-ns`` When ``true``, uses a single-level namespace with a ``true`` + virtual ``default`` schema. +``lance.read-batch-size`` Number of rows per Arrow batch during reads. ``8192`` +``lance.max-rows-per-file`` Maximum number of rows per Lance data file. ``1000000`` +``lance.max-rows-per-group`` Maximum number of rows per row group. ``100000`` +``lance.write-batch-size`` Number of rows to batch before writing to Arrow. ``10000`` +=============================== ============================================================= =============== + +``lance.impl`` +^^^^^^^^^^^^^^ + +Namespace implementation to use. The default ``dir`` uses a directory-based +table store where each table is a ``.lance`` directory under the root. + +``lance.root-url`` +^^^^^^^^^^^^^^^^^^ + +Root storage path for Lance datasets. All tables are stored as subdirectories +named ``.lance`` under this path. For example, if ``lance.root-url`` +is set to ``/data/lance``, a table named ``my_table`` is stored at +``/data/lance/my_table.lance``. + +``lance.single-level-ns`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +When set to ``true`` (the default), the connector exposes a single ``default`` +schema that maps directly to the root directory. All tables are accessed as +``lance.default.``. + +``lance.read-batch-size`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Controls the number of rows read per Arrow batch from Lance. Larger values may +improve read throughput at the cost of higher memory usage. The default is +``8192``. + +``lance.max-rows-per-file`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maximum number of rows per Lance data file. The default is ``1000000``. + +.. note:: + + This property is reserved for future use and is not yet wired into the + write path. + +``lance.max-rows-per-group`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maximum number of rows per row group within a Lance data file. The default is +``100000``. + +.. note:: + + This property is reserved for future use and is not yet wired into the + write path. + +``lance.write-batch-size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Number of rows to batch before converting to Arrow format during writes. The +default is ``10000``. + +.. note:: + + This property is reserved for future use and is not yet wired into the + write path. + +Data Types +---------- + +The following table lists the supported data type mappings between Lance +(Arrow) types and Presto types: + +================= =============== ====================================== +Lance (Arrow) Presto Notes +================= =============== ====================================== +``Bool`` ``BOOLEAN`` +``Int(8)`` ``TINYINT`` +``Int(16)`` ``SMALLINT`` +``Int(32)`` ``INTEGER`` +``Int(64)`` ``BIGINT`` +``Float(SINGLE)`` ``REAL`` +``Float(DOUBLE)`` ``DOUBLE`` +``Utf8`` ``VARCHAR`` +``LargeUtf8`` ``VARCHAR`` +``Binary`` ``VARBINARY`` +``LargeBinary`` ``VARBINARY`` +``Date(DAY)`` ``DATE`` +``Timestamp`` ``TIMESTAMP`` Microsecond precision; reads support + both with and without timezone +``List`` ``ARRAY`` Read only; element type mapped + recursively +``FixedSizeList`` ``ARRAY`` Read only; element type mapped + recursively +================= =============== ====================================== + +.. note:: + + Arrow types not listed above are unsupported and will cause an error. + +SQL Support +----------- + +The Lance connector supports the following SQL operations. + +CREATE TABLE +^^^^^^^^^^^^ + +Create a new Lance table: + +.. code-block:: sql + + CREATE TABLE lance.default.my_table ( + id BIGINT, + name VARCHAR, + score DOUBLE + ); + +CREATE TABLE AS +^^^^^^^^^^^^^^^ + +Create a Lance table from a query: + +.. code-block:: sql + + CREATE TABLE lance.default.my_table AS + SELECT * FROM tpch.tiny.nation; + +INSERT INTO +^^^^^^^^^^^ + +Append data to an existing Lance table: + +.. code-block:: sql + + INSERT INTO lance.default.my_table + SELECT * FROM tpch.tiny.nation; + +SELECT +^^^^^^ + +Query data from a Lance table: + +.. code-block:: sql + + SELECT * FROM lance.default.my_table; + +Column projection is pushed down to Lance, so queries that select a subset +of columns only read those columns from disk: + +.. code-block:: sql + + SELECT id, name FROM lance.default.my_table; + +DROP TABLE +^^^^^^^^^^ + +Drop a Lance table and delete all its data: + +.. code-block:: sql + + DROP TABLE lance.default.my_table; + +SHOW TABLES +^^^^^^^^^^^ + +List all tables in the catalog: + +.. code-block:: sql + + SHOW TABLES FROM lance.default; + +DESCRIBE +^^^^^^^^ + +Show the columns and types of a Lance table: + +.. code-block:: sql + + DESCRIBE lance.default.my_table; + +Limitations +----------- + +* Only a single schema (``default``) is supported when ``lance.single-level-ns`` + is ``true``. +* The following SQL statements are not supported: + + * :doc:`/sql/alter-table` + * :doc:`/sql/delete` + * :doc:`/sql/update` + +* Predicate pushdown is not supported. Only column projection is pushed down + to the Lance reader. +* ``ARRAY`` types are supported for reads but cannot be written. +* Only local filesystem paths are supported in the current ``dir`` implementation. +* Data written by one Presto cluster is not visible to another cluster until the + write transaction commits. diff --git a/presto-docs/src/main/sphinx/develop.rst b/presto-docs/src/main/sphinx/develop.rst index 8f58863395ad8..2ef0136f8a6ee 100644 --- a/presto-docs/src/main/sphinx/develop.rst +++ b/presto-docs/src/main/sphinx/develop.rst @@ -17,6 +17,7 @@ This guide is intended for Presto contributors and plugin developers. develop/system-access-control develop/password-authenticator develop/event-listener + develop/openlineage-event-listener develop/client-protocol develop/worker-protocol develop/serialized-page diff --git a/presto-docs/src/main/sphinx/develop/openlineage-event-listener.rst b/presto-docs/src/main/sphinx/develop/openlineage-event-listener.rst new file mode 100644 index 0000000000000..45b11a2b7b598 --- /dev/null +++ b/presto-docs/src/main/sphinx/develop/openlineage-event-listener.rst @@ -0,0 +1,163 @@ +========================== +OpenLineage Event Listener +========================== + +The OpenLineage event listener plugin emits query events in the +`OpenLineage `_ format, enabling integration with +lineage tracking systems such as `Marquez `_, +`Atlan `_, and `DataHub `_. + +The plugin captures: + +* Query start events (``START``) +* Query completion events (``COMPLETE`` or ``FAIL``) +* Input and output dataset information including column-level lineage + +Installation +------------ + +The OpenLineage event listener plugin is bundled with Presto and requires +no additional installation. + +Configuration +------------- + +Create an ``etc/event-listener.properties`` file on the coordinator with the +following required properties: + +.. code-block:: none + + event-listener.name=openlineage-event-listener + openlineage-event-listener.presto.uri=http://presto-coordinator:8080 + openlineage-event-listener.transport.type=CONSOLE + +Transport Types +^^^^^^^^^^^^^^^ + +The plugin supports two transport types for emitting OpenLineage events: + +**Console Transport** + +Writes OpenLineage events as JSON to stdout. Useful for debugging and +development. + +.. code-block:: none + + event-listener.name=openlineage-event-listener + openlineage-event-listener.presto.uri=http://presto-coordinator:8080 + openlineage-event-listener.transport.type=CONSOLE + +**HTTP Transport** + +Sends OpenLineage events to an HTTP endpoint such as the Marquez API. + +.. code-block:: none + + event-listener.name=openlineage-event-listener + openlineage-event-listener.presto.uri=http://presto-coordinator:8080 + openlineage-event-listener.transport.type=HTTP + openlineage-event-listener.transport.url=http://marquez:5000 + openlineage-event-listener.transport.endpoint=/api/v1/lineage + +Configuration Properties +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 40 10 10 40 + :header-rows: 1 + + * - Property + - Required + - Default + - Description + * - ``openlineage-event-listener.presto.uri`` + - Yes + - + - URI of the Presto server. Used for namespace rendering in OpenLineage events. + * - ``openlineage-event-listener.transport.type`` + - No + - ``CONSOLE`` + - Transport type for emitting events. Supported values: ``CONSOLE``, ``HTTP``. + * - ``openlineage-event-listener.namespace`` + - No + - + - Override the default namespace for OpenLineage jobs. Defaults to the Presto URI with ``presto://`` scheme. + * - ``openlineage-event-listener.job.name-format`` + - No + - ``$QUERY_ID`` + - Format string for the OpenLineage job name. Supported placeholders: ``$QUERY_ID``, ``$USER``, ``$SOURCE``, ``$CLIENT_IP``. + * - ``openlineage-event-listener.presto.include-query-types`` + - No + - ``DELETE,INSERT,MERGE,UPDATE,DATA_DEFINITION`` + - Comma-separated list of query types that generate OpenLineage events. Other query types are filtered out on completion. + * - ``openlineage-event-listener.disabled-facets`` + - No + - + - Comma-separated list of facets to exclude from events. Supported values: ``PRESTO_METADATA``, ``PRESTO_QUERY_STATISTICS``, ``PRESTO_QUERY_CONTEXT``. + +HTTP Transport Properties +^^^^^^^^^^^^^^^^^^^^^^^^^ + +These properties apply when ``openlineage-event-listener.transport.type`` is set to ``HTTP``. + +.. list-table:: + :widths: 40 10 10 40 + :header-rows: 1 + + * - Property + - Required + - Default + - Description + * - ``openlineage-event-listener.transport.url`` + - Yes + - + - URL of the OpenLineage API server. + * - ``openlineage-event-listener.transport.endpoint`` + - No + - + - Custom API path for receiving events. + * - ``openlineage-event-listener.transport.api-key`` + - No + - + - API key for authentication. Sent as a ``Bearer`` token. + * - ``openlineage-event-listener.transport.timeout`` + - No + - ``5s`` + - HTTP request timeout. Accepts duration strings. For example: ``5s``, ``30s``, ``1m``. + * - ``openlineage-event-listener.transport.headers`` + - No + - + - Custom HTTP headers as comma-separated ``key:value`` pairs. + * - ``openlineage-event-listener.transport.url-params`` + - No + - + - Custom URL query parameters as comma-separated ``key:value`` pairs. + * - ``openlineage-event-listener.transport.compression`` + - No + - ``NONE`` + - HTTP body compression. Supported values: ``NONE``, ``GZIP``. + +Event Details +------------- + +The plugin emits the following OpenLineage facets: + +**Run Facets** + +* ``processing_engine`` - Presto server version information +* ``presto_metadata`` - Query ID, transaction ID, and query plan +* ``presto_query_context`` - User, server address, environment, source, client info +* ``presto_query_statistics`` - Detailed query execution statistics (on completion only) +* ``nominalTime`` - Query start and end times (on completion only) +* ``errorMessage`` - Failure message (on failure only) + +**Job Facets** + +* ``jobType`` - ``BATCH`` / ``PRESTO`` / ``QUERY`` +* ``sql`` - The SQL query text with dialect ``presto`` + +**Dataset Facets** + +* ``schema`` - Column names and types for input and output datasets +* ``dataSource`` - Catalog and schema information +* ``columnLineage`` - Column-level lineage mapping from input to output columns diff --git a/presto-docs/src/main/sphinx/ecosystem/list.rst b/presto-docs/src/main/sphinx/ecosystem/list.rst index 48d0fede5decf..34c72ec9721c8 100644 --- a/presto-docs/src/main/sphinx/ecosystem/list.rst +++ b/presto-docs/src/main/sphinx/ecosystem/list.rst @@ -2,11 +2,6 @@ Ecosystem ========= -.. contents:: - :local: - :backlinks: none - :depth: 1 - Overview -------- diff --git a/presto-docs/src/main/sphinx/functions.rst b/presto-docs/src/main/sphinx/functions.rst index 8ba846f7d7285..9717a1455d6f1 100644 --- a/presto-docs/src/main/sphinx/functions.rst +++ b/presto-docs/src/main/sphinx/functions.rst @@ -38,3 +38,4 @@ Functions and Operators functions/setdigest functions/sketch functions/pinot + functions/plugin-loaded-functions diff --git a/presto-docs/src/main/sphinx/functions/array.rst b/presto-docs/src/main/sphinx/functions/array.rst index 0f7a2fb2d5c2b..339cc9a8bdfc5 100644 --- a/presto-docs/src/main/sphinx/functions/array.rst +++ b/presto-docs/src/main/sphinx/functions/array.rst @@ -21,6 +21,8 @@ The ``||`` operator is used to concatenate an array with an array or an element Array Functions --------------- +For plugin-loaded array functions, see :ref:`functions/plugin-loaded-functions:array functions`. + .. function:: all_match(array(T), function(T,boolean)) -> boolean Returns whether all elements of an array match the given predicate. Returns ``true`` if all the elements @@ -35,11 +37,6 @@ Array Functions array is empty); ``NULL`` if the predicate function returns ``NULL`` for one or more elements and ``false`` for all other elements. -.. function:: array_average(array(double)) -> double - - Returns the average of all non-null elements of the ``array``. If there is no non-null elements, returns - ``null``. - .. function:: array_cum_sum(array(T)) -> array(T) Returns the array whose elements are the cumulative sum of the input array, i.e. result[i] = input[1]+input[2]+...+input[i]. @@ -55,14 +52,6 @@ Array Functions SELECT array_distinct(ARRAY [1, 2, null, null, 2]) -- ARRAY[1, 2, null] SELECT array_distinct(ARRAY [ROW(1, null), ROW (1, null)] -- ARRAY[ROW(1, null) -.. function:: array_duplicates(array(T)) -> array(bigint/varchar) - - Returns a set of elements that occur more than once in ``array``. - Throws an exception if any of the elements are rows or arrays that contain nulls. :: - - SELECT array_duplicates(ARRAY[1, 2, null, 1, null, 3]) -- ARRAY[1, null] - SELECT array_duplicates(ARRAY[ROW(1, null), ROW(1, null)]) -- "map key cannot be null or contain nulls" - .. function:: array_except(x, y) -> array Returns an array of elements in ``x`` but not in ``y``, without duplicates. @@ -70,19 +59,6 @@ Array Functions SELECT array_except(ARRAY[1, 3, 3, 2, null], ARRAY[1,2, 2, 4]) -- ARRAY[3, null] -.. function:: array_frequency(array(E)) -> map(E, int) - - Returns a map: keys are the unique elements in the ``array``, values are how many times the key appears. - Ignores null elements. Empty array returns empty map. - -.. function:: array_has_duplicates(array(T)) -> boolean - - Returns a boolean: whether ``array`` has any elements that occur more than once. - Throws an exception if any of the elements are rows or arrays that contain nulls. :: - - SELECT array_has_duplicates(ARRAY[1, 2, null, 1, null, 3]) -- true - SELECT array_has_duplicates(ARRAY[ROW(1, null), ROW(1, null)]) -- "map key cannot be null or contain nulls" - .. function:: array_intersect(x, y) -> array Returns an array of the elements in the intersection of ``x`` and ``y``, without duplicates. @@ -90,36 +66,10 @@ Array Functions SELECT array_intersect(ARRAY[1, 2, 3, 2, null], ARRAY[1,2, 2, 4, null]) -- ARRAY[1, 2, null] -.. function:: array_intersect(array(array(E))) -> array(E) - - Returns an array of the elements in the intersection of all arrays in the given array, without duplicates. - This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: - - SELECT array_intersect(ARRAY[ARRAY[1, 2, 3, 2, null], ARRAY[1,2,2, 4, null], ARRAY [1, 2, 3, 4 null]]) -- ARRAY[1, 2, null] - .. function:: array_join(x, delimiter, null_replacement) -> varchar Concatenates the elements of the given array using the delimiter and an optional string to replace nulls. -.. function:: array_least_frequent(array(T)) -> array(T) - - Returns the least frequent non-null element of an array. If there are multiple elements with the same frequency, the function returns the smallest element. - If the array has more than one element and any elements are ``ROWS`` with null fields or ``ARRAYS`` with null elements, an exception is returned. :: - - SELECT array_least_frequent(ARRAY[1, 0 , 5]) -- ARRAY[0] - select array_least_frequent(ARRAY[1, null, 1]) -- ARRAY[1] - select array_least_frequent(ARRAY[ROW(1,null), ROW(1, null)]) -- "map key cannot be null or contain nulls" - -.. function:: array_least_frequent(array(T), n) -> array(T) - - Returns ``n`` least frequent non-null elements of an array. The elements are ordered in increasing order of their frequencies. - If two elements have the same frequency, smaller elements will appear first. - If the array has more than one element and any elements are ``ROWS`` with null fields or ``ARRAYS`` with null elements, an exception is returned. :: - - SELECT array_least_frequent(ARRAY[3, 2, 2, 6, 6, 1, 1], 3) -- ARRAY[3, 1, 2] - select array_least_frequent(ARRAY[1, null, 1], 2) -- ARRAY[1] - select array_least_frequent(ARRAY[ROW(1,null), ROW(1, null)], 2) -- "map key cannot be null or contain nulls" - .. function:: array_max(x) -> x Returns the maximum value of input array. @@ -128,20 +78,6 @@ Array Functions Returns the minimum value of input array. -.. function:: array_max_by(array(T), function(T, U)) -> T - - Applies the provided function to each element, and returns the element that gives the maximum value. - ``U`` can be any orderable type. :: - - SELECT array_max_by(ARRAY ['a', 'bbb', 'cc'], x -> LENGTH(x)) -- 'bbb' - -.. function:: array_min_by(array(T), function(T, U)) -> T - - Applies the provided function to each element, and returns the element that gives the minimum value. - ``U`` can be any orderable type. :: - - SELECT array_min_by(ARRAY ['a', 'bbb', 'cc'], x -> LENGTH(x)) -- 'a' - .. function:: array_normalize(x, p) -> array Normalizes array ``x`` by dividing each element by the p-norm of the array. @@ -210,15 +146,6 @@ Array Functions SELECT array_sort(ARRAY[CAST(0.0 AS DOUBLE), CAST('NaN' AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)], x -> x); -- [-Infinity, 0.0, Infinity, NaN] SELECT array_sort(ARRAY[ROW('a', 3), ROW('b', 1), ROW('c', 2)], x -> x[2]); -- [ROW('b', 1), ROW('c', 2), ROW('a', 3)] -.. function:: array_sort_desc(x) -> array - - Returns the ``array`` sorted in the descending order. Elements of the ``array`` must be orderable. - Null elements are placed at the end of the returned array. :: - - SELECT array_sort_desc(ARRAY [100, 1, 10, 50]); -- [100, 50, 10, 1] - SELECT array_sort_desc(ARRAY [null, 100, null, 1, 10, 50]); -- [100, 50, 10, 1, null, null] - SELECT array_sort_desc(ARRAY [ARRAY ["a", null], null, ARRAY ["a"]); -- [["a", null], ["a"], null] - .. function:: array_sort_desc(array(T), function(T,U)) -> array(T) Sorts and returns the ``array`` in descending order using a lambda function to extract sorting keys. @@ -231,16 +158,6 @@ Array Functions SELECT array_sort_desc(ARRAY[CAST(0.0 AS DOUBLE), CAST('NaN' AS DOUBLE), CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)], x -> x); -- [NaN, Infinity, 0.0, -Infinity] SELECT array_sort_desc(ARRAY[ROW('a', 3), ROW('b', 1), ROW('c', 2)], x -> x[2]); -- [ROW('a', 3), ROW('c', 2), ROW('b', 1)] -.. function:: array_split_into_chunks(array(T), int) -> array(array(T)) - - Returns an ``array`` of arrays splitting the input ``array`` into chunks of given length. - The last chunk will be shorter than the chunk length if the array's length is not an integer multiple of - the chunk length. Ignores null inputs, but not elements. - - SELECT array_split_into_chunks(ARRAY [1, 2, 3, 4], 3); -- [[1, 2, 3], [4]] - SELECT array_split_into_chunks(null, null); -- null - SELECT array_split_into_chunks(array[1, 2, 3, cast(null as int)], 2]); -- [[1, 2], [3, null]] - .. function:: array_sum(array(T)) -> bigint/double Returns the sum of all non-null elements of the ``array``. If there is no non-null elements, returns ``0``. @@ -249,26 +166,6 @@ Array Functions ``T`` must be coercible to ``double``. Returns ``bigint`` if T is coercible to ``bigint``. Otherwise, returns ``double``. -.. function:: array_top_n(array(T), int) -> array(T) - - Returns an array of the top ``n`` elements from a given ``array``, sorted according to its natural descending order. - If ``n`` is larger than the size of the given ``array``, the returned list will be the same size as the input instead of ``n``. :: - - SELECT array_top_n(ARRAY [1, 100, 2, 5, 3], 3); -- [100, 5, 3] - SELECT array_top_n(ARRAY [1, 100], 5); -- [100, 1] - SELECT array_top_n(ARRAY ['a', 'zzz', 'zz', 'b', 'g', 'f'], 3); -- ['zzz', 'zz', 'g'] - -.. function:: array_transpose(array(array(T))) -> array(array(T)) - - Returns a transpose of a 2D array (matrix), where rows become columns and columns become rows. - Converts ``a[x][y]`` to ``transpose(a)[y][x]``. All rows in the input array must have the same length, otherwise the function will fail with an error. - Returns an empty array if the input is empty or if all rows are empty. :: - - SELECT array_transpose(ARRAY [ARRAY [1, 2, 3], ARRAY [4, 5, 6]]) -- [[1, 4], [2, 5], [3, 6]] - SELECT array_transpose(ARRAY [ARRAY ['a', 'b'], ARRAY ['c', 'd'], ARRAY ['e', 'f']]) -- [['a', 'c', 'e'], ['b', 'd', 'f']] - SELECT array_transpose(ARRAY [ARRAY [1]]) -- [[1]] - SELECT array_transpose(ARRAY []) -- [] - .. function:: arrays_overlap(x, y) -> boolean Tests if arrays ``x`` and ``y`` have any non-null elements in common. @@ -401,10 +298,6 @@ Array Functions (s, x) -> CAST(ROW(x + s.sum, s.count + 1) AS ROW(sum DOUBLE, count INTEGER)), s -> IF(s.count = 0, NULL, s.sum / s.count)); -.. function:: remove_nulls(array(T)) -> array - - Remove all null elements in the array. - .. function:: repeat(element, count) -> array Repeat ``element`` for ``count`` times. diff --git a/presto-docs/src/main/sphinx/functions/map.rst b/presto-docs/src/main/sphinx/functions/map.rst index 8205869e504db..dcb7ee6e30519 100644 --- a/presto-docs/src/main/sphinx/functions/map.rst +++ b/presto-docs/src/main/sphinx/functions/map.rst @@ -12,23 +12,7 @@ The ``[]`` operator is used to retrieve the value corresponding to a given key f Map Functions ------------- -.. function:: all_keys_match(x(K,V), function(K, boolean)) -> boolean - - Returns whether all keys of a map match the given predicate. Returns true if all the keys match the predicate (a special case is when the map is empty); false if one or more keys don’t match; NULL if the predicate function returns NULL for one or more keys and true for all other keys. :: - - SELECT all_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> length(x) = 1); -- true - -.. function:: any_keys_match(x(K,V), function(K, boolean)) -> boolean - - Returns whether any keys of a map match the given predicate. Returns true if one or more keys match the predicate; false if none of the keys match (a special case is when the map is empty); NULL if the predicate function returns NULL for one or more keys and false for all other keys. :: - - SELECT any_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'a'); -- true - -.. function:: any_values_match(x(K,V), function(V, boolean)) -> boolean - - Returns whether any values of a map matches the given predicate. Returns true if one or more values match the predicate; false if none of the values match (a special case is when the map is empty); NULL if the predicate function returns NULL for one or more values and false for all other values. :: - - SELECT ANY_VALUES_MATCH(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), x -> x = 1); -- true +For plugin-loaded map functions, see :ref:`functions/plugin-loaded-functions:map functions`. .. function:: cardinality(x) -> bigint :noindex: @@ -85,10 +69,6 @@ Map Functions SELECT map_filter(MAP(ARRAY[10, 20, 30], ARRAY['a', NULL, 'c']), (k, v) -> v IS NOT NULL); -- {10 -> a, 30 -> c} SELECT map_filter(MAP(ARRAY['k1', 'k2', 'k3'], ARRAY[20, 3, 15]), (k, v) -> v > 10); -- {k1 -> 20, k3 -> 15} -.. function:: map_remove_null_values(x(K,V)) -> map(K, V) - - Removes all the entries where the value is null from the map ``x``. - .. function:: map_subset(map(K,V), array(k)) -> map(K,V) Constructs a map from those entries of ``map`` for which the key is in the array given:: @@ -99,73 +79,14 @@ Map Functions SELECT map_subset(MAP(ARRAY[1,2], ARRAY['a','b']), ARRAY[]); -- {} SELECT map_subset(MAP(ARRAY[], ARRAY[]), ARRAY[1,2]); -- {} -.. function:: map_key_exists(x(K, V), k) -> boolean - - Returns whether the given key exists in the map. Returns ``true`` if key is present in the input map, returns ``false`` if not present.:: - - SELECT map_key_exists(MAP(ARRAY['x','y'], ARRAY[100,200]), 'x'); -- TRUE - .. function:: map_keys(x(K,V)) -> array(K) Returns all the keys in the map ``x``. -.. function:: map_top_n_keys(x(K,V), n) -> array(K) - - Returns top ``n`` keys in the map ``x`` by sorting its keys in descending order. - ``n`` must be a non-negative integer. - - For bottom ``n`` keys, use the function with lambda operator to perform custom sorting :: - - SELECT map_top_n_keys(map(ARRAY['a', 'b', 'c'], ARRAY[3, 2, 1]), 2) --- ['c', 'b'] - -.. function:: map_top_n_keys(x(K,V), n, function(K,K,int)) -> array(K) - - Returns top ``n`` keys in the map ``x`` by sorting its keys using the given comparator ``function``. The comparator takes - two non-nullable arguments representing two keys of the ``map``. It returns -1, 0, or 1 - as the first key is less than, equal to, or greater than the second key. - If the comparator function returns other values (including ``NULL``), the query will fail and raise an error :: - - SELECT map_top_n_keys(map(ARRAY['a', 'b', 'c'], ARRAY[3, 2, 1]), 2, (x, y) -> IF(x < y, -1, IF(x = y, 0, 1))) --- ['c', 'b'] - -.. function:: map_keys_by_top_n_values(x(K,V), n) -> array(K) - - Returns top ``n`` keys in the map ``x`` by sorting its values in descending order. If two or more keys have equal values, the higher key takes precedence. - ``n`` must be a non-negative integer.:: - - SELECT map_keys_by_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[2, 1, 3]), 2) --- ['c', 'a'] - -.. function:: map_top_n(x(K,V), n) -> map(K, V) - - Truncates map items. Keeps only the top ``n`` elements by value. Keys are used to break ties with the max key being chosen. Both keys and values should be orderable. - ``n`` must be a non-negative integer. :: - - SELECT map_top_n(map(ARRAY['a', 'b', 'c'], ARRAY[2, 3, 1]), 2) --- {'b' -> 3, 'a' -> 2} - -.. function:: map_normalize(x(varchar,double)) -> map(varchar,double) - - Returns the map with the same keys but all non-null values are scaled proportionally so that the sum of values becomes 1. - Map entries with null values remain unchanged. - .. function:: map_values(x(K,V)) -> array(V) Returns all the values in the map ``x``. -.. function:: map_top_n_values(x(K,V), n) -> array(V) - - Returns top ``n`` values in the map ``x`` by sorting its values in descending order. - ``n`` must be a non-negative integer. :: - - SELECT map_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), 2) --- [3, 2] - -.. function:: map_top_n_values(x(K,V), n, function(V,V,int)) -> array(V) - - Returns top n values in the map ``x`` based on the given comparator ``function``. The comparator will take - two nullable arguments representing two values of the ``map``. It returns -1, 0, or 1 - as the first value is less than, equal to, or greater than the second value. - If the comparator function returns other values (including ``NULL``), the query will fail and raise an error :: - - SELECT map_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), 2, (x, y) -> IF(x < y, -1, IF(x = y, 0, 1))) --- [3, 2] - .. function:: map_zip_with(map(K,V1), map(K,V2), function(K,V1,V2,V3)) -> map(K,V3) Merges the two given maps into a single map by applying ``function`` to the pair of values with the same key. @@ -181,18 +102,6 @@ Map Functions MAP(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), (k, v1, v2) -> k || CAST(v1/v2 AS VARCHAR)); -.. function:: no_keys_match(x(K,V), function(K, boolean)) -> boolean - - Returns whether no keys of a map match the given predicate. Returns true if none of the keys match the predicate (a special case is when the map is empty); false if one or more keys match; NULL if the predicate function returns NULL for one or more keys and false for all other keys. :: - - SELECT no_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'd'); -- true - -.. function:: no_values_match(x(K,V), function(V, boolean)) -> boolean - - Returns whether no values of a map match the given predicate. Returns true if none of the values match the predicate (a special case is when the map is empty); false if one or more values match; NULL if the predicate function returns NULL for one or more values and false for all other values. :: - - SELECT no_values_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'd'); -- true - .. function:: transform_keys(map(K1,V), function(K1,V,K2)) -> map(K2,V) Returns a map that applies ``function`` to each entry of ``map`` and transforms the keys:: @@ -214,14 +123,3 @@ Map Functions SELECT transform_values(MAP(ARRAY ['a', 'b'], ARRAY [1, 2]), (k, v) -> k || CAST(v as VARCHAR)); -- {a -> a1, b -> b2} SELECT transform_values(MAP(ARRAY [1, 2], ARRAY [1.0, 1.4]), -- {1 -> one_1.0, 2 -> two_1.4} (k, v) -> MAP(ARRAY[1, 2], ARRAY['one', 'two'])[k] || '_' || CAST(v AS VARCHAR)); - -.. function:: map_int_keys_to_array(map(int,V)) -> array(V) - Returns an ``array`` of values from the ``map`` with value at indexed by the original keys from ``map``:: - SELECT MAP_INT_KEYS_TO_ARRAY(MAP(ARRAY[3, 5, 6, 9], ARRAY['a', 'b', 'c', 'd'])) -> ARRAY[null, null, 'a', null, 'b', 'c', null, null, 'd'] - SELECT MAP_INT_KEYS_TO_ARRAY(MAP(ARRAY[3, 5, 6, 9], ARRAY['a', null, 'c', 'd'])) -> ARRAY[null, null, 'a', null, null, 'c', 'd'] - -.. function:: array_to_map_int_keys(array(v)) -> map(int, v) - Returns an ``map`` with indices of all non-null values from the ``array`` as keys and element at the specified index as the value:: - SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, 6, 9] AS ARRAY)) -> MAP(ARRAY[1, 2, 3,4], ARRAY[3, 5, 6, 9]) - SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, null, 6, 9] AS ARRAY)) -> MAP(ARRAY[1, 2, 4, 5], ARRAY[3, 5, 6, 9]) - SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, null, 6, 9, null, null, 1] AS ARRAY)) -> MAP(ARRAY[1, 2, 4, 5, 8], ARRAY[3, 5, 6, 9, 1]) \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/functions/plugin-loaded-functions.rst b/presto-docs/src/main/sphinx/functions/plugin-loaded-functions.rst new file mode 100644 index 0000000000000..0492c7041bc31 --- /dev/null +++ b/presto-docs/src/main/sphinx/functions/plugin-loaded-functions.rst @@ -0,0 +1,261 @@ +======================= +Plugin Loaded Functions +======================= + +These functions are optional, opt-in functions that can be loaded as needed. +For more details on loading these functions, refer to the +`presto-sql-helpers README. `_ + +Array Functions +--------------- + +.. function:: array_intersect(array(array(E))) -> array(E) + + Returns an array of the elements in the intersection of all arrays in the given array, without duplicates. + This function uses ``IS NOT DISTINCT FROM`` to determine which elements are the same. :: + + SELECT array_intersect(ARRAY[ARRAY[1, 2, 3, 2, null], ARRAY[1, 2, 2, 4, null], ARRAY [1, 2, 3, 4, null]]) -- ARRAY[1, 2, null] + +.. function:: array_average(array(double)) -> double + + Returns the average of all non-null elements of the ``array``. If there are no non-null elements, returns + ``null``. + +.. function:: array_split_into_chunks(array(T), int) -> array(array(T)) + + Returns an ``array`` of arrays splitting the input ``array`` into chunks of given length. + The last chunk will be shorter than the chunk length if the array's length is not an integer multiple of + the chunk length. Ignores null inputs, but not elements. :: + + SELECT array_split_into_chunks(ARRAY [1, 2, 3, 4], 3); -- [[1, 2, 3], [4]] + SELECT array_split_into_chunks(null, null); -- null + SELECT array_split_into_chunks(array[1, 2, 3, cast(null as int)], 2); -- [[1, 2], [3, null]] + +.. function:: array_frequency(array(E)) -> map(E, int) + + Returns a map: keys are the unique elements in the ``array``, values are how many times the key appears. + Ignores null elements. Empty array returns empty map. + +.. function:: array_duplicates(array(T)) -> array(bigint/varchar) + + Returns a set of elements that occur more than once in ``array``. + Throws an exception if any of the elements are rows or arrays that contain nulls. :: + + SELECT array_duplicates(ARRAY[1, 2, null, 1, null, 3]) -- ARRAY[1, null] + SELECT array_duplicates(ARRAY[ROW(1, null), ROW(1, null)]) -- "map key cannot be null or contain nulls" + +.. function:: array_has_duplicates(array(T)) -> boolean + + Returns a boolean: whether ``array`` has any elements that occur more than once. + Throws an exception if any of the elements are rows or arrays that contain nulls. :: + + SELECT array_has_duplicates(ARRAY[1, 2, null, 1, null, 3]) -- true + SELECT array_has_duplicates(ARRAY[ROW(1, null), ROW(1, null)]) -- "map key cannot be null or contain nulls" + +.. function:: array_least_frequent(array(T)) -> array(T) + + Returns the least frequent non-null element of an array. If there are multiple elements with the same frequency, the function returns the smallest element. + If the array has more than one element and any elements are ``ROWS`` with null fields or ``ARRAYS`` with null elements, an exception is returned. :: + + SELECT array_least_frequent(ARRAY[1, 0 , 5]) -- ARRAY[0] + select array_least_frequent(ARRAY[1, null, 1]) -- ARRAY[1] + select array_least_frequent(ARRAY[ROW(1,null), ROW(1, null)]) -- "map key cannot be null or contain nulls" + +.. function:: array_least_frequent(array(T), n) -> array(T) + + Returns ``n`` least frequent non-null elements of an array. The elements are ordered in increasing order of their frequencies. + If two elements have the same frequency, smaller elements will appear first. + If the array has more than one element and any elements are ``ROWS`` with null fields or ``ARRAYS`` with null elements, an exception is returned. :: + + SELECT array_least_frequent(ARRAY[3, 2, 2, 6, 6, 1, 1], 3) -- ARRAY[3, 1, 2] + select array_least_frequent(ARRAY[1, null, 1], 2) -- ARRAY[1] + select array_least_frequent(ARRAY[ROW(1,null), ROW(1, null)], 2) -- "map key cannot be null or contain nulls" + +.. function:: array_max_by(array(T), function(T, U)) -> T + + Applies the provided function to each element, and returns the element that gives the maximum value. + ``U`` can be any orderable type. :: + + SELECT array_max_by(ARRAY ['a', 'bbb', 'cc'], x -> LENGTH(x)) -- 'bbb' + +.. function:: array_min_by(array(T), function(T, U)) -> T + + Applies the provided function to each element, and returns the element that gives the minimum value. + ``U`` can be any orderable type. :: + + SELECT array_min_by(ARRAY ['a', 'bbb', 'cc'], x -> LENGTH(x)) -- 'a' + +.. function:: array_sort_desc(x) -> array + + Returns the ``array`` sorted in the descending order. Elements of the ``array`` must be orderable. + Null elements are placed at the end of the returned array. :: + + SELECT array_sort_desc(ARRAY [100, 1, 10, 50]); -- [100, 50, 10, 1] + SELECT array_sort_desc(ARRAY [null, 100, null, 1, 10, 50]); -- [100, 50, 10, 1, null, null] + SELECT array_sort_desc(ARRAY [ARRAY ["a", null], null, ARRAY ["a"]]); -- [["a", null], ["a"], null] + +.. function:: remove_nulls(array(T)) -> array + + Remove all null elements in the array. + +.. function:: array_top_n(array(T), int) -> array(T) + + Returns an array of the top ``n`` elements from a given ``array``, sorted according to its natural descending order. + If ``n`` is larger than the size of the given ``array``, the returned list will be the same size as the input instead of ``n``. :: + + SELECT array_top_n(ARRAY [1, 100, 2, 5, 3], 3); -- [100, 5, 3] + SELECT array_top_n(ARRAY [1, 100], 5); -- [100, 1] + SELECT array_top_n(ARRAY ['a', 'zzz', 'zz', 'b', 'g', 'f'], 3); -- ['zzz', 'zz', 'g'] + +.. function:: array_top_n(array(T), int, function(T,T,int)) -> array(T) + + Returns an array of the top ``n`` elements from a given ``array`` using the specified comparator ``function``. + The comparator will take two nullable arguments representing two nullable elements of the ``array``. It returns -1, 0, or 1 + as the first nullable element is less than, equal to, or greater than the second nullable element. + If the comparator function returns other values (including ``NULL``), the query will fail and raise an error. + If ``n`` is larger than the size of the given ``array``, the returned list will be the same size as the input instead of ``n``. :: + + SELECT array_top_n(ARRAY [100, 1, 3, -10, 6, -5], 3, (x, y) -> IF(abs(x) < abs(y), -1, IF(abs(x) = abs(y), 0, 1))); -- [100, -10, 6] + SELECT array_top_n(ARRAY [CAST(ROW(1, 2) AS ROW(x INT, y INT)), CAST(ROW(0, 11) AS ROW(x INT, y INT)), CAST(ROW(5, 10) AS ROW(x INT, y INT))], 2, (a, b) -> IF(a.x*a.y < b.x*b.y, -1, IF(a.x*a.y = b.x*b.y, 0, 1))); -- [ROW(5, 10), ROW(1, 2)] + +.. function:: array_transpose(array(array(T))) -> array(array(T)) + + Returns a transpose of a 2D array (matrix), where rows become columns and columns become rows. + Converts ``a[x][y]`` to ``transpose(a)[y][x]``. All rows in the input array must have the same length, otherwise the function will fail with an error. + Returns an empty array if the input is empty or if all rows are empty. :: + + SELECT array_transpose(ARRAY [ARRAY [1, 2, 3], ARRAY [4, 5, 6]]) -- [[1, 4], [2, 5], [3, 6]] + SELECT array_transpose(ARRAY [ARRAY ['a', 'b'], ARRAY ['c', 'd'], ARRAY ['e', 'f']]) -- [['a', 'c', 'e'], ['b', 'd', 'f']] + SELECT array_transpose(ARRAY [ARRAY [1]]) -- [[1]] + SELECT array_transpose(ARRAY []) -- [] + +Map Functions +-------------- + +.. function:: map_normalize(x(varchar,double)) -> map(varchar,double) + + Returns the map with the same keys but all non-null values are scaled proportionally so that the sum of values becomes 1. + Map entries with null values remain unchanged. + +.. function:: map_keys_by_top_n_values(x(K,V), n) -> array(K) + + Returns top ``n`` keys in the map ``x`` by sorting its values in descending order. If two or more keys have equal values, the higher key takes precedence. + ``n`` must be a non-negative integer.:: + + SELECT map_keys_by_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[2, 1, 3]), 2) --- ['c', 'a'] + +.. function:: map_key_exists(x(K, V), k) -> boolean + + Returns whether the given key exists in the map. Returns ``true`` if key is present in the input map, returns ``false`` if not present.:: + + SELECT map_key_exists(MAP(ARRAY['x','y'], ARRAY[100,200]), 'x'); -- TRUE + +.. function:: map_top_n(x(K,V), n) -> map(K, V) + + Truncates map items. Keeps only the top ``n`` elements by value. Keys are used to break ties with the max key being chosen. Both keys and values should be orderable. + ``n`` must be a non-negative integer. :: + + SELECT map_top_n(map(ARRAY['a', 'b', 'c'], ARRAY[2, 3, 1]), 2) --- {'b' -> 3, 'a' -> 2} + +.. function:: map_top_n_keys(x(K,V), n) -> array(K) + + Returns top ``n`` keys in the map ``x`` by sorting its keys in descending order. + ``n`` must be a non-negative integer. + + For bottom ``n`` keys, use the function with lambda operator to perform custom sorting. :: + + SELECT map_top_n_keys(map(ARRAY['a', 'b', 'c'], ARRAY[3, 2, 1]), 2) --- ['c', 'b'] + +.. function:: map_top_n_keys(x(K,V), n, function(K,K,int)) -> array(K) + + Returns top ``n`` keys in the map ``x`` by sorting its keys using the given comparator ``function``. The comparator takes + two non-nullable arguments representing two keys of the ``map``. It returns -1, 0, or 1 + as the first key is less than, equal to, or greater than the second key. + If the comparator function returns other values (including ``NULL``), the query will fail and raise an error. :: + + SELECT map_top_n_keys(map(ARRAY['a', 'b', 'c'], ARRAY[3, 2, 1]), 2, (x, y) -> IF(x < y, -1, IF(x = y, 0, 1))) --- ['c', 'b'] + +.. function:: map_top_n_values(x(K,V), n) -> array(V) + + Returns top ``n`` values in the map ``x`` by sorting its values in descending order. + ``n`` must be a non-negative integer. :: + + SELECT map_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), 2) --- [3, 2] + +.. function:: map_top_n_values(x(K,V), n, function(V,V,int)) -> array(V) + + Returns top n values in the map ``x`` based on the given comparator ``function``. The comparator will take + two nullable arguments representing two values of the ``map``. It returns -1, 0, or 1 + as the first value is less than, equal to, or greater than the second value. + If the comparator function returns other values (including ``NULL``), the query will fail and raise an error. :: + + SELECT map_top_n_values(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), 2, (x, y) -> IF(x < y, -1, IF(x = y, 0, 1))) --- [3, 2] + +.. function:: map_remove_null_values(x(K,V)) -> map(K, V) + + Removes all the entries where the value is null from the map ``x``. + +.. function:: all_keys_match(x(K,V), function(K, boolean)) -> boolean + + Returns whether all keys of a map match the given predicate. Returns true if all the keys match the predicate (a special case is when the map is empty); false if one or more keys don’t match; NULL if the predicate function returns NULL for one or more keys and true for all other keys. :: + + SELECT all_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> length(x) = 1); -- true + +.. function:: any_keys_match(x(K,V), function(K, boolean)) -> boolean + + Returns whether any keys of a map match the given predicate. Returns true if one or more keys match the predicate; false if none of the keys match (a special case is when the map is empty); NULL if the predicate function returns NULL for one or more keys and false for all other keys. :: + + SELECT any_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'a'); -- true + +.. function:: any_values_match(x(K,V), function(V, boolean)) -> boolean + + Returns whether any values of a map matches the given predicate. Returns true if one or more values match the predicate; false if none of the values match (a special case is when the map is empty); NULL if the predicate function returns NULL for one or more values and false for all other values. :: + + SELECT ANY_VALUES_MATCH(map(ARRAY['a', 'b', 'c'], ARRAY[1, 2, 3]), x -> x = 1); -- true + +.. function:: no_keys_match(x(K,V), function(K, boolean)) -> boolean + + Returns whether no keys of a map match the given predicate. Returns true if none of the keys match the predicate (a special case is when the map is empty); false if one or more keys match; NULL if the predicate function returns NULL for one or more keys and false for all other keys. :: + + SELECT no_keys_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'd'); -- true + +.. function:: no_values_match(x(K,V), function(V, boolean)) -> boolean + + Returns whether no values of a map match the given predicate. Returns true if none of the values match the predicate (a special case is when the map is empty); false if one or more values match; NULL if the predicate function returns NULL for one or more values and false for all other values. :: + + SELECT no_values_match(map(array['a', 'b', 'c'], array[1, 2, 3]), x -> x = 'd'); -- true + +.. function:: map_int_keys_to_array(map(int,V)) -> array(V) + + Returns an ``array`` of values from the ``map`` with value at indexed by the original keys from ``map``. :: + + SELECT MAP_INT_KEYS_TO_ARRAY(MAP(ARRAY[3, 5, 6, 9], ARRAY['a', 'b', 'c', 'd'])) -> ARRAY[null, null, 'a', null, 'b', 'c', null, null, 'd'] + SELECT MAP_INT_KEYS_TO_ARRAY(MAP(ARRAY[3, 5, 6, 9], ARRAY['a', null, 'c', 'd'])) -> ARRAY[null, null, 'a', null, null, 'c', 'd'] + + +.. function:: array_to_map_int_keys(array(v)) -> map(int, v) + + Returns an ``map`` with indices of all non-null values from the ``array`` as keys and element at the specified index as the value. :: + + SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, 6, 9] AS ARRAY)) -> MAP(ARRAY[1, 2, 3,4], ARRAY[3, 5, 6, 9]) + SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, null, 6, 9] AS ARRAY)) -> MAP(ARRAY[1, 2, 4, 5], ARRAY[3, 5, 6, 9]) + SELECT ARRAY_TO_MAP_INT_KEYS(CAST(ARRAY[3, 5, null, 6, 9, null, null, 1] AS ARRAY)) -> MAP(ARRAY[1, 2, 4, 5, 8], ARRAY[3, 5, 6, 9, 1]) + +String Functions +---------------- + +.. function:: replace_first(string, search, replace) -> varchar + + Replaces the first instance of ``search`` with ``replace`` in ``string``. + + If ``search`` is an empty string, it inserts ``replace`` at the beginning of the ``string``. + +.. function:: trail(string, N) -> varchar + + Returns the last N characters of the input string. + +.. function:: key_sampling_percent(varchar) -> double + + Generates a double value between 0.0 and 1.0 based on the hash of the given ``varchar``. + This function is useful for deterministic sampling of data. + diff --git a/presto-docs/src/main/sphinx/functions/string.rst b/presto-docs/src/main/sphinx/functions/string.rst index 62e94acb7ae48..a0f3d7cb42c37 100644 --- a/presto-docs/src/main/sphinx/functions/string.rst +++ b/presto-docs/src/main/sphinx/functions/string.rst @@ -10,6 +10,8 @@ The ``||`` operator performs concatenation. String Functions ---------------- +For plugin-loaded string functions, see :ref:`functions/plugin-loaded-functions:string functions`. + .. note:: These functions assume that the input strings contain valid UTF-8 encoded @@ -109,11 +111,6 @@ String Functions If ``search`` is an empty string, inserts ``replace`` in front of every character and at the end of the ``string``. -.. function:: replace_first(string, search, replace) -> varchar - Replaces the first instances of ``search`` with ``replace`` in ``string``. - - If ``search`` is an empty string, it inserts ``replace`` at the beginning of the ``string``. - .. function:: reverse(string) -> varchar Returns ``string`` with the characters in reverse order. @@ -224,10 +221,6 @@ String Functions position ``start``. Positions start with ``1``. A negative starting position is interpreted as being relative to the end of the string. -.. function:: trail(string, N) -> varchar - - Returns the last N characters of the input string. - .. function:: trim(string) -> varchar Removes leading and trailing whitespace from ``string``. @@ -317,7 +310,3 @@ Unicode Functions be a single character or empty (in which case invalid characters are removed). -.. function:: key_sampling_percent(varchar) -> double - - Generates a double value between 0.0 and 1.0 based on the hash of the given ``varchar``. - This function is useful for deterministic sampling of data. diff --git a/presto-docs/src/main/sphinx/installation.rst b/presto-docs/src/main/sphinx/installation.rst index c67f47b4324c4..daef03acb5288 100644 --- a/presto-docs/src/main/sphinx/installation.rst +++ b/presto-docs/src/main/sphinx/installation.rst @@ -6,6 +6,6 @@ Installation :maxdepth: 1 installation/deployment - installation/deploy-docker installation/deploy-brew + installation/deploy-docker installation/deploy-helm diff --git a/presto-docs/src/main/sphinx/installation/deploy-brew.rst b/presto-docs/src/main/sphinx/installation/deploy-brew.rst index bed202f4bd4d0..3f7b768535753 100644 --- a/presto-docs/src/main/sphinx/installation/deploy-brew.rst +++ b/presto-docs/src/main/sphinx/installation/deploy-brew.rst @@ -1,6 +1,6 @@ -============================ -Deploy Presto using Homebrew -============================ +=========================== +Deploy Presto with Homebrew +=========================== This guide explains how to install and get started with Presto on macOS, Linux or WSL2 using the Homebrew package manager. diff --git a/presto-docs/src/main/sphinx/installation/deploy-docker.rst b/presto-docs/src/main/sphinx/installation/deploy-docker.rst index b6a824916c40e..d1a7882c7fced 100644 --- a/presto-docs/src/main/sphinx/installation/deploy-docker.rst +++ b/presto-docs/src/main/sphinx/installation/deploy-docker.rst @@ -1,60 +1,66 @@ -================================= -Deploy Presto From a Docker Image -================================= +========================= +Deploy Presto with Docker +========================= + +This guide explains how to install and get started with Presto using Docker. + +.. note:: + + These steps were developed and tested on Mac OS X, on both Intel and Apple Silicon chips. -These steps were developed and tested on Mac OS X, on both Intel and Apple Silicon chips. +Prepare the container environment +================================= -Follow these steps to: +If Docker is already installed, skip to step 4 to verify the setup. +Otherwise, follow the instructions below to install Docker and Colima using Homebrew or choose an alternative method. -- install the command line tools for brew, docker, and `Colima `_ -- verify your Docker setup -- pull the Docker image of the Presto server -- start your local Presto server +1. Install `Homebrew `_ if it is not already present on the system. -Installing brew, Docker, and Colima -=================================== +2. Install the Docker command line and `Colima `_ tools via the following command: -This task shows how to install brew, then to use brew to install Docker and Colima. + .. code-block:: shell -Note: If you have Docker installed you can skip steps 1-3, but you should -verify your Docker setup by running the command in step 4. + brew install docker colima -1. If you do not have brew installed, run the following command: +3. Run the following command to start Colima with defaults: - ``/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"`` + .. code-block:: shell -2. To install the Docker command line and `Colima `_ tools, run the following command: + colima start - ``brew install docker colima`` + .. note:: -3. Run the following command: + The default VM created by Colima uses 2 CPUs, 2GiB memory and 100GiB storage. To customize the VM resources, + see the Colima README for `Customizing the VM `_. - ``colima start`` +4. Verify the local setup by running the following command: - *Note*: The default VM created by Colima uses 2 CPUs, 2GB memory and 60GB storage. To customize the VM resources, - see the Colima README for `Customizing the VM `_. + .. code-block:: shell -4. To verify your local setup, run the following command: + docker run hello-world - ``docker run hello-world`` + The following output confirms a successful installation. - If you see a response similar to the following, you are ready. + .. code-block:: shell + :class: no-copy - ``Hello from Docker!`` - ``This message shows that your installation appears to be working correctly.`` + Hello from Docker! + This message shows that your installation appears to be working correctly. Installing and Running the Presto Docker container ================================================== -1. Download the latest non-edge Presto container from `Presto on DockerHub `_. Run the following command: +1. Download the latest non-edge Presto container from `Presto on DockerHub `_: + + .. code-block:: shell - ``docker pull prestodb/presto:latest`` + docker pull prestodb/presto:latest Downloading the container may take a few minutes. When the download completes, go on to the next step. -2. On your local system, create a file named ``config.properties`` containing the following text: +2. On the local system, create a file named ``config.properties`` containing the following text: - .. code-block:: none + .. code-block:: properties coordinator=true node-scheduler.include-coordinator=true @@ -62,7 +68,7 @@ Installing and Running the Presto Docker container discovery-server.enabled=true discovery.uri=http://localhost:8080 -3. On your local system, create a file named ``jvm.config`` containing the following text: +3. On the local system, create a file named ``jvm.config`` containing the following text: .. code-block:: none @@ -78,20 +84,26 @@ Installing and Running the Presto Docker container 4. To start the Presto server in the Docker container, run the command: - ``docker run -p 8080:8080 -it -v ./config.properties:/opt/presto-server/etc/config.properties -v ./jvm.config:/opt/presto-server/etc/jvm.config --name presto prestodb/presto:latest`` + .. code-block:: shell + + docker run -p 8080:8080 -it -v ./config.properties:/opt/presto-server/etc/config.properties -v ./jvm.config:/opt/presto-server/etc/jvm.config --name presto prestodb/presto:latest This command assigns the name ``presto`` for the newly-created container that uses the downloaded image ``prestodb/presto:latest``. - The Presto server logs startup information in the terminal window. Once you see a response similar to the following, the Presto server is running in the Docker container. + The Presto server logs startup information in the terminal window. The following output confirms the Presto server is running in the Docker container. + + .. code-block:: shell + :class: no-copy - ``======== SERVER STARTED ========`` + ======== SERVER STARTED ======== Removing the Presto Docker container ==================================== -To remove the Presto Docker container, run the following two commands: +To stop and remove the Presto Docker container, run the following commands: -``docker stop presto`` +.. code-block:: shell -``docker rm presto`` + docker stop presto + docker rm presto These commands return the name of the container ``presto`` when they succeed. diff --git a/presto-docs/src/main/sphinx/installation/deploy-helm.rst b/presto-docs/src/main/sphinx/installation/deploy-helm.rst index 5fe7d0225e8c2..1fefb226fd142 100644 --- a/presto-docs/src/main/sphinx/installation/deploy-helm.rst +++ b/presto-docs/src/main/sphinx/installation/deploy-helm.rst @@ -1,5 +1,5 @@ -=============================== -Deploy Presto Using Helm Charts -=============================== +======================= +Deploy Presto with Helm +======================= -To deploy Presto using Helm charts, see the `Presto Helm Charts README `_. \ No newline at end of file +To deploy Presto using Helm, see the `Presto Helm Charts README `_. diff --git a/presto-docs/src/main/sphinx/plugin/native-sidecar-plugin.rst b/presto-docs/src/main/sphinx/plugin/native-sidecar-plugin.rst index c2557dd2f94a6..f64a36d58485c 100644 --- a/presto-docs/src/main/sphinx/plugin/native-sidecar-plugin.rst +++ b/presto-docs/src/main/sphinx/plugin/native-sidecar-plugin.rst @@ -26,7 +26,7 @@ Property Name Description ``coordinator-sidecar-enabled`` Enables sidecar in the coordinator true ``native-execution-enabled`` Enables native execution true ``presto.default-namespace`` Sets the default function namespace `native.default` -``plugin.dir`` Specifies which directory under installation root `{root-directory}/native-plugins/` +``plugin.dir`` Specifies which directory under installation root `{root-directory}/native-plugin/` to scan for plugins at startup. ============================================ ===================================================================== ============================== diff --git a/presto-docs/src/main/sphinx/plugin/redis-hbo-provider.rst b/presto-docs/src/main/sphinx/plugin/redis-hbo-provider.rst index 85079d719736f..423e9e4db9cc1 100644 --- a/presto-docs/src/main/sphinx/plugin/redis-hbo-provider.rst +++ b/presto-docs/src/main/sphinx/plugin/redis-hbo-provider.rst @@ -9,27 +9,27 @@ Redis HBO Provider supports loading a custom configured Redis Client for storing Configuration ------------- -Create ``etc/catalog/redis-provider.properties`` to mount the Redis HBO Provider Plugin. +Create ``etc/redis-provider.properties`` to mount the Redis HBO Provider Plugin. Edit the configuration properties as appropriate: Configuration properties ------------------------ -The following configuration properties are available for use in ``etc/catalog/redis-provider.properties``: +The following configuration properties are available for use in ``etc/redis-provider.properties``: ============================================ ===================================================================== Property Name Description ============================================ ===================================================================== -``coordinator`` Boolean property whether Presto server is a coordinator +``coordinator`` Boolean property to decide whether Presto server is a coordinator ``hbo.redis-provider.server_uri`` Redis Server URI ``hbo.redis-provider.total-fetch-timeoutms`` Maximum timeout in ms for Redis fetch requests ``hbo.redis-provider.total-set-timeoutms`` Maximum timeout in ms for Redis set requests ``hbo.redis-provider.default-ttl-seconds`` TTL in seconds of the Redis data to be stored -``hbo.redis-provider.enabled`` Boolean property whether this plugin is enabled in production +``hbo.redis-provider.enabled`` Boolean property to enable this plugin ``credentials-path`` Path for Redis credentials -``hbo.redis-provider.cluster-mode-enabled`` Boolean property whether cluster mode is enabled +``hbo.redis-provider.cluster-mode-enabled`` Boolean property to enable cluster mode ============================================ ===================================================================== Coordinator Configuration for Historical Based Optimization @@ -80,29 +80,18 @@ You can place the plugin JARs in the production's ``plugins`` directory. Alternatively, follow this method to ensure that the plugin is loaded during the Presto build. -1. Add the following to register the plugin in ```` in ``presto-server/src/main/assembly/presto.xml``: +1. Add the following to register the plugin in ``presto-server/src/main/provisio/presto.xml``: .. code-block:: text - - - ${project.build.directory}/dependency/redis-hbo-provider-${project.version} - plugin/redis-hbo-provider - + + + + + + 2. In ``redis-hbo-provider/src/main/resources``, create the file ``META-INF.services`` with the Plugin entry class ``com.facebook.presto.statistic.RedisProviderPlugin``. -3. Add the dependency on the module in ``presto-server/pom.xml``: - - .. code-block:: text - - - com.facebook.presto - redis-hbo-provider - ${project.version} - zip - provided - - -4. (Optional) Add your custom Redis client connection login in ``com.facebook.presto.statistic.RedisClusterAsyncCommandsFactory``. +3. (Optional) Add your custom Redis client connection login in ``com.facebook.presto.statistic.RedisClusterAsyncCommandsFactory``. Note: The AsyncCommands must be provided properly. diff --git a/presto-docs/src/main/sphinx/presto-cpp.rst b/presto-docs/src/main/sphinx/presto-cpp.rst index bd71e95daf6ed..f826d6beebafb 100644 --- a/presto-docs/src/main/sphinx/presto-cpp.rst +++ b/presto-docs/src/main/sphinx/presto-cpp.rst @@ -7,12 +7,15 @@ Note: Presto C++ is in active development. See :doc:`Limitations `_. + + For comprehensive documentation of all available runtime metrics, see :doc:`metrics`. diff --git a/presto-docs/src/main/sphinx/presto_cpp/functions.rst b/presto-docs/src/main/sphinx/presto_cpp/functions.rst new file mode 100644 index 0000000000000..1c5134b08d8ae --- /dev/null +++ b/presto-docs/src/main/sphinx/presto_cpp/functions.rst @@ -0,0 +1,8 @@ +******************** +Presto C++ Functions +******************** + +.. toctree:: + :maxdepth: 1 + + functions/sketch.rst \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/presto_cpp/functions/sketch.rst b/presto-docs/src/main/sphinx/presto_cpp/functions/sketch.rst new file mode 100644 index 0000000000000..a3165ddb32fb3 --- /dev/null +++ b/presto-docs/src/main/sphinx/presto_cpp/functions/sketch.rst @@ -0,0 +1,38 @@ +================ +Sketch Functions +================ + +Sketches are data structures that can approximately answer particular questions +about a dataset when full accuracy is not required. Approximate answers are +often faster and more efficient to compute than functions which result in full +accuracy. + +Presto C++ provides support for computing some sketches available in the `Apache +DataSketches`_ library. + +Theta Sketches +-------------- + +Theta sketches enable distinct value counting on datasets and also provide the +ability to perform set operations. For more information on Theta sketches, +please see the Apache DataSketches `Theta sketch documentation`_. + +.. function:: sketch_theta(x) -> varbinary + + Computes a theta sketch from an input dataset. The output from + this function can be used as an input to any of the other ``sketch_theta_*`` + family of functions. + +.. function:: sketch_theta_estimate(sketch) -> double + + Returns the estimate of distinct values from the input sketch. + +.. function:: sketch_theta_summary(sketch) -> row(estimate double, theta double, upper_bound_std double, lower_bound_std double, retained_entries int) + + Returns a summary of the input sketch which includes the distinct values + estimate alongside other useful information such as the sketch theta + parameter, current error bounds corresponding to 1 standard deviation, and + the number of retained entries in the sketch. + +.. _Apache DataSketches: https://datasketches.apache.org/ +.. _Theta sketch documentation: https://datasketches.apache.org/docs/Theta/ThetaSketches.html#theta-sketch-framework diff --git a/presto-docs/src/main/sphinx/presto_cpp/installation.rst b/presto-docs/src/main/sphinx/presto_cpp/installation.rst new file mode 100644 index 0000000000000..9182c16ae043c --- /dev/null +++ b/presto-docs/src/main/sphinx/presto_cpp/installation.rst @@ -0,0 +1,254 @@ +======================= +Presto C++ Installation +======================= + +.. contents:: + :local: + :backlinks: none + :depth: 1 + +This shows how to install and run a lightweight Presto cluster utilizing a PrestoDB Java Coordinator and Prestissimo (Presto C++) Workers using Docker. + +For more information about Presto C++, see the :ref:`presto-cpp:overview`. + +The setup uses Meta's high-performance Velox engine for worker-side query execution to configure a cluster and run a test query with the built-in TPC-H connector. + +Prerequisites +------------- + +To follow this tutorial, you need: + +* Docker installed. +* Basic familiarity with the terminal and shell commands. + +Create a Working Directory +-------------------------- +The recommended directory structure uses ``presto-lab`` as the root directory. + +Create a clean root directory to hold all necessary configuration files and the ``docker-compose.yml`` file. + +.. code-block:: bash + + mkdir -p ~/presto-lab + cd ~/presto-lab + +Configure the Presto Java Coordinator +------------------------------------- + +The Coordinator requires configuration to define its role, enable the discovery service, and set up a catalog for querying. + +1. Create Configuration Directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To create the necessary directories for the coordinator and its catalogs, run the following command: + +.. code-block:: bash + + mkdir -p coordinator/etc/catalog + + +2. Create the Coordinator Configuration File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the file ``coordinator/etc/config.properties`` with the following contents. This file enables the coordinator mode, the discovery server, and sets the HTTP port to ``8080``. + +.. code-block:: properties + + # coordinator/etc/config.properties + coordinator=true + node-scheduler.include-coordinator=true + http-server.http.port=8080 + discovery-server.enabled=true + discovery.uri=http://localhost:8080 + +* ``coordinator=true``: Enables the coordinator mode. +* ``discovery-server.enabled=true``: Designates the coordinator as the host for the worker discovery service. +* ``http-server.http.port=8080S``: Start the HTTP server on port 8080 for the coordinator (and workers, if enabled). + +3. Create the JVM Configuration File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the file ``coordinator/etc/jvm.config`` with the following content. These are standard Java 17 flags for Presto that ensures compatibility with Java 17's module system, provides stable garbage collection and memory behavior, and enforces safe failure handling. + +.. code-block:: properties + + # coordinator/etc/jvm.config + -server + -Xmx1G + -XX:+UseG1GC + -XX:G1HeapRegionSize=32M + -XX:+UseGCOverheadLimit + -XX:+ExplicitGCInvokesConcurrent + -XX:+HeapDumpOnOutOfMemoryError + -XX:+ExitOnOutOfMemoryError + -Djdk.attach.allowAttachSelf=true + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.ref=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.security=ALL-UNNAMED + --add-opens=java.base/javax.security.auth=ALL-UNNAMED + --add-opens=java.base/javax.security.auth.login=ALL-UNNAMED + --add-opens=java.base/java.text=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/java.util.regex=ALL-UNNAMED + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + +4. Create the Node Properties File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the file ``coordinator/etc/node.properties`` with the following content to set the node environment and the data directory. + +.. code-block:: properties + + # coordinator/etc/node.properties + node.id=${ENV:HOSTNAME} + node.environment=test + node.data-dir=/var/lib/presto/data + +5. Create the TPC-H Catalog Configuration File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the file ``coordinator/etc/catalog/tpch.properties`` with the following content. The TPC-H catalog enables running test queries against an in-memory dataset. + +.. code-block:: properties + + # coordinator/etc/catalog/tpch.properties + connector.name=tpch + +Configure the Prestissimo (C++) Worker +-------------------------------------- + +Configure the Worker to locate the Coordinator or Discovery service and identify itself within the network. + +1. Create Worker Configuration Directory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: bash + + mkdir -p worker-1/etc/catalog + +2. Create ``worker-1/etc/config.properties`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configure the worker to point to the discovery service running on the coordinator. + +Note: You can repeat this step to add more workers, such as ``worker-2``. + +.. code-block:: properties + + # worker-1/etc/config.properties + discovery.uri=http://coordinator:8080 + presto.version=0.288-15f14bb + http-server.http.port=7777 + shutdown-onset-sec=1 + runtime-metrics-collection-enabled=true + +* ``discovery.uri=http://coordinator:8080``: This uses the coordinator service name as defined in the ``docker-compose.yml`` file for network communication within Docker. + +3. Configure ``worker-1/etc/node.properties`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Define the worker’s internal address to ensure reliable registration. + +.. code-block:: properties + + # worker-1/etc/node.properties + node.environment=test + node.internal-address=worker-1 + node.location=docker + node.id=worker-1 + +* ``node.internal-address=worker-1``: This setting matches the service name defined in :ref:`Docker Compose `. + +4. Add TPC-H Catalog Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configure the worker with the same catalog definitions as the coordinator to execute query stages + +.. code-block:: properties + + # worker-1/etc/catalog/tpch.properties + connector.name=tpch + +.. _create-docker-compose-yml: + +Create ``docker-compose.yml`` +----------------------------- + +Create a ``docker-compose.yml`` file in the ``~/presto-lab`` directory to orchestrate both the Java Coordinator and the C++ Worker containers. + +.. code-block:: yaml + + # docker-compose.yml + services: + coordinator: + image: public.ecr.aws/oss-presto/presto:latest + platform: linux/amd64 + container_name: presto-coordinator + hostname: coordinator + ports: + - "8080:8080" + volumes: + - ./coordinator/etc:/opt/presto-server/etc:ro + restart: unless-stopped + + worker-1: + image: public.ecr.aws/oss-presto/presto-native:latest + platform: linux/amd64 + container_name: prestissimo-worker-1 + hostname: worker-1 + depends_on: + - coordinator + volumes: + - ./worker-1/etc:/opt/presto-server/etc:ro + restart: unless-stopped + + worker-2: + image: public.ecr.aws/oss-presto/presto-native:latest + platform: linux/amd64 + container_name: prestissimo-worker-2 + hostname: worker-2 + depends_on: + - coordinator + volumes: + - ./worker-2/etc:/opt/presto-server/etc:ro + restart: unless-stopped + +* The coordinator service uses the standard Java Presto image (presto:latest). +* The worker-1 and worker-2 services use the Prestissimo (C++ Native) image (presto-native:latest). +* The setting ``platform: linux/amd64`` is essential for users running on Apple Silicon Macs. +* The ``volumes`` section mounts your local configuration directories (``./coordinator/etc``, ``./worker-1/etc``) into the container's expected path (``/opt/presto-server/etc``). + +Start the Cluster and Verify +---------------------------- + +1. Start the Cluster +^^^^^^^^^^^^^^^^^^^^ + +Use Docker Compose to start the cluster in detached mode (``-d``). + +.. code-block:: bash + + docker compose up -d + +2. Verify +^^^^^^^^^ + +1. **Check the Web UI:** Open the Presto Web UI at http://localhost:8080. + + * You should see the UI displaying 3 Active Workers (1 Coordinator and 2 Workers). + +2. **Check Detailed Node Status** : Run the following SQL query to check the detailed status and metadata about every node (Coordinator and Workers). + + .. code-block:: sql + + select * from system.runtime.nodes; + + This confirms the cluster nodes are registered and active. \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/presto_cpp/limitations.rst b/presto-docs/src/main/sphinx/presto_cpp/limitations.rst index d86dedb95e2c5..ea957b5e60cea 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/limitations.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/limitations.rst @@ -7,6 +7,7 @@ Presto C++ Limitations :backlinks: none :depth: 1 + General Limitations =================== @@ -38,13 +39,369 @@ The C++ evaluation engine has a number of limitations: * The reserved pool is not supported. * In general, queries may use more memory than they are allowed to through memory arbitration. See `Memory Management `_. + Functions ========= -reduce_agg ----------- +Aggregate Functions +------------------- + +reduce_agg +^^^^^^^^^^ In C++ based Presto, ``reduce_agg`` is not permitted to return ``null`` in either the ``inputFunction`` or the ``combineFunction``. In Presto (Java), this is permitted but undefined behavior. For more information about ``reduce_agg`` in Presto, -see `reduce_agg <../functions/aggregate.html#reduce_agg>`_. \ No newline at end of file +see `reduce_agg <../functions/aggregate.html#reduce_agg>`_. + +reduce lambda +^^^^^^^^^^^^^ +For the reduce lambda function, the array size is controlled by the session property +``native_expression_max_array_size_in_reduce``, as it is inefficient to support such +cases for arbitrarily large arrays. This property is set at ``100K``. Queries that +fail due to this limit must be revised to meet this limit. + + +Array Functions +--------------- + +Array sort with lambda comparator +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``Case`` is not supported for the lambda comparator. Use ``If`` Instead. The following +example is not supported in Presto C++: + +.. code-block:: sql + + (x, y) -> + CASE + WHEN x.event_time < y.event_time THEN + -1 + WHEN x.event_time > y.event_time THEN + 1 + ELSE 0 + END + +To work with Presto C++, the best option is to use transform lambda whenever possible. +For example: + +.. code-block:: sql + + (x) -> x.event_time + +Or, rewrite using ``if`` as in the following example: + +.. code-block:: sql + + (x, y) -> IF (x.event_time < y.event_time, -1, + IF (x.event_time > y.event_time, 1, 0)) + +When using ``If``, follow these rules when using a lambda in array sort: + +* The lambda should use ``if else``. Case is not supported. +* The lambda should return ``1``, ``0``, ``-1``. Cover all the cases. +* The lambda should use the same expression when doing the comparison. + For example, in the above case ``event_time`` is used for comparison throughout the lambda. + If we rewrote the expression as following, where ``x`` and ``y`` have different fields, it will fail: + ``(x, y) -> if (x.event_time < y.event_start_time, -1, if (x.event_time > y.event_start_time, 1, 0))`` +* Any additional nesting other than the two ``if`` uses shown above will fail. + +``Array_sort`` can support any transformation lambda that returns a comparable type. +This example is not supported in Presto C++: + +.. code-block:: sql + + "array_sort"("map_values"(m), (a, b) -> ( + CASE WHEN (a[1] [2] > b[1] [2]) THEN 1 + WHEN (a[1] [2] < b[1] [2]) THEN -1 + WHEN (a[1] [2] = b[1] [2]) THEN + IF((a[3] > b[3]), 1, -1) END) + +To run in Presto C++, rewrite the query as shown in this example: + +.. code-block:: sql + + "array_sort"("map_values"(m), (a) -> ROW(a[1][2], a[3])) + + +Casting +------- + +Casting of Unicode strings to digits is not supported. The following example is not supported in Presto C++: + +.. code-block:: sql + + CAST ('Ⅶ' as integer) + + +Date and Time Functions +----------------------- +The maximum date range supported by ``from_unixtime`` is between (292 Million BCE, 292 Million CE). +The exact values corresponding to this are [292,275,055-05-16 08:54:06.192 BC, +292,278,994-08-17 00:12:55.807 CE], +corresponding to a UNIX time between [-9223372036854775, 9223372036854775]. + +Presto and Presto C++ both support the same range but Presto queries succeed because Presto silently +truncates. Presto C++ throws an error if the values exceed this range. + + +Geospatial Differences +---------------------- +There are cosmetic representation changes as well as numerical precision differences. +Some of these differences result in different output for spatial predicates such +as ST_Intersects. Differences include: + +* Equivalent but different representations for geometries. Polygons may have their rings + rotated, EMPTY geometries may be of a different type, MULTI-types and + GEOMETRYCOLLECTIONs may have their elements in a different order. In general, + WKTs/WKBs may be different. +* Numerical precision: Differences in numerical techniques may result in different + coordinate values, and also different results for predicates (ST_Relates and children, + including ST_Contains, ST_Crosses, ST_Disjoint, ST_Equals, ST_Intersects, + ST_Overlaps, ST_Relate, ST_Touches, ST_Within). +* ST_IsSimple, ST_IsValid, simplify_geometry and geometry_invalid_reason may give different results. + + +JSON Functions +-------------- +``json_extract`` has several topics to consider when rewriting Presto queries to run successfully in Presto C++. + +Use of functions in JSON path +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Using functions inside a JSON path is not supported. + +To run queries with functions inside a JSON path in Presto C++, rewrite paths to +use equivalent and often faster UDFs (User-Defined Functions) outside the JSON +path, improving job portability and efficiency. Aggregates might be necessary. + +Generally, functions should be extracted from the JSON path for better portability. + +For example, this Presto query: + +.. code-block:: sql + + CAST(JSON_EXTRACT(config, '$.table_name_to_properties.keys()' + ) AS ARRAY(ARRAY(VARCHAR))) + +can be revised to work in both Presto and Presto C++ as the following: + +.. code-block:: sql + + map_keys(JSON_EXTRACT( config, '$.table_name_to_properties') ) + +Use of expressions in JSON path +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Paths containing filter expressions are not supported. + +To run such queries in Presto C++, revise the query to do the filtering as a +part of the SQL expression query, rather than in the JSON path. + +For example, consider this Presto query: + +.. code-block:: sql + + JSON_EXTRACT(config, '$.store.book[?(@.price > 10)]') + +The same query rewritten to run in Presto C++: + +.. code-block:: sql + + filter( + CAST(json_extract(data, '$.store.book') AS ARRAY), + x -> CAST(json_extract_scalar(x.value, '$.price') AS DOUBLE) > 10) + ) + +Erroring on Invalid JSON +^^^^^^^^^^^^^^^^^^^^^^^^ +Presto can successfully run ``json_extract`` on certain invalid JSON, but Presto C++ +always fails. Extracting data from invalid JSON is indeterminate and relying on +that behavior can have unintended consequences. + +Because Presto C++ takes the safe approach to always throw an error on invalid +JSON, wrap calls in a try to ensure the query succeeds and validate that the +results correspond to your expectations. + +Canonicalization +^^^^^^^^^^^^^^^^ +Presto ``json_extract`` can return `JSON that is not canonicalized `_. +``json_extract`` has been rewritten in Presto C++ to always return canonical JSON. + + +Regex Functions +--------------- + +Unsupported Cases +^^^^^^^^^^^^^^^^^ +Presto C++ uses `RE2 `_, a widely adopted modern regular +expression parsing library. + +Presto uses `JONI `_, a deprecated port of Oniguruma (ONIG). + +While both frameworks support almost all regular expression syntaxes, RE2 differs from +JONI and PCRE in certain cases. The following are not supported in Presto C++ but are supported in Presto: + +* before text matching (?=re) +* before text not matching (?!re) +* after text matching (?<=re) +* after text not matching (?`_, +must be rewritten to run in Presto C++. See `Syntax `_ +for a full list of unsupported regular expressions in RE2 and +`Caveats `_ for an explanation of +why RE2 skips certain syntax in Perl. + +Regex Compilation Limit in Velox +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Because Regex compilation is CPU intensive, unbounded compilation can cause problems. +The number of regular expressions that can be dynamically compiled for a query is limited +to 250 to keep the overall shared cluster environment healthy. + +If this limit is reached, rewrite the query to use fewer compiled regular expressions. + +In this example the regex can change based on the ``test_name`` column value, which could exceed the 250 limit: + +.. code-block:: sql + + code_location_path LIKE '%' || test_name || '%' + +Revise the query as follows to avoid this limit: + +.. code-block:: sql + + strpos(code_location, test_name) > 0 + + +Time and Time with Time Zone +---------------------------- + +IANA Named Timezones Support +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Support for IANA named time zones - for example, `Europe/London`, `UTC`, `America/New_York`, +`Asia/Kolkata` - in ``TIME`` and ``TIME WITH TIME ZONE`` was removed from Presto C++ +to align with the SQL standard. Only fixed-offset time zones such as `+02:00` are +now supported for these types. + +Named time zones may still work when the Presto coordinator handles the query. + +To run queries involving ``TIME`` and ``TIME WITH TIME ZONE``, migrate to fixed-offset +time zones as soon as possible. + +These queries will fail in Presto C++, but may still work in Presto: + +.. code-block:: sql + + cast('14:00:01 UTC' as TIME WITH TIME ZONE) + cast('14:00:01 Europe/Paris' as TIME WITH TIME ZONE) + cast('14:00:01 America/New_York' as TIME WITH TIME ZONE) + cast('14:00:01 Asia/Kolkata' as TIME WITH TIME ZONE) + +These queries using fixed offsets will run successfully in Presto C++: + +.. code-block:: sql + + cast('14:00:01 +00:00' as TIME WITH TIME ZONE) + cast('14:00:01 +05:30' as TIME WITH TIME ZONE) + +Casting from TIMESTAMP to TIME +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In Presto, the result of CAST(TIMESTAMP AS TIME) or CAST(TIMESTAMP AS TIME WITH TIME ZONE) +would change based on the session property ``legacy_timestamp`` (true by default) when +applied to the user's time zone. In Presto C++ for ``TIME`` and ``TIME WITH TIME ZONE``, +the behavior is equivalent to the property being `false`. + +Note: ``TIMESTAMP`` behavior in Presto and Presto C++ is unchanged. + +For examples, consider the following queries and their responses when run in Presto: + +.. code-block:: sql + + -- Default behavior with legacy_timestamp=true: + -- Session Timezone - America/Los_Angeles + + -- DST Active Dates + select cast(TIMESTAMP '2023-08-05 10:15:00.000' as TIME); + -- Returns: 09:15:00.000 + select cast(TIMESTAMP '2023-08-05 10:15:00.000' as TIME WITH TIME ZONE); + -- Returns: 09:15:00.000 America/Los_Angeles + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME); + -- Returns: 09:15:00.000 + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME WITH TIME ZONE); + -- Returns: 09:15:00.000 + + -- DST Inactive Dates + select cast(TIMESTAMP '2023-12-05 10:15:00.000' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-12-05 10:15:00.000' as TIME WITH TIME ZONE); + -- Returns: 10:15:00.000 America/Los_Angeles + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-12-05 10:15:00.000 America/Los_Angeles' as TIME WITH TIME ZONE); + -- 10:15:00.000 America/Los_Angeles + +Consider the following queries and their responses when run in Presto C++ (Velox): + +.. code-block:: sql + + -- New Expected behavior similar to what currently exists if legacy_timestamp=false: + -- Session Timezone - America/Los_Angeles + + + -- DST Active Dates + select cast(TIMESTAMP '2023-08-05 10:15:00.000' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-08-05 10:15:00.000' as TIME WITH TIME ZONE); + -- Returns: 10:15:00.000 -07:00 + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME WITH TIME ZONE); + -- Returns: 10:15:00.000 -07:00 + + -- DST Inactive Dates + select cast(TIMESTAMP '2023-12-05 10:15:00.000' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-12-05 10:15:00.000' as TIME WITH TIME ZONE); + -- Returns: 10:15:00.000 -08:00 + select cast(TIMESTAMP '2023-08-05 10:15:00.000 America/Los_Angeles' as TIME); + -- Returns: 10:15:00.000 + select cast(TIMESTAMP '2023-12-05 10:15:00.000 America/Los_Angeles' as TIME WITH TIME ZONE); + -- Returns: 10:15:00.000 -08:00 + +Note: ``TIMESTAMP`` supports named time zones, unlike ``TIME`` and ``TIME WITH TIME ZONE``. + +DST Implications +^^^^^^^^^^^^^^^^ +Because IANA zones are not supported for ``TIME``, Presto C++ does not manage DST transitions. +All time interpretation is strictly in the provided offset, not local civil time. + +For example, ``14:00:00 +02:00`` always means 14:00 at a +02:00 fixed offset, regardless +of DST changes that might apply under an IANA zone. + +Recommendations +^^^^^^^^^^^^^^^ +* Use fixed-offset time zones like +02:00 with ``TIME`` and ``TIME WITH TIME ZONE``. +* Do not use IANA time zone names for ``TIME`` and ``TIME WITH TIME ZONE``. +* Confirm that your Presto C++ usage does not depend on legacy timestamp behavior. If your workload + depends on legacy ``TIME`` behavior, including support of IANA timezones, handle this outside + Presto or reach out so that we can discuss alternative solutions. +* Test: Try your most critical workflows with these settings. + + +URL Functions +------------- + +Presto and Presto C++ implement different URL function specifications which can lead to +some URL function mismatches. Presto C++ implements `RFC-3986 `_ whereas Presto +implements `RFC-2396 `_. This can lead to subtle differences as presented in +`this issue `_. + +Window Functions +---------------- + +Aggregate window functions do not support ``IGNORE NULLS``, returning the following error message: + +``!ignoreNulls Aggregate window functions do not support IGNORE NULLS.`` + +For Presto C++, remove the ``IGNORE NULLS`` clause. This clause is only defined for value functions +and does not apply to aggregate window functions. In Presto the results obtained with and without +the clause are similar, Presto C++ includes this clause whereas Presto just warns. \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/presto_cpp/metrics.rst b/presto-docs/src/main/sphinx/presto_cpp/metrics.rst new file mode 100644 index 0000000000000..81f85b19c9260 --- /dev/null +++ b/presto-docs/src/main/sphinx/presto_cpp/metrics.rst @@ -0,0 +1,674 @@ +========================== +Presto C++ Runtime Metrics +========================== + +.. contents:: + :local: + :backlinks: none + :depth: 1 + +Overview +======== + +Presto C++ workers expose various runtime metrics that can be collected and monitored when +``runtime-metrics-collection-enabled`` is set to true. These metrics are available through the +``GET /v1/info/metrics`` endpoint in Prometheus data format. + +For information on enabling metrics collection, see :doc:`features`. + +Executor Metrics +================ + +These metrics track the performance and queue sizes of various executors in the Presto C++ worker. + +``presto_cpp.driver_cpu_executor_queue_size`` +--------------------------------------------- + +* **Type:** gauge +* **Description:** Number of tasks currently queued in the driver CPU executor waiting to be processed. + +``presto_cpp.driver_cpu_executor_latency_ms`` +--------------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Latency distribution of tasks in the driver CPU executor, measuring the time from task submission to execution start. + +``presto_cpp.spiller_executor_queue_size`` +------------------------------------------ + +* **Type:** gauge +* **Description:** Number of spilling tasks currently queued in the spiller executor. + +``presto_cpp.spiller_executor_latency_ms`` +------------------------------------------ + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Latency distribution of spilling tasks in the spiller executor. + +``presto_cpp.http_executor_latency_ms`` +--------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Latency distribution of HTTP request processing tasks in the HTTP executor. + +HTTP Metrics +============ + +These metrics track HTTP requests and responses in the Presto C++ worker. + +``presto_cpp.num_http_request`` +------------------------------- + +* **Type:** counter +* **Description:** Total number of HTTP requests received by the worker since startup. + +``presto_cpp.num_http_request_error`` +------------------------------------- + +* **Type:** counter +* **Description:** Total number of HTTP request errors encountered by the worker since startup. + +``presto_cpp.http_request_latency_ms`` +-------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Latency distribution of HTTP request processing, from receipt to response. + +``presto_cpp.http_request_size_bytes`` +-------------------------------------- + +* **Type:** histogram +* **Unit:** bytes +* **Description:** Size distribution of HTTP request payloads. + +HTTP Client Metrics +=================== + +These metrics track HTTP client connection behavior for outbound requests. + +``presto_cpp.http.client.num_connections_created`` +-------------------------------------------------- + +* **Type:** counter +* **Description:** Total number of HTTP client connections created by the worker. + +``presto_cpp.http.client.connection_first_use`` +----------------------------------------------- + +* **Type:** counter +* **Description:** Number of HTTP requests that are the first request on a new connection (sequence number == 0). + +``presto_cpp.http.client.connection_reuse`` +------------------------------------------- + +* **Type:** counter +* **Description:** Number of HTTP requests sent on reused connections (sequence number > 0). + +``presto_cpp.http.client.transaction_create_delay_ms`` +------------------------------------------------------ + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Delay in creating HTTP client transactions. + +Exchange Metrics +================ + +These metrics track data exchange operations between workers. + +``presto_cpp.exchange_source_peak_queued_bytes`` +------------------------------------------------ + +* **Type:** gauge +* **Unit:** bytes +* **Description:** Peak number of bytes queued in PrestoExchangeSource waiting to be consumed. + +``presto_cpp.exchange.request.duration`` +---------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Duration distribution of exchange data fetch requests. + +``presto_cpp.exchange.request.num_tries`` +----------------------------------------- + +* **Type:** histogram +* **Description:** Number of retry attempts for exchange data fetch requests. + +``presto_cpp.exchange.request.page_size`` +----------------------------------------- +* **Type:** histogram +* **Unit:** bytes +* **Description:** Size distribution of data pages fetched through exchange requests. + +``presto_cpp.exchange.get_data_size.duration`` +---------------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Duration distribution of operations to get the size of exchange data. + +``presto_cpp.exchange.get_data_size.num_tries`` +----------------------------------------------- + +* **Type:** histogram +* **Description:** Number of retry attempts for getting exchange data size. + +Query Context and Memory Metrics +================================ + +These metrics track query execution contexts and memory usage. + +``presto_cpp.num_query_contexts`` +--------------------------------- + +* **Type:** gauge +* **Description:** Current number of active query contexts in the worker. + +``presto_cpp.memory_manager_total_bytes`` +----------------------------------------- + +* **Type:** gauge +* **Unit:** bytes +* **Description:** Total bytes currently used by the memory manager across all queries' memory pools. + +Task Metrics +============ + +These metrics track task lifecycle and execution states. + +Task Counts +----------- + +``presto_cpp.num_tasks`` +^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks created on this worker since startup. + +``presto_cpp.num_tasks_bytes_processed`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Unit:** bytes +* **Description:** Total bytes processed by all tasks on this worker. + +``presto_cpp.num_tasks_running`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Current number of tasks in running state. + +``presto_cpp.num_tasks_finished`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks that completed successfully. + +``presto_cpp.num_tasks_cancelled`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks that were cancelled. + +``presto_cpp.num_tasks_aborted`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks that were aborted. + +``presto_cpp.num_tasks_failed`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks that failed with an error. + +``presto_cpp.num_tasks_planned`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of tasks that have been created but not yet started, including queued tasks. + +``presto_cpp.num_tasks_queued`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of tasks currently waiting in the task queue. + +Task Health Metrics +------------------- + +``presto_cpp.num_zombie_velox_tasks`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of zombie Velox tasks (tasks that are no longer active but not cleaned up). + +``presto_cpp.num_zombie_presto_tasks`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of zombie Presto tasks (tasks that are no longer active but not cleaned up). + +``presto_cpp.num_tasks_with_stuck_operator`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of tasks that have at least one stuck operator. + +``presto_cpp.num_cancelled_tasks_by_stuck_driver`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks cancelled due to stuck drivers. + +``presto_cpp.num_tasks_deadlock`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of tasks that encountered deadlock conditions. + +``presto_cpp.num_tasks_manager_lock_timeout`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Number of times the task manager lock acquisition timed out. + +Driver Metrics +============== + +These metrics track the state and execution of drivers within tasks. + +Driver States +------------- + +``presto_cpp.num_queued_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers currently queued and waiting to execute. + +``presto_cpp.num_on_thread_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers currently executing on threads. + +``presto_cpp.num_suspended_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers that are suspended. + +Driver Blocking Reasons +----------------------- + +``presto_cpp.num_blocked_wait_for_consumer_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for downstream consumers to consume data. + +``presto_cpp.num_blocked_wait_for_split_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for new splits to be assigned. + +``presto_cpp.num_blocked_wait_for_producer_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for upstream producers to provide data. + +``presto_cpp.num_blocked_wait_for_join_build_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for join build side to complete. + +``presto_cpp.num_blocked_wait_for_join_probe_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked during join probe operations. + +``presto_cpp.num_blocked_wait_for_merge_join_right_side_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for merge join right side data. + +``presto_cpp.num_blocked_wait_for_memory_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for memory to become available. + +``presto_cpp.num_blocked_wait_for_connector_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers blocked waiting for connector operations to complete. + +``presto_cpp.num_blocked_yield_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers that have yielded execution. + +``presto_cpp.num_stuck_drivers`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of drivers that appear to be stuck and not making progress. + +Worker Overload Metrics +======================= + +These metrics indicate when the worker is overloaded and may reject new work. + +``presto_cpp.overloaded_mem`` +----------------------------- + +* **Type:** gauge +* **Description:** Exports 100 if the worker is overloaded in terms of memory usage, 0 otherwise. + +``presto_cpp.overloaded_cpu`` +----------------------------- + +* **Type:** gauge +* **Description:** Exports 100 if the worker is overloaded in terms of CPU usage, 0 otherwise. + +``presto_cpp.overloaded`` +------------------------- + +* **Type:** gauge +* **Description:** Exports 100 if the worker is overloaded in terms of either memory or CPU, 0 otherwise. + +``presto_cpp.task_planned_time_ms`` +----------------------------------- + +* **Type:** gauge +* **Unit:** milliseconds +* **Description:** Average time tasks spend in the planned state (queued) before starting execution. + +``presto_cpp.overloaded_duration_sec`` +-------------------------------------- + +* **Type:** gauge +* **Unit:** seconds +* **Description:** Duration in seconds that the worker has been continuously overloaded, or 0 if not currently overloaded. + +Output Buffer Metrics +===================== + +These metrics track the partitioned output buffers used for shuffling data. + +``presto_cpp.num_partitioned_output_buffer`` +-------------------------------------------- + +* **Type:** gauge +* **Description:** Total number of output buffers currently managed by all OutputBufferManagers. + +``presto_cpp.partitioned_output_buffer_get_data_latency_ms`` +------------------------------------------------------------ + +* **Type:** histogram +* **Unit:** milliseconds +* **Description:** Latency distribution of getData() calls on OutputBufferManager instances. + +Worker Runtime Metrics +====================== + +``presto_cpp.worker_runtime_uptime_secs`` +----------------------------------------- + +* **Type:** counter +* **Unit:** seconds +* **Description:** Worker runtime uptime in seconds after the worker process started. This metric tracks how long the worker has been running. + +Operating System Metrics +======================== + +These metrics provide insight into OS-level resource usage by the worker process. + +CPU Time Metrics +---------------- + +``presto_cpp.os_user_cpu_time_micros`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Unit:** microseconds +* **Description:** User CPU time consumed by the presto_server process since the process started. + +``presto_cpp.os_system_cpu_time_micros`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Unit:** microseconds +* **Description:** System CPU time consumed by the presto_server process since the process started. + +Page Fault Metrics +------------------ + +``presto_cpp.os_num_soft_page_faults`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of soft page faults (page faults that can be resolved without disk I/O) encountered by the presto_server process since startup. + +``presto_cpp.os_num_hard_page_faults`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of hard page faults (page faults requiring disk I/O) encountered by the presto_server process since startup. + +Context Switch Metrics +---------------------- + +``presto_cpp.os_num_voluntary_context_switches`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of voluntary context switches in the presto_server process (when the process yields the CPU voluntarily). + +``presto_cpp.os_num_forced_context_switches`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Total number of involuntary context switches in the presto_server process (when the process is preempted by the OS). + +Hive Connector Metrics +====================== + +These metrics track the performance of the Hive connector's file handle cache. The metrics include +a placeholder ``{}`` in their name which is replaced with the connector name at runtime. + +File Handle Cache Metrics +------------------------- + +``presto_cpp.{connector}.hive_file_handle_cache_num_elements`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Description:** Number of elements currently in the Hive file handle cache. + +``presto_cpp.{connector}.hive_file_handle_cache_pinned_size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Unit:** bytes +* **Description:** Total size of pinned (in-use) entries in the Hive file handle cache. + +``presto_cpp.{connector}.hive_file_handle_cache_cur_size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** gauge +* **Unit:** bytes +* **Description:** Current total size of the Hive file handle cache. + +``presto_cpp.{connector}.hive_file_handle_cache_num_accumulative_hits`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Cumulative number of cache hits in the Hive file handle cache since startup. + +``presto_cpp.{connector}.hive_file_handle_cache_num_accumulative_lookups`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Cumulative number of cache lookups in the Hive file handle cache since startup. + +``presto_cpp.{connector}.hive_file_handle_cache_num_hits`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Number of cache hits in the Hive file handle cache (recent window). + +``presto_cpp.{connector}.hive_file_handle_cache_num_lookups`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** counter +* **Description:** Number of cache lookups in the Hive file handle cache (recent window). + +Thread Pool Metrics +=================== + +These metrics track the state of various thread pools. The metrics include a placeholder ``{}`` +in their name which is replaced with the thread pool name at runtime. + +``presto_cpp.{pool}.num_threads`` +--------------------------------- + +* **Type:** gauge +* **Description:** Current number of threads in the thread pool. + +``presto_cpp.{pool}.num_active_threads`` +---------------------------------------- + +* **Type:** gauge +* **Description:** Number of threads currently executing tasks in the thread pool. + +``presto_cpp.{pool}.num_pending_tasks`` +--------------------------------------- + +* **Type:** gauge +* **Description:** Number of tasks waiting to be executed in the thread pool. + +``presto_cpp.{pool}.num_total_tasks`` +------------------------------------- + +* **Type:** counter +* **Description:** Total number of tasks that have been submitted to the thread pool since startup. + +``presto_cpp.{pool}.max_idle_time_ns`` +-------------------------------------- + +* **Type:** gauge +* **Unit:** nanoseconds +* **Description:** Maximum idle time for threads in the pool before they are terminated. + +EventBase Violation Metrics +=========================== + +These metrics track violations of the EventBase (event loop) threading model. + +``presto_cpp.exchange_io_evb_violation_count`` +---------------------------------------------- + +* **Type:** counter +* **Description:** Number of times the exchange I/O EventBase threading model was violated (operations performed on wrong thread). + +``presto_cpp.http_server_io_evb_violation_count`` +------------------------------------------------- + +* **Type:** counter +* **Description:** Number of times the HTTP server I/O EventBase threading model was violated. + +Memory Pushback Metrics +======================= + +These metrics track the memory pushback mechanism that helps prevent out-of-memory conditions. + +``presto_cpp.memory_pushback_count`` +------------------------------------ + +* **Type:** counter +* **Description:** Number of times the memory pushback mechanism has been triggered. + +``presto_cpp.memory_pushback_latency_ms`` +----------------------------------------- + +* **Type:** histogram +* **Unit:** milliseconds +* **Range:** 0-100,000 ms (0-100 seconds) +* **Percentiles:** P50, P90, P99, P100 +* **Description:** Latency distribution of memory pushback operations, measuring how long each pushback attempt takes. + +``presto_cpp.memory_pushback_reduction_bytes`` +---------------------------------------------- + +* **Type:** histogram +* **Unit:** bytes +* **Range:** 0-15 GB (150 buckets) +* **Percentiles:** P50, P90, P99, P100 +* **Description:** Distribution of actual memory usage reduction achieved by each memory pushback attempt. This metric helps gauge the effectiveness of the memory pushback mechanism. + +``presto_cpp.memory_pushback_expected_reduction_bytes`` +------------------------------------------------------- + +* **Type:** histogram +* **Unit:** bytes +* **Range:** 0-15 GB (150 buckets) +* **Percentiles:** P50, P90, P99, P100 +* **Description:** Distribution of expected memory usage reduction for each memory pushback attempt. The expected reduction may differ from actual reduction as other threads might allocate memory during the pushback operation. + +Additional Runtime Metrics +========================== + +For additional runtime metrics related to specific subsystems: + +* **S3 FileSystem Metrics:** When Presto C++ workers interact with S3, additional runtime metrics are collected. See the `Velox S3 FileSystem documentation `_. + +* **Velox Metrics:** Metrics from the underlying Velox execution engine are also available. These are prefixed with ``velox.`` instead of ``presto_cpp.``. See the `Velox metrics documentation `_. + +Accessing Metrics +================= + +To access these metrics: + +1. Enable metrics collection by setting ``runtime-metrics-collection-enabled=true`` in your worker configuration. + +2. Query the metrics endpoint: + + .. code-block:: bash + + curl http://worker-host:7777/v1/info/metrics + +3. The response will be in Prometheus text format, suitable for scraping by Prometheus or other monitoring systems. + +Example Output +-------------- + +.. code-block:: text + + # TYPE presto_cpp_worker_runtime_uptime_secs counter + presto_cpp_worker_runtime_uptime_secs{cluster="production",worker="worker-01"} 3600 + # TYPE presto_cpp_num_tasks_running gauge + presto_cpp_num_tasks_running{cluster="production",worker="worker-01"} 42 + # TYPE presto_cpp_memory_manager_total_bytes gauge + presto_cpp_memory_manager_total_bytes{cluster="production",worker="worker-01"} 8589934592 + +See Also +======== + +* :doc:`features` - For information on enabling metrics collection +* :doc:`properties` - For worker configuration properties +* `Velox Metrics Documentation `_ - For metrics from the Velox execution engine diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index cac0763034dba..fd2357cf3f19e 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -300,6 +300,15 @@ The maximum bytes to buffer per PartitionedOutput operator to avoid creating tin For PartitionedOutputNode::Kind::kPartitioned, PartitionedOutput operator would buffer up to that number of bytes / number of destinations for each destination before producing a SerializedPage. Default is 32MB. +``native_partitioned_output_eager_flush`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +Native Execution only. If true, the PartitionedOutput operator will flush rows eagerly, without waiting +until buffers reach a certain size. Default is false. + ``native_max_local_exchange_partition_count`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -472,6 +481,17 @@ In streaming aggregation, wait until there are enough output rows to produce a batch of the size specified by this property. If set to ``0``, then ``Operator::outputBatchRows`` is used as the minimum number of output batch rows. +``native_merge_join_output_batch_start_size`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``integer`` +* **Default value:** ``0`` + +Native Execution only. Initial output batch size in rows for MergeJoin operator. +When non-zero, the batch size starts at this value and is dynamically adjusted +based on the average row size of previous output batches. When zero (default), +dynamic adjustment is disabled and the batch size is fixed at ``preferred_output_batch_rows``. + ``native_request_data_sizes_max_wait_sec`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -568,3 +588,28 @@ with StringView type during global aggregation. Native Execution only. Ratio of unused (evicted) bytes to total bytes that triggers compaction. The value is in the range of [0, 1). Currently only applies to approx_most_frequent aggregate with StringView type during global aggregation. + +``native_aggregation_memory_compaction_reclaim_enabled`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +Native Execution only. If true, enables lightweight memory compaction before +spilling during memory reclaim in aggregation. When enabled, the aggregation +operator will try to compact aggregate function state (for example, free dead strings) +before resorting to spilling. + +``optimizer.optimize_top_n_rank`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``boolean`` +* **Default value:** ``false`` + +If this is true, then filter and limit queries for ``n`` rows of +``rank()`` and ``dense_rank()`` window function values are executed +with a special TopNRowNumber operator instead of the +WindowFunction operator. + +The TopNRowNumber operator is more efficient than window as +it has a streaming behavior and does not need to buffer all input rows. diff --git a/presto-docs/src/main/sphinx/router/deployment.rst b/presto-docs/src/main/sphinx/router/deployment.rst index 04f4956db8e52..a1426471f49bf 100644 --- a/presto-docs/src/main/sphinx/router/deployment.rst +++ b/presto-docs/src/main/sphinx/router/deployment.rst @@ -2,11 +2,6 @@ Deploying Presto Router ======================= -.. contents:: - :local: - :backlinks: none - :depth: 1 - Installing Router ----------------- diff --git a/presto-docs/src/main/sphinx/router/scheduler.rst b/presto-docs/src/main/sphinx/router/scheduler.rst index bb9292e9fa927..693ec5c6d7044 100644 --- a/presto-docs/src/main/sphinx/router/scheduler.rst +++ b/presto-docs/src/main/sphinx/router/scheduler.rst @@ -2,36 +2,31 @@ Router Schedulers ================= -.. contents:: - :local: - :backlinks: none - :depth: 1 - Presto router provides multiple scheduling algorithms for load balancing across multiple clusters. * ``RANDOM_CHOICE`` -Randomly selecting a cluster from a list of candidates. + Randomly selecting a cluster from a list of candidates. * ``ROUND_ROBIN`` -Selecting clusters from a list of candidates in turn. Note that as the algorithm -keeps the state of the selected index, it can only be used when the candidates -are always consistent. + Selecting clusters from a list of candidates in turn. Because the algorithm + keeps the state of the selected index, it can only be used when the candidates + are always consistent. * ``USER_HASH`` -Selecting a clusters by hashing the username. This ensures queries from the same -user will always be routed to the same cluster. + Selecting a cluster by hashing the username. This ensures queries from the same + user are always routed to the same cluster. * ``WEIGHTED_RANDOM_CHOICE`` -Randomly selecting a cluster from a list of candidates with pre-defined weights. -Clusters with higher weights have higher opportunity to be selected. + Randomly selecting a cluster from a list of candidates with pre-defined weights. + Clusters with higher weights have higher opportunity to be selected. * ``WEIGHTED_ROUND_ROBIN`` -Selecting clusters from a list of candidates with pre-defined weights in turn. -Note that similar to the `ROUND_ROBIN` approach, this algorithm keeps the state -of the selected index so candidates and weights should be consistent. + Selecting clusters from a list of candidates with pre-defined weights in turn. + Similar to the `ROUND_ROBIN` approach, this algorithm keeps the state + of the selected index so candidates and weights should be consistent. diff --git a/presto-docs/src/main/sphinx/sql/alter-table.rst b/presto-docs/src/main/sphinx/sql/alter-table.rst index 93778714a24bb..e128a7c3694da 100644 --- a/presto-docs/src/main/sphinx/sql/alter-table.rst +++ b/presto-docs/src/main/sphinx/sql/alter-table.rst @@ -17,6 +17,15 @@ Synopsis ALTER TABLE [ IF EXISTS ] name SET PROPERTIES (property_name=value, [, ...]) ALTER TABLE [ IF EXISTS ] name DROP BRANCH [ IF EXISTS ] branch_name ALTER TABLE [ IF EXISTS ] name DROP TAG [ IF EXISTS ] tag_name + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name FOR SYSTEM_VERSION AS OF version + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name FOR SYSTEM_TIME AS OF timestamp + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name FOR SYSTEM_VERSION AS OF version RETAIN retention_period + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name FOR SYSTEM_VERSION AS OF version RETAIN retention_period WITH SNAPSHOT RETENTION min_snapshots SNAPSHOTS min_retention_period + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] TAG [ IF NOT EXISTS ] tag_name + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] TAG [ IF NOT EXISTS ] tag_name FOR SYSTEM_VERSION AS OF version + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] TAG [ IF NOT EXISTS ] tag_name FOR SYSTEM_TIME AS OF timestamp + ALTER TABLE [ IF EXISTS ] name CREATE [ OR REPLACE ] TAG [ IF NOT EXISTS ] tag_name FOR SYSTEM_VERSION AS OF version RETAIN retention_period Description ----------- @@ -29,6 +38,18 @@ The optional ``IF EXISTS`` (when used before the column name) clause causes the The optional ``IF NOT EXISTS`` clause causes the error to be suppressed if the column already exists. +For ``CREATE BRANCH`` statements: + +* The optional ``OR REPLACE`` clause causes the branch to be replaced if it already exists. +* The optional ``IF NOT EXISTS`` clause causes the error to be suppressed if the branch already exists. +* ``OR REPLACE`` and ``IF NOT EXISTS`` cannot be specified together. + +For ``CREATE TAG`` statements: + +* The optional ``OR REPLACE`` clause causes the tag to be replaced if it already exists. +* The optional ``IF NOT EXISTS`` clause causes the error to be suppressed if the tag already exists. +* ``OR REPLACE`` and ``IF NOT EXISTS`` cannot be specified together. + Examples -------- @@ -104,6 +125,66 @@ Drop tag ``tag1`` from the ``users`` table:: ALTER TABLE users DROP TAG 'tag1'; +Create branch ``branch1`` from the ``users`` table:: + + ALTER TABLE users CREATE BRANCH 'branch1'; + +Create branch ``branch1`` from the ``users`` table only if it doesn't already exist:: + + ALTER TABLE users CREATE BRANCH IF NOT EXISTS 'branch1'; + +Create or replace branch ``branch1`` from the ``users`` table:: + + ALTER TABLE users CREATE OR REPLACE BRANCH 'branch1'; + +Create branch ``branch1`` from the ``users`` table for system version as of version 5:: + + ALTER TABLE users CREATE BRANCH 'branch1' FOR SYSTEM_VERSION AS OF 5; + +Create branch ``branch1`` from the ``users`` table for system version as of version 5, only if it doesn't exist:: + + ALTER TABLE users CREATE BRANCH IF NOT EXISTS 'branch1' FOR SYSTEM_VERSION AS OF 5; + +Create or replace branch ``branch1`` from the ``users`` table for system time as of timestamp '2026-01-02 17:30:35.247 Asia/Kolkata':: + + ALTER TABLE users CREATE OR REPLACE BRANCH 'branch1' FOR SYSTEM_TIME AS OF TIMESTAMP '2026-01-02 17:30:35.247 Asia/Kolkata'; + +Create branch ``branch1`` from the ``users`` table for system version as of version 5 with retention period of 30 days:: + + ALTER TABLE users CREATE BRANCH 'branch1' FOR SYSTEM_VERSION AS OF 5 RETAIN INTERVAL 30 DAY; + +Create branch ``branch1`` from the ``users`` table for system version as of version 5 with snapshot retention of minimum 3 snapshots and minimum retention period of 7 days:: + + ALTER TABLE users CREATE BRANCH 'branch1' FOR SYSTEM_VERSION AS OF 5 RETAIN INTERVAL 7 DAY WITH SNAPSHOT RETENTION 3 SNAPSHOTS INTERVAL 7 DAYS; + +Create tag ``tag1`` from the ``users`` table:: + + ALTER TABLE users CREATE TAG 'tag1'; + +Create tag ``tag1`` from the ``users`` table only if it doesn't already exist:: + + ALTER TABLE users CREATE TAG IF NOT EXISTS 'tag1'; + +Create or replace tag ``tag1`` from the ``users`` table:: + + ALTER TABLE users CREATE OR REPLACE TAG 'tag1'; + +Create tag ``tag1`` from the ``users`` table for system version as of version 5:: + + ALTER TABLE users CREATE TAG 'tag1' FOR SYSTEM_VERSION AS OF 5; + +Create tag ``tag1`` from the ``users`` table for system version as of version 5, only if it doesn't exist:: + + ALTER TABLE users CREATE TAG IF NOT EXISTS 'tag1' FOR SYSTEM_VERSION AS OF 5; + +Create or replace tag ``tag1`` from the ``users`` table for system time as of timestamp '2026-01-02 17:30:35.247 Asia/Kolkata':: + + ALTER TABLE users CREATE OR REPLACE TAG 'tag1' FOR SYSTEM_TIME AS OF TIMESTAMP '2026-01-02 17:30:35.247 Asia/Kolkata'; + +Create tag ``tag1`` from the ``users`` table for system version as of version 5 with retention period of 30 days:: + + ALTER TABLE users CREATE TAG 'tag1' FOR SYSTEM_VERSION AS OF 5 RETAIN INTERVAL 30 DAY; + See Also -------- diff --git a/presto-docs/src/main/sphinx/sql/analyze.rst b/presto-docs/src/main/sphinx/sql/analyze.rst index 5d91a928b7fdb..bdcb793bc849c 100644 --- a/presto-docs/src/main/sphinx/sql/analyze.rst +++ b/presto-docs/src/main/sphinx/sql/analyze.rst @@ -21,7 +21,8 @@ connector-specific properties. To list all available properties, run the followi SELECT * FROM system.metadata.analyze_properties Currently, this statement is only supported by the -:ref:`Hive connector `. +:ref:`Hive connector ` and +:ref:`Iceberg connector `. Examples -------- @@ -34,6 +35,10 @@ Analyze table ``stores`` in catalog ``hive`` and schema ``default``:: ANALYZE hive.default.stores; +Analyze table ``stores`` in catalog ``iceberg`` and schema ``default``:: + + ANALYZE iceberg.default.stores; + Analyze partitions ``'1992-01-01', '1992-01-02'`` from a Hive partitioned table ``sales``:: ANALYZE hive.default.sales WITH (partitions = ARRAY[ARRAY['1992-01-01'], ARRAY['1992-01-02']]); diff --git a/presto-docs/src/main/sphinx/sql/explain-analyze.rst b/presto-docs/src/main/sphinx/sql/explain-analyze.rst index 9111aa0b02355..0cb0c2c5e6104 100644 --- a/presto-docs/src/main/sphinx/sql/explain-analyze.rst +++ b/presto-docs/src/main/sphinx/sql/explain-analyze.rst @@ -36,43 +36,91 @@ relevant plan nodes). Such statistics are useful when one wants to detect data a .. code-block:: none - presto:sf1> EXPLAIN ANALYZE SELECT count(*), clerk FROM orders WHERE orderdate > date '1995-01-01' GROUP BY clerk; + presto:tiny> EXPLAIN ANALYZE SELECT + -> s.acctbal, + -> s.name, + -> n.name, + -> p.partkey, + -> p.mfgr, + -> s.address, + -> s.phone, + -> s.comment + -> FROM + -> part p, + -> supplier s, + -> partsupp ps, + -> nation n, + -> region r + -> WHERE + -> p.partkey = ps.partkey + -> AND s.suppkey = ps.suppkey + -> AND p.size = 15 + -> AND p.type like '%BRASS' + -> AND s.nationkey = n.nationkey + -> AND n.regionkey = r.regionkey + -> AND r.name = 'EUROPE' + -> AND ps.supplycost = ( + -> SELECT + -> min(ps.supplycost) + -> FROM + -> partsupp ps, + -> supplier s, + -> nation n, + -> region r + -> WHERE + -> p.partkey = ps.partkey + -> AND s.suppkey = ps.suppkey + -> AND s.nationkey = n.nationkey + -> AND n.regionkey = r.regionkey + -> AND r.name = 'EUROPE' + -> ) + -> ORDER BY + -> s.acctbal desc, + -> n.name, + -> s.name, + -> p.partkey + -> LIMIT 100; + + Query Plan ----------------------------------------------------------------------------------------------- - Fragment 1 [HASH] - Cost: CPU 88.57ms, Input: 4000 rows (148.44kB), Output: 1000 rows (28.32kB) - Output layout: [count, clerk] - Output partitioning: SINGLE [] - - Project[] => [count:bigint, clerk:varchar(15)] - Cost: 26.24%, Input: 1000 rows (37.11kB), Output: 1000 rows (28.32kB), Filtered: 0.00% - Input avg.: 62.50 lines, Input std.dev.: 14.77% - - Aggregate(FINAL)[clerk][$hashvalue] => [clerk:varchar(15), $hashvalue:bigint, count:bigint] - Cost: 16.83%, Output: 1000 rows (37.11kB) - Input avg.: 250.00 lines, Input std.dev.: 14.77% - count := "count"("count_8") - - LocalExchange[HASH][$hashvalue] ("clerk") => clerk:varchar(15), count_8:bigint, $hashvalue:bigint - Cost: 47.28%, Output: 4000 rows (148.44kB) - Input avg.: 4000.00 lines, Input std.dev.: 0.00% - - RemoteSource[2] => [clerk:varchar(15), count_8:bigint, $hashvalue_9:bigint] - Cost: 9.65%, Output: 4000 rows (148.44kB) - Input avg.: 4000.00 lines, Input std.dev.: 0.00% - - Fragment 2 [tpch:orders:1500000] - Cost: CPU 14.00s, Input: 818058 rows (22.62MB), Output: 4000 rows (148.44kB) - Output layout: [clerk, count_8, $hashvalue_10] - Output partitioning: HASH [clerk][$hashvalue_10] - - Aggregate(PARTIAL)[clerk][$hashvalue_10] => [clerk:varchar(15), $hashvalue_10:bigint, count_8:bigint] - Cost: 4.47%, Output: 4000 rows (148.44kB) - Input avg.: 204514.50 lines, Input std.dev.: 0.05% - Collisions avg.: 5701.28 (17569.93% est.), Collisions std.dev.: 1.12% - count_8 := "count"(*) - - ScanFilterProject[table = tpch:tpch:orders:sf1.0, originalConstraint = ("orderdate" > "$literal$date"(BIGINT '9131')), filterPredicate = ("orderdate" > "$literal$date"(BIGINT '9131'))] => [cler - Cost: 95.53%, Input: 1500000 rows (0B), Output: 818058 rows (22.62MB), Filtered: 45.46% - Input avg.: 375000.00 lines, Input std.dev.: 0.00% - $hashvalue_10 := "combine_hash"(BIGINT '0', COALESCE("$operator$hash_code"("clerk"), 0)) - orderdate := tpch:orderdate - clerk := tpch:clerk + ... + Fragment 4 [SOURCE] + CPU: 31.55ms, Scheduled: 38.34ms, Input: 8,020 rows (260B); per task: avg.: 8,020.00 std.dev.: 0.00, Output: 1,196 rows (21.02kB), 1 tasks + Output layout: [partkey_15, min_73] + Output partitioning: HASH [partkey_15] + Output encoding: COLUMNAR + Stage Execution Strategy: UNGROUPED_EXECUTION + - Aggregate(PARTIAL)[partkey_15][PlanNodeId 3023] => [partkey_15:bigint, min_73:double] + CPU: 3.00ms (1.74%), Scheduled: 4.00ms (0.54%), Output: 1,196 rows (21.02kB) + Input total: 1,600 rows (40.63kB), avg.: 400.00 rows, std.dev.: 0.00% + Collisions avg.: 4.50 (160.41% est.), Collisions std.dev.: 86.78% + min_73 := "presto.default.min"((supplycost_18)) (1:365) + - InnerJoin[PlanNodeId 2455][("suppkey_16" = "suppkey_21")] => [partkey_15:bigint, supplycost_18:double] + Estimates: {source: CostBasedSourceInfo, rows: 1,600 (28.13kB), cpu: 684,460.00, memory: 225.00, network: 234.00} + CPU: 11.00ms (6.40%), Scheduled: 13.00ms (1.77%), Output: 1,600 rows (40.63kB) + Left (probe) Input total: 8,000 rows (210.94kB), avg.: 2,000.00 rows, std.dev.: 0.00% + Right (build) Input total: 20 rows (260B), avg.: 1.25 rows, std.dev.: 60.00% + Collisions avg.: 0.40 (30.84% est.), Collisions std.dev.: 183.71% + Distribution: REPLICATED + - ScanFilter[PlanNodeId 9,2699][table = TableHandle {connectorId='tpch', connectorHandle='partsupp:sf0.01', layout='Optional[partsupp:sf0.01]'}, grouped = false, filterPredicate = (not(IS_NULL(partkey_15))) AND (not(IS_NULL(suppkey_16)))] => [partkey_15:bigint, suppkey_16:bigint, supplycost_18:double] + Estimates: {source: CostBasedSourceInfo, rows: 8,000 (210.94kB), cpu: 216,000.00, memory: 0.00, network: 0.00}/{source: CostBasedSourceInfo, rows: 8,000 (210.94kB), cpu: 432,000.00, memory: 0.00, network: 0.00} + CPU: 14.00ms (8.14%), Scheduled: 16.00ms (2.17%), Output: 8,000 rows (210.94kB) + Input total: 8,000 rows (0B), avg.: 2,000.00 rows, std.dev.: 0.00% + partkey_15 := tpch:partkey (1:389) + supplycost_18 := tpch:supplycost (1:389) + suppkey_16 := tpch:suppkey (1:389) + Input: 8,000 rows (0B), Filtered: 0.00% + - LocalExchange[PlanNodeId 2949][HASH] (suppkey_21) => [suppkey_21:bigint] + Estimates: {source: CostBasedSourceInfo, rows: 20 (180B), cpu: 7,480.00, memory: 54.00, network: 234.00} + CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 20 rows (260B) + Input total: 20 rows (260B), avg.: 1.25 rows, std.dev.: 225.39% + - RemoteSource[5] => [suppkey_21:bigint] + CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 20 rows (260B) + Input total: 20 rows (260B), avg.: 1.25 rows, std.dev.: 225.39% + ... + When the ``VERBOSE`` option is used, some operators may report additional information. For example, the window function operator will output the following: diff --git a/presto-docs/src/main/sphinx/sql/explain.rst b/presto-docs/src/main/sphinx/sql/explain.rst index 1b541b1d82e9a..dc5cf1339fd4e 100644 --- a/presto-docs/src/main/sphinx/sql/explain.rst +++ b/presto-docs/src/main/sphinx/sql/explain.rst @@ -20,9 +20,9 @@ Description ----------- Show the logical or distributed execution plan of a statement, or validate the statement. -Use ``TYPE DISTRIBUTED`` option to display fragmented plan. Each -`plan fragment `_ -is executed by a single or multiple Presto nodes. Fragment type specifies how the fragment +Use ``TYPE DISTRIBUTED`` option to display a fragmented plan. Each +`plan fragment `_ +is executed by a single or multiple Presto nodes. Fragment type specifies how the fragment is executed by Presto nodes and how the data is distributed between fragments: ``SINGLE`` @@ -152,6 +152,48 @@ IO: } } +DDL Statements +^^^^^^^^^^^^^^ + +``EXPLAIN`` can also be used with DDL statements such as ``CREATE TABLE`` and ``DROP TABLE``. +For these statements, the output shows a summary of the operation rather than an execution plan. + +CREATE TABLE: + +.. code-block:: none + + presto:tiny> EXPLAIN CREATE TABLE new_table (id BIGINT, name VARCHAR); + Query Plan + -------------------------- + CREATE TABLE new_table + +CREATE TABLE IF NOT EXISTS: + +.. code-block:: none + + presto:tiny> EXPLAIN CREATE TABLE IF NOT EXISTS new_table (id BIGINT, name VARCHAR); + Query Plan + -------------------------------------- + CREATE TABLE IF NOT EXISTS new_table + +DROP TABLE: + +.. code-block:: none + + presto:tiny> EXPLAIN DROP TABLE test_table; + Query Plan + -------------------------------------------------------------- + DROP TABLE test_table + +DROP TABLE IF EXISTS: + +.. code-block:: none + + presto:tiny> EXPLAIN DROP TABLE IF EXISTS test_table; + Query Plan + -------------------------------------------------------------- + DROP TABLE IF EXISTS test_table + See Also -------- diff --git a/presto-docs/src/main/sphinx/sql/merge.rst b/presto-docs/src/main/sphinx/sql/merge.rst index b4b738dda86ab..74a0b6473a39b 100644 --- a/presto-docs/src/main/sphinx/sql/merge.rst +++ b/presto-docs/src/main/sphinx/sql/merge.rst @@ -26,7 +26,7 @@ In the ``MATCHED`` case, the ``UPDATE`` column value expressions can depend on a In the ``NOT MATCHED`` case, the ``INSERT`` expressions can depend on any field of the source. The ``MERGE`` command requires each target row to match at most one source row. An exception is raised when a single target table row matches more than one source row. -If a source row is not matched by the ``WHEN`` clause and there is no ``WHEN NOT MATCHED`` clause, the source row is ignored. +If a source row is not matched by the ``WHEN MATCHED`` clause and there is no ``WHEN NOT MATCHED`` clause, the source row is ignored. The ``MERGE`` statement is commonly used to integrate data from two tables with different contents but similar structures. For example, the source table could be part of a production transactional system, while the target table might be located in a data warehouse for analytics. diff --git a/presto-docs/src/main/sphinx/sql/select.rst b/presto-docs/src/main/sphinx/sql/select.rst index 549030f9ec48b..3dc5e48603b08 100644 --- a/presto-docs/src/main/sphinx/sql/select.rst +++ b/presto-docs/src/main/sphinx/sql/select.rst @@ -229,6 +229,11 @@ is equivalent to:: (destination_state), ()); +.. note:: + + ``CUBE`` supports at most 30 columns. This is because CUBE generates 2^n + grouping sets, and 2^30 (approximately 1 billion) is the practical limit. + .. code-block:: none origin_state | destination_state | _col0 diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index aec5e2d885830..a24aa6d0b591a 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -32,14 +32,23 @@ - org.hibernate + org.hibernate.validator hibernate-validator 8.0.3.Final org.glassfish jakarta.el - 4.0.1 + 5.0.0-M1 + + + at.yawk.lz4 + lz4-java + + + org.mozilla + rhino + 1.8.1 diff --git a/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java b/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java new file mode 100644 index 0000000000000..24d4fa716b6fe --- /dev/null +++ b/presto-druid/src/main/java/com/facebook/presto/druid/segment/PrestoQueryableIndex.java @@ -0,0 +1,170 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.druid.segment; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import org.apache.druid.collections.bitmap.BitmapFactory; +import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper; +import org.apache.druid.query.OrderBy; +import org.apache.druid.segment.DimensionHandler; +import org.apache.druid.segment.Metadata; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.BaseColumnHolder; +import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.data.Indexed; +import org.joda.time.Interval; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class PrestoQueryableIndex + implements QueryableIndex +{ + private final Interval dataInterval; + private final List columnNames; + private final Indexed availableDimensions; + private final BitmapFactory bitmapFactory; + private final Map> columns; + private final SmooshedFileMapper fileMapper; + @Nullable + private final Metadata metadata; + private final Supplier> dimensionHandlers; + + public PrestoQueryableIndex( + Interval dataInterval, + Indexed dimNames, + BitmapFactory bitmapFactory, + Map> columns, + SmooshedFileMapper fileMapper, + @Nullable Metadata metadata, + boolean lazy) + { + Preconditions.checkNotNull(columns.get(ColumnHolder.TIME_COLUMN_NAME)); + this.dataInterval = Preconditions.checkNotNull(dataInterval, "dataInterval"); + ImmutableList.Builder columnNamesBuilder = ImmutableList.builder(); + for (String column : columns.keySet()) { + if (!ColumnHolder.TIME_COLUMN_NAME.equals(column)) { + columnNamesBuilder.add(column); + } + } + this.columnNames = columnNamesBuilder.build(); + this.availableDimensions = dimNames; + this.bitmapFactory = bitmapFactory; + this.columns = columns; + this.fileMapper = fileMapper; + this.metadata = metadata; + + if (lazy) { + this.dimensionHandlers = Suppliers.memoize(() -> initDimensionHandlers(availableDimensions)); + } + else { + this.dimensionHandlers = () -> initDimensionHandlers(availableDimensions); + } + } + + @Override + public Interval getDataInterval() + { + return dataInterval; + } + + @Override + public int getNumRows() + { + return columns.get(ColumnHolder.TIME_COLUMN_NAME).get().getLength(); + } + + @Override + public List getColumnNames() + { + return columnNames; + } + + @Override + public Indexed getAvailableDimensions() + { + return availableDimensions; + } + + @Override + public BitmapFactory getBitmapFactoryForDimensions() + { + return bitmapFactory; + } + + @Nullable + @Override + public BaseColumnHolder getColumnHolder(String columnName) + { + Supplier columnHolderSupplier = (Supplier) this.columns.get(columnName); + return columnHolderSupplier == null ? null : (BaseColumnHolder) columnHolderSupplier.get(); + } + + @VisibleForTesting + public Map> getColumns() + { + return columns; + } + + @VisibleForTesting + public SmooshedFileMapper getFileMapper() + { + return fileMapper; + } + + @Override + public void close() + { + if (fileMapper != null) { + fileMapper.close(); + } + } + + @Override + public Metadata getMetadata() + { + return metadata; + } + + @Override + public List getOrdering() + { + return Collections.emptyList(); + } + + @Override + public Map getDimensionHandlers() + { + return dimensionHandlers.get(); + } + + private Map initDimensionHandlers(Indexed availableDimensions) + { + Map dimensionHandlerMap = Maps.newLinkedHashMap(); + for (String dim : availableDimensions) { + final ColumnHolder columnHolder = getColumnHolder(dim); + final DimensionHandler handler = columnHolder.getColumnFormat().getColumnHandler(dim); + dimensionHandlerMap.put(dim, handler); + } + return dimensionHandlerMap; + } +} diff --git a/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java b/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java index 23c1b5dcc4afd..f8e277dd313d7 100644 --- a/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java +++ b/presto-druid/src/main/java/com/facebook/presto/druid/segment/V9SegmentIndexSource.java @@ -22,13 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.common.collect.Streams; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.common.utils.SerializerUtils; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.segment.Metadata; import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.SimpleQueryableIndex; import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; @@ -66,7 +64,6 @@ public class V9SegmentIndexSource public V9SegmentIndexSource(SegmentColumnSource segmentColumnSource) { this.segmentColumnSource = requireNonNull(segmentColumnSource, "segmentColumnSource is null"); - NullHandling.initializeForTests(); } @Override @@ -74,10 +71,11 @@ public QueryableIndex loadIndex(List columnHandles) throws IOException { ByteBuffer indexBuffer = ByteBuffer.wrap(segmentColumnSource.getColumnData(INDEX_METADATA_FILE_NAME)); - GenericIndexed.read(indexBuffer, STRING_STRATEGY); + GenericIndexed.read(indexBuffer, STRING_STRATEGY, null); GenericIndexed allDimensions = GenericIndexed.read( indexBuffer, - STRING_STRATEGY); + STRING_STRATEGY, + null); Interval dataInterval = Intervals.utc(indexBuffer.getLong(), indexBuffer.getLong()); @@ -114,8 +112,13 @@ public QueryableIndex loadIndex(List columnHandles) columns.put(TIME_COLUMN_NAME, () -> createColumnHolder(TIME_COLUMN_NAME)); Indexed indexed = new ListIndexed<>(availableDimensions); - // TODO: get rid of the time column by creating Presto's SimpleQueryableIndex impl - return new SimpleQueryableIndex( + /* + * Druid 35.0.1 made SimpleQueryableIndex abstract, so created PrestoQueryableIndex + * based on the original implementation. + * TODO: Refactor PrestoQueryableIndex to remove the dependency on the __time column + * and implement a fully Presto-specific QueryableIndex. + */ + return new PrestoQueryableIndex( dataInterval, indexed, segmentBitmapSerdeFactory.getBitmapFactory(), @@ -136,7 +139,7 @@ private ColumnHolder createColumnHolder(String columnName) try { ByteBuffer columnData = ByteBuffer.wrap(segmentColumnSource.getColumnData(columnName)); ColumnDescriptor columnDescriptor = readColumnDescriptor(columnData); - return columnDescriptor.read(columnData, ColumnConfig.DEFAULT, null); + return columnDescriptor.read(columnData, ColumnConfig.DEFAULT, null, null); } catch (IOException e) { throw new PrestoException(DRUID_SEGMENT_LOAD_ERROR, e); diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index 3f12d301702bd..b5ec354e6d18a 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -247,7 +247,7 @@ org.testcontainers - elasticsearch + testcontainers-elasticsearch test diff --git a/presto-function-namespace-managers/pom.xml b/presto-function-namespace-managers/pom.xml index 2bc7e2c752283..1329c0554f9fb 100644 --- a/presto-function-namespace-managers/pom.xml +++ b/presto-function-namespace-managers/pom.xml @@ -189,6 +189,11 @@ drift-transport-netty + + io.netty + netty-buffer + + com.facebook.presto diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/mysql/MySqlFunctionNamespaceManagerFactory.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/mysql/MySqlFunctionNamespaceManagerFactory.java index 666d47ce13f7c..ab37fcd3d844f 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/mysql/MySqlFunctionNamespaceManagerFactory.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/mysql/MySqlFunctionNamespaceManagerFactory.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.function.SqlFunctionHandle; import com.google.inject.Injector; +import io.netty.buffer.PooledByteBufAllocator; import java.util.Map; @@ -51,7 +52,7 @@ public FunctionNamespaceManager create(String catalogName, Maplog + + com.facebook.airlift + stats + + com.google.errorprone error_prone_annotations @@ -97,6 +102,21 @@ jakarta.inject-api + + software.amazon.awssdk + metrics-spi + + + + software.amazon.awssdk + sdk-core + + + + org.weakref + jmxutils + + com.facebook.presto presto-hdfs-core diff --git a/presto-hive-common/src/main/java/com/facebook/presto/hive/aws/metrics/AwsSdkClientStats.java b/presto-hive-common/src/main/java/com/facebook/presto/hive/aws/metrics/AwsSdkClientStats.java new file mode 100644 index 0000000000000..5b7fd14ee9261 --- /dev/null +++ b/presto-hive-common/src/main/java/com/facebook/presto/hive/aws/metrics/AwsSdkClientStats.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.aws.metrics; + +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; +import com.google.errorprone.annotations.ThreadSafe; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; +import software.amazon.awssdk.metrics.MetricCollection; +import software.amazon.awssdk.metrics.MetricPublisher; + +import java.time.Duration; + +import static java.time.Duration.ZERO; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.THROTTLING; +import static software.amazon.awssdk.core.metrics.CoreMetric.API_CALL_DURATION; +import static software.amazon.awssdk.core.metrics.CoreMetric.BACKOFF_DELAY_DURATION; +import static software.amazon.awssdk.core.metrics.CoreMetric.ERROR_TYPE; +import static software.amazon.awssdk.core.metrics.CoreMetric.RETRY_COUNT; +import static software.amazon.awssdk.core.metrics.CoreMetric.SERVICE_CALL_DURATION; + +/** + * For reference on AWS SDK v2 Metrics: https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/metrics-list.html + * Metrics Publisher: https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/metrics.html + */ +@ThreadSafe +public final class AwsSdkClientStats +{ + private final CounterStat awsRequestCount = new CounterStat(); + private final CounterStat awsRetryCount = new CounterStat(); + private final CounterStat awsThrottleExceptions = new CounterStat(); + private final TimeStat awsServiceCallDuration = new TimeStat(MILLISECONDS); + private final TimeStat awsApiCallDuration = new TimeStat(MILLISECONDS); + private final TimeStat awsBackoffDelayDuration = new TimeStat(MILLISECONDS); + + @Managed + @Nested + public CounterStat getAwsRequestCount() + { + return awsRequestCount; + } + + @Managed + @Nested + public CounterStat getAwsRetryCount() + { + return awsRetryCount; + } + + @Managed + @Nested + public CounterStat getAwsThrottleExceptions() + { + return awsThrottleExceptions; + } + + @Managed + @Nested + public TimeStat getAwsServiceCallDuration() + { + return awsServiceCallDuration; + } + + @Managed + @Nested + public TimeStat getAwsApiCallDuration() + { + return awsApiCallDuration; + } + + @Managed + @Nested + public TimeStat getAwsBackoffDelayDuration() + { + return awsBackoffDelayDuration; + } + + public AwsSdkClientRequestMetricsPublisher newRequestMetricsPublisher() + { + return new AwsSdkClientRequestMetricsPublisher(this); + } + + public static class AwsSdkClientRequestMetricsPublisher + implements MetricPublisher + { + private final AwsSdkClientStats stats; + + protected AwsSdkClientRequestMetricsPublisher(AwsSdkClientStats stats) + { + this.stats = requireNonNull(stats, "stats is null"); + } + + @Override + public void publish(MetricCollection metricCollection) + { + long requestCount = metricCollection.metricValues(RETRY_COUNT) + .stream() + .map(i -> i + 1) + .reduce(Integer::sum).orElse(0); + + stats.awsRequestCount.update(requestCount); + + long retryCount = metricCollection.metricValues(RETRY_COUNT) + .stream() + .reduce(Integer::sum).orElse(0); + + stats.awsRetryCount.update(retryCount); + + long throttleExceptions = metricCollection + .childrenWithName("ApiCallAttempt") + .flatMap(mc -> mc.metricValues(ERROR_TYPE).stream()) + .filter(s -> s.equals(THROTTLING.toString())) + .count(); + + stats.awsThrottleExceptions.update(throttleExceptions); + + Duration serviceCallDuration = metricCollection + .childrenWithName("ApiCallAttempt") + .flatMap(mc -> mc.metricValues(SERVICE_CALL_DURATION).stream()) + .reduce(Duration::plus).orElse(ZERO); + + stats.awsServiceCallDuration.add(serviceCallDuration.toMillis(), MILLISECONDS); + + Duration apiCallDuration = metricCollection + .metricValues(API_CALL_DURATION) + .stream().reduce(Duration::plus).orElse(ZERO); + + stats.awsApiCallDuration.add(apiCallDuration.toMillis(), MILLISECONDS); + + Duration backoffDelayDuration = metricCollection + .childrenWithName("ApiCallAttempt") + .flatMap(mc -> mc.metricValues(BACKOFF_DELAY_DURATION).stream()) + .reduce(Duration::plus).orElse(ZERO); + + stats.awsBackoffDelayDuration.add(backoffDelayDuration.toMillis(), MILLISECONDS); + } + + @Override + public void close() + { + } + } +} diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 533261baae67e..72d97b91bea6e 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -124,13 +124,63 @@ - com.amazonaws - aws-java-sdk-glue + software.amazon.awssdk + auth - com.amazonaws - aws-java-sdk-sts + software.amazon.awssdk + aws-core + + + + software.amazon.awssdk + glue + + + + software.amazon.awssdk + http-client-spi + + + + software.amazon.awssdk + metrics-spi + + + + software.amazon.awssdk + netty-nio-client + + + + software.amazon.awssdk + regions + + + + software.amazon.awssdk + sdk-core + + + + software.amazon.awssdk + sts + + + + software.amazon.awssdk + utils + + + + software.amazon.awssdk + retries-spi + + + + software.amazon.awssdk + retries @@ -245,18 +295,6 @@ test - - org.slf4j - slf4j-jdk14 - test - - - - org.slf4j - jcl-over-slf4j - test - - org.assertj assertj-core diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java index a0aec5f817698..a58a1edfa5d09 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/MetastoreClientConfig.java @@ -15,28 +15,49 @@ import com.facebook.airlift.configuration.Config; import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.configuration.LegacyConfig; import com.facebook.airlift.units.Duration; import com.facebook.airlift.units.MinDuration; import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; +import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; +import com.google.inject.ConfigurationException; +import com.google.inject.spi.Message; +import jakarta.annotation.PostConstruct; import jakarta.validation.constraints.DecimalMax; import jakarta.validation.constraints.DecimalMin; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Iterables.transform; +import static java.util.Locale.ENGLISH; import static java.util.concurrent.TimeUnit.MINUTES; public class MetastoreClientConfig { + private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + private HostAndPort metastoreSocksProxy; private Duration metastoreTimeout = new Duration(10, TimeUnit.SECONDS); private boolean verifyChecksum = true; private boolean requireHadoopNative = true; - private Duration metastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); - private Duration metastoreRefreshInterval = new Duration(0, TimeUnit.SECONDS); + private Set enabledCaches = ImmutableSet.of(); + private Set disabledCaches = ImmutableSet.of(); + private Duration defaultMetastoreCacheTtl = new Duration(0, TimeUnit.SECONDS); + private Map metastoreCacheTtlByType = ImmutableMap.of(); + private Duration defaultMetastoreCacheRefreshInterval = new Duration(0, TimeUnit.SECONDS); + private Map metastoreCacheRefreshIntervalByType = ImmutableMap.of(); private long metastoreCacheMaximumSize = 10000; private long perTransactionMetastoreCacheMaximumSize = 1000; private int maxMetastoreRefreshThreads = 100; @@ -90,31 +111,135 @@ public MetastoreClientConfig setVerifyChecksum(boolean verifyChecksum) return this; } + public Set getEnabledCaches() + { + return enabledCaches; + } + + @Config("hive.metastore.cache.enabled-caches") + @ConfigDescription("Comma-separated list of metastore cache types to enable") + public MetastoreClientConfig setEnabledCaches(String caches) + { + if (caches == null) { + this.enabledCaches = ImmutableSet.of(); + return this; + } + + this.enabledCaches = ImmutableSet.copyOf(transform( + SPLITTER.split(caches), + cache -> MetastoreCacheType.valueOf(cache.toUpperCase(ENGLISH)))); + return this; + } + + public Set getDisabledCaches() + { + return disabledCaches; + } + + @Config("hive.metastore.cache.disabled-caches") + @ConfigDescription("Comma-separated list of metastore cache types to disable") + public MetastoreClientConfig setDisabledCaches(String caches) + { + if (caches == null) { + this.disabledCaches = ImmutableSet.of(); + return this; + } + + this.disabledCaches = ImmutableSet.copyOf(transform( + SPLITTER.split(caches), + cache -> MetastoreCacheType.valueOf(cache.toUpperCase(ENGLISH)))); + return this; + } + + @PostConstruct + public void isBothEnabledAndDisabledConfigured() + { + if (!getEnabledCaches().isEmpty() && !getDisabledCaches().isEmpty()) { + throw new ConfigurationException(ImmutableList.of(new Message("Only one of 'hive.metastore.cache.enabled-caches' or 'hive.metastore.cache.disabled-caches' can be set. " + + "These configs are mutually exclusive."))); + } + } + @NotNull - public Duration getMetastoreCacheTtl() + public Duration getDefaultMetastoreCacheTtl() { - return metastoreCacheTtl; + return defaultMetastoreCacheTtl; } @MinDuration("0ms") - @Config("hive.metastore-cache-ttl") - public MetastoreClientConfig setMetastoreCacheTtl(Duration metastoreCacheTtl) + @Config("hive.metastore.cache.ttl.default") + @ConfigDescription("Default time-to-live for Hive metastore cache entries. " + + "It is used when no per-cache TTL override is configured. " + + "TTL of 0ms would mean cache is disabled.") + @LegacyConfig("hive.metastore-cache-ttl") + public MetastoreClientConfig setDefaultMetastoreCacheTtl(Duration defaultMetastoreCacheTtl) + { + this.defaultMetastoreCacheTtl = defaultMetastoreCacheTtl; + return this; + } + + public Map getMetastoreCacheTtlByType() { - this.metastoreCacheTtl = metastoreCacheTtl; + return metastoreCacheTtlByType; + } + + @Config("hive.metastore.cache.ttl-by-type") + @ConfigDescription("Per-cache time-to-live (TTL) overrides for Hive metastore caches.\n" + + "The value is a comma-separated list of : pairs.") + public MetastoreClientConfig setMetastoreCacheTtlByType(String metastoreCacheTtlByTypeValues) + { + if (metastoreCacheTtlByTypeValues == null || metastoreCacheTtlByTypeValues.isEmpty()) { + return this; + } + + this.metastoreCacheTtlByType = Arrays.stream(metastoreCacheTtlByTypeValues.split(",")) + .map(entry -> entry.split(":")) + .filter(parts -> parts.length == 2) + .collect(toImmutableMap( + parts -> MetastoreCacheType.valueOf(parts[0].trim().toUpperCase(ENGLISH)), + parts -> Duration.valueOf(parts[1].trim()))); + return this; } @NotNull - public Duration getMetastoreRefreshInterval() + public Duration getDefaultMetastoreCacheRefreshInterval() { - return metastoreRefreshInterval; + return defaultMetastoreCacheRefreshInterval; } @MinDuration("1ms") - @Config("hive.metastore-refresh-interval") - public MetastoreClientConfig setMetastoreRefreshInterval(Duration metastoreRefreshInterval) + @Config("hive.metastore.cache.refresh-interval.default") + @ConfigDescription("Default refresh interval for Hive metastore cache entries.\n" + + "Controls how often cached values are asynchronously refreshed.") + @LegacyConfig("hive.metastore-refresh-interval") + public MetastoreClientConfig setDefaultMetastoreCacheRefreshInterval(Duration defaultMetastoreCacheRefreshInterval) { - this.metastoreRefreshInterval = metastoreRefreshInterval; + this.defaultMetastoreCacheRefreshInterval = defaultMetastoreCacheRefreshInterval; + return this; + } + + public Map getMetastoreCacheRefreshIntervalByType() + { + return metastoreCacheRefreshIntervalByType; + } + + @Config("hive.metastore.cache.refresh-interval-by-type") + @ConfigDescription("Per-cache refresh interval overrides for Hive metastore caches.\n" + + "The value is a comma-separated list of : pairs.") + public MetastoreClientConfig setMetastoreCacheRefreshIntervalByType(String metastoreCacheRefreshIntervalByTypeValues) + { + if (metastoreCacheRefreshIntervalByTypeValues == null || metastoreCacheRefreshIntervalByTypeValues.isEmpty()) { + return this; + } + + this.metastoreCacheRefreshIntervalByType = Arrays.stream(metastoreCacheRefreshIntervalByTypeValues.split(",")) + .map(entry -> entry.split(":")) + .filter(parts -> parts.length == 2) + .collect(toImmutableMap( + parts -> MetastoreCacheType.valueOf(parts[0].trim().toUpperCase(ENGLISH)), + parts -> Duration.valueOf(parts[1].trim()))); + return this; } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/AbstractCachingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/AbstractCachingHiveMetastore.java index ef2feeec481b8..1f1bb61723889 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/AbstractCachingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/AbstractCachingHiveMetastore.java @@ -39,6 +39,25 @@ public enum MetastoreCacheScope ALL, PARTITION } + public enum MetastoreCacheType + { + ALL, + DATABASE, + DATABASE_NAMES, + TABLE, + TABLE_NAMES, + TABLE_STATISTICS, + TABLE_CONSTRAINTS, + PARTITION, + PARTITION_STATISTICS, + PARTITION_FILTER, + PARTITION_NAMES, + VIEW_NAMES, + TABLE_PRIVILEGES, + ROLES, + ROLE_GRANTS + } + public abstract ExtendedHiveMetastore getDelegate(); @Override diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/InMemoryCachingHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/InMemoryCachingHiveMetastore.java index fdf142b4a25ef..a4388a501a9e3 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/InMemoryCachingHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/InMemoryCachingHiveMetastore.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.hive.metastore; -import com.facebook.airlift.units.Duration; import com.facebook.presto.common.predicate.Domain; import com.facebook.presto.hive.ForCachingHiveMetastore; import com.facebook.presto.hive.HiveTableHandle; @@ -53,7 +52,20 @@ import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_PARTITION_CACHE; import static com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY; -import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope.ALL; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.DATABASE; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.DATABASE_NAMES; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION_FILTER; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION_NAMES; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION_STATISTICS; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ROLES; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ROLE_GRANTS; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE_CONSTRAINTS; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE_NAMES; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE_PRIVILEGES; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE_STATISTICS; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.VIEW_NAMES; import static com.facebook.presto.hive.metastore.HivePartitionName.hivePartitionName; import static com.facebook.presto.hive.metastore.HiveTableName.hiveTableName; import static com.facebook.presto.hive.metastore.NoopMetastoreCacheStats.NOOP_METASTORE_CACHE_STATS; @@ -95,6 +107,7 @@ public class InMemoryCachingHiveMetastore private final LoadingCache, Set> rolesCache; private final LoadingCache, Set> roleGrantsCache; private final MetastoreCacheStats metastoreCacheStats; + private final MetastoreCacheSpecProvider metastoreCacheSpecProvider; private final boolean metastoreImpersonationEnabled; private final boolean partitionVersioningEnabled; @@ -106,47 +119,43 @@ public InMemoryCachingHiveMetastore( @ForCachingHiveMetastore ExtendedHiveMetastore delegate, @ForCachingHiveMetastore ExecutorService executor, MetastoreCacheStats metastoreCacheStats, - MetastoreClientConfig metastoreClientConfig) + MetastoreClientConfig metastoreClientConfig, + MetastoreCacheSpecProvider metastoreCacheSpecProvider) { this( delegate, executor, metastoreClientConfig.isMetastoreImpersonationEnabled(), - metastoreClientConfig.getMetastoreCacheTtl(), - metastoreClientConfig.getMetastoreRefreshInterval(), metastoreClientConfig.getMetastoreCacheMaximumSize(), metastoreClientConfig.isPartitionVersioningEnabled(), - metastoreClientConfig.getMetastoreCacheScope(), metastoreClientConfig.getPartitionCacheValidationPercentage(), metastoreClientConfig.getPartitionCacheColumnCountLimit(), - metastoreCacheStats); + metastoreCacheStats, + metastoreCacheSpecProvider); } public InMemoryCachingHiveMetastore( ExtendedHiveMetastore delegate, ExecutorService executor, boolean metastoreImpersonationEnabled, - Duration cacheTtl, - Duration refreshInterval, long maximumSize, boolean partitionVersioningEnabled, - MetastoreCacheScope metastoreCacheScope, double partitionCacheValidationPercentage, int partitionCacheColumnCountLimit, - MetastoreCacheStats metastoreCacheStats) + MetastoreCacheStats metastoreCacheStats, + MetastoreCacheSpecProvider metastoreCacheSpecProvider) { this( delegate, executor, metastoreImpersonationEnabled, - OptionalLong.of(cacheTtl.toMillis()), - refreshInterval.toMillis() >= cacheTtl.toMillis() ? OptionalLong.empty() : OptionalLong.of(refreshInterval.toMillis()), maximumSize, partitionVersioningEnabled, - metastoreCacheScope, partitionCacheValidationPercentage, partitionCacheColumnCountLimit, - metastoreCacheStats); + metastoreCacheStats, + Optional.of(metastoreCacheSpecProvider), + false); } public static InMemoryCachingHiveMetastore memoizeMetastore(ExtendedHiveMetastore delegate, boolean isMetastoreImpersonationEnabled, long maximumSize, int partitionCacheMaxColumnCount) @@ -155,28 +164,26 @@ public static InMemoryCachingHiveMetastore memoizeMetastore(ExtendedHiveMetastor delegate, newDirectExecutorService(), isMetastoreImpersonationEnabled, - OptionalLong.empty(), - OptionalLong.empty(), maximumSize, false, - ALL, 0.0, partitionCacheMaxColumnCount, - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + Optional.empty(), + true); } private InMemoryCachingHiveMetastore( ExtendedHiveMetastore delegate, ExecutorService executor, boolean metastoreImpersonationEnabled, - OptionalLong expiresAfterWriteMillis, - OptionalLong refreshMills, long maximumSize, boolean partitionVersioningEnabled, - MetastoreCacheScope metastoreCacheScope, double partitionCacheValidationPercentage, int partitionCacheColumnCountLimit, - MetastoreCacheStats metastoreCacheStats) + MetastoreCacheStats metastoreCacheStats, + Optional metastoreCacheSpecProvider, + boolean perTransactionCache) { this.delegate = requireNonNull(delegate, "delegate is null"); requireNonNull(executor, "executor is null"); @@ -185,59 +192,40 @@ private InMemoryCachingHiveMetastore( this.partitionCacheValidationPercentage = partitionCacheValidationPercentage; this.partitionCacheColumnCountLimit = partitionCacheColumnCountLimit; this.metastoreCacheStats = metastoreCacheStats; + this.metastoreCacheSpecProvider = metastoreCacheSpecProvider.orElse(null); - OptionalLong cacheExpiresAfterWriteMillis; - OptionalLong cacheRefreshMills; - long cacheMaxSize; - - OptionalLong partitionCacheExpiresAfterWriteMillis; - OptionalLong partitionCacheRefreshMills; - long partitionCacheMaxSize; - - switch (metastoreCacheScope) { - case PARTITION: - partitionCacheExpiresAfterWriteMillis = expiresAfterWriteMillis; - partitionCacheRefreshMills = refreshMills; - partitionCacheMaxSize = maximumSize; - cacheExpiresAfterWriteMillis = OptionalLong.of(0); - cacheRefreshMills = OptionalLong.of(0); - cacheMaxSize = 0; - break; - - case ALL: - partitionCacheExpiresAfterWriteMillis = expiresAfterWriteMillis; - partitionCacheRefreshMills = refreshMills; - partitionCacheMaxSize = maximumSize; - cacheExpiresAfterWriteMillis = expiresAfterWriteMillis; - cacheRefreshMills = refreshMills; - cacheMaxSize = maximumSize; - break; - - default: - throw new IllegalArgumentException("Unknown metastore-cache-scope: " + metastoreCacheScope); - } - - databaseNamesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadAllDatabases), executor)); + databaseNamesCache = buildCache( + executor, + DATABASE_NAMES, + CacheLoader.from(this::loadAllDatabases), + perTransactionCache, + maximumSize); - databaseCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadDatabase), executor)); + databaseCache = buildCache( + executor, + DATABASE, + CacheLoader.from(this::loadDatabase), + perTransactionCache, + maximumSize); - tableNamesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadAllTables), executor)); + tableNamesCache = buildCache( + executor, + TABLE_NAMES, + CacheLoader.from(this::loadAllTables), + perTransactionCache, + maximumSize); - tableStatisticsCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(new CacheLoader, PartitionStatistics>() - { - @Override - public PartitionStatistics load(KeyAndContext key) - { - return loadTableColumnStatistics(key); - } - }, executor)); + tableStatisticsCache = buildCache( + executor, + TABLE_STATISTICS, + CacheLoader.from(this::loadTableColumnStatistics), + perTransactionCache, + maximumSize); - partitionStatisticsCache = newCacheBuilder(partitionCacheExpiresAfterWriteMillis, partitionCacheRefreshMills, partitionCacheMaxSize) - .build(asyncReloading(new CacheLoader, PartitionStatistics>() + partitionStatisticsCache = buildCache( + executor, + PARTITION_STATISTICS, + new CacheLoader, PartitionStatistics>() { @Override public PartitionStatistics load(KeyAndContext key) @@ -250,27 +238,51 @@ public Map, PartitionStatistics> loadAll(Iterab { return loadPartitionColumnStatistics(keys); } - }, executor)); + }, + perTransactionCache, + maximumSize); - tableCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadTable), executor)); + tableCache = buildCache( + executor, + TABLE, + CacheLoader.from(this::loadTable), + perTransactionCache, + maximumSize); metastoreCacheStats.setTableCache(tableCache); - tableConstraintsCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadTableConstraints), executor)); + tableConstraintsCache = buildCache( + executor, + TABLE_CONSTRAINTS, + CacheLoader.from(this::loadTableConstraints), + perTransactionCache, + maximumSize); - viewNamesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadAllViews), executor)); + viewNamesCache = buildCache( + executor, + VIEW_NAMES, + CacheLoader.from(this::loadAllViews), + perTransactionCache, + maximumSize); - partitionNamesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadPartitionNames), executor)); + partitionNamesCache = buildCache( + executor, + PARTITION_NAMES, + CacheLoader.from(this::loadPartitionNames), + perTransactionCache, + maximumSize); - partitionFilterCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadPartitionNamesByFilter), executor)); + partitionFilterCache = buildCache( + executor, + PARTITION_FILTER, + CacheLoader.from(this::loadPartitionNamesByFilter), + perTransactionCache, + maximumSize); metastoreCacheStats.setPartitionNamesCache(partitionFilterCache); - partitionCache = newCacheBuilder(partitionCacheExpiresAfterWriteMillis, partitionCacheRefreshMills, partitionCacheMaxSize) - .build(asyncReloading(new CacheLoader, Optional>() + partitionCache = buildCache( + executor, + PARTITION, + new CacheLoader, Optional>() { @Override public Optional load(KeyAndContext partitionName) @@ -283,17 +295,31 @@ public Map, Optional> loadAll(Iterab { return loadPartitionsByNames(partitionNames); } - }, executor)); + }, + perTransactionCache, + maximumSize); metastoreCacheStats.setPartitionCache(partitionCache); - tablePrivilegesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadTablePrivileges), executor)); + tablePrivilegesCache = buildCache( + executor, + TABLE_PRIVILEGES, + CacheLoader.from(this::loadTablePrivileges), + perTransactionCache, + maximumSize); - rolesCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadAllRoles), executor)); + rolesCache = buildCache( + executor, + ROLES, + CacheLoader.from(this::loadAllRoles), + perTransactionCache, + maximumSize); - roleGrantsCache = newCacheBuilder(cacheExpiresAfterWriteMillis, cacheRefreshMills, cacheMaxSize) - .build(asyncReloading(CacheLoader.from(this::loadRoleGrants), executor)); + roleGrantsCache = buildCache( + executor, + ROLE_GRANTS, + CacheLoader.from(this::loadRoleGrants), + perTransactionCache, + maximumSize); } @Override @@ -611,30 +637,6 @@ public List getPartitionNamesByFilter( return get(partitionFilterCache, getCachingKey(metastoreContext, partitionFilter(databaseName, tableName, partitionPredicates))); } - private void invalidateStalePartitions( - List partitionNamesWithVersion, - String databaseName, - String tableName, - MetastoreContext metastoreContext) - { - for (PartitionNameWithVersion partitionNameWithVersion : partitionNamesWithVersion) { - HivePartitionName hivePartitionName = hivePartitionName(databaseName, tableName, partitionNameWithVersion.getPartitionName()); - KeyAndContext partitionNameKey = getCachingKey(metastoreContext, hivePartitionName); - Optional partition = partitionCache.getIfPresent(partitionNameKey); - if (partition == null || !partition.isPresent()) { - partitionCache.invalidate(partitionNameKey); - partitionStatisticsCache.invalidate(partitionNameKey); - } - else { - Optional partitionVersion = partition.get().getPartitionVersion(); - if (!partitionVersion.isPresent() || !partitionVersion.equals(partitionNameWithVersion.getPartitionVersion())) { - partitionCache.invalidate(partitionNameKey); - partitionStatisticsCache.invalidate(partitionNameKey); - } - } - } - } - private void invalidatePartitionsWithHighColumnCount(Optional partition, KeyAndContext partitionCacheKey) { // Do NOT cache partitions with # of columns > partitionCacheColumnLimit @@ -1088,4 +1090,30 @@ private static CacheBuilder newCacheBuilder(OptionalLong expires } return cacheBuilder.maximumSize(maximumSize).recordStats(); } + + private LoadingCache buildCache( + ExecutorService executor, + MetastoreCacheType cacheType, + CacheLoader loader, + boolean isPerTransactionCache, + long maximumSize) + { + if (isPerTransactionCache) { + return newCacheBuilder( + OptionalLong.empty(), + OptionalLong.empty(), + maximumSize) + .build(asyncReloading(loader, executor)); + } + + MetastoreCacheSpec spec = metastoreCacheSpecProvider.getMetastoreCacheSpec(cacheType); + long cacheTtlMillis = spec.getCacheTtlMillis(); + long refreshMillis = spec.getRefreshIntervalMillis(); + + return newCacheBuilder( + OptionalLong.of(cacheTtlMillis), + refreshMillis >= cacheTtlMillis ? OptionalLong.empty() : OptionalLong.of(refreshMillis), + spec.getMaximumSize()) + .build(asyncReloading(loader, executor)); + } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpec.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpec.java new file mode 100644 index 0000000000000..c690d76109e0d --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpec.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +public class MetastoreCacheSpec +{ + private static final MetastoreCacheSpec DISABLED = new MetastoreCacheSpec(0, 0, 0); + private final long cacheTtlMillis; + private final long refreshIntervalMillis; + private final long maximumSize; + + public static MetastoreCacheSpec disabled() + { + return DISABLED; + } + + public static MetastoreCacheSpec enabled(long cacheTtlMillis, long refreshIntervalMillis, long maximumSize) + { + return new MetastoreCacheSpec(cacheTtlMillis, refreshIntervalMillis, maximumSize); + } + + private MetastoreCacheSpec(long cacheTtlMillis, long refreshIntervalMillis, long maximumSize) + { + this.cacheTtlMillis = cacheTtlMillis; + this.refreshIntervalMillis = refreshIntervalMillis; + this.maximumSize = maximumSize; + } + + public long getCacheTtlMillis() + { + return cacheTtlMillis; + } + + public long getRefreshIntervalMillis() + { + return refreshIntervalMillis; + } + + public long getMaximumSize() + { + return maximumSize; + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpecProvider.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpecProvider.java new file mode 100644 index 0000000000000..1b645c7870917 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/MetastoreCacheSpecProvider.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.hive.metastore; + +import com.facebook.presto.hive.MetastoreClientConfig; +import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType; +import jakarta.inject.Inject; + +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ALL; +import static java.util.Objects.requireNonNull; + +public class MetastoreCacheSpecProvider +{ + private final MetastoreClientConfig clientConfig; + + @Inject + public MetastoreCacheSpecProvider(MetastoreClientConfig clientConfig) + { + this.clientConfig = requireNonNull(clientConfig, "clientConfig is null"); + } + + public MetastoreCacheSpec getMetastoreCacheSpec(MetastoreCacheType type) + { + boolean enabled = isEnabled(type); + if (!enabled) { + return MetastoreCacheSpec.disabled(); + } + + long cacheTtlMillis = clientConfig.getMetastoreCacheTtlByType().getOrDefault( + type, clientConfig.getDefaultMetastoreCacheTtl()).toMillis(); + long refreshIntervalMillis = clientConfig.getMetastoreCacheRefreshIntervalByType().getOrDefault( + type, clientConfig.getDefaultMetastoreCacheRefreshInterval()).toMillis(); + + return MetastoreCacheSpec.enabled( + cacheTtlMillis, + refreshIntervalMillis, + clientConfig.getMetastoreCacheMaximumSize()); + } + + private boolean isEnabled(MetastoreCacheType type) + { + if (!clientConfig.getEnabledCaches().isEmpty()) { + return clientConfig.getEnabledCaches().contains(type) || clientConfig.getEnabledCaches().contains(ALL); + } + if (!clientConfig.getDisabledCaches().isEmpty()) { + return !(clientConfig.getDisabledCaches().contains(type) || clientConfig.getDisabledCaches().contains(ALL)); + } + + return isEnabledByLegacyMetastoreScope(type); + } + + private boolean isEnabledByLegacyMetastoreScope(MetastoreCacheType type) + { + switch (clientConfig.getMetastoreCacheScope()) { + case ALL: + return true; + case PARTITION: + return type == MetastoreCacheType.PARTITION || type == MetastoreCacheType.PARTITION_STATISTICS; + default: + return false; + } + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java index c37a22af47b7f..5ae8fd5cb7fb3 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/FileMetastoreModule.java @@ -18,6 +18,7 @@ import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.google.inject.Binder; import com.google.inject.Scopes; @@ -42,6 +43,7 @@ public void setup(Binder binder) { checkArgument(buildConfigObject(HiveCommonClientConfig.class).getCatalogName() == null, "'hive.metastore.catalog.name' should not be set for file metastore"); configBinder(binder).bindConfig(FileHiveMetastoreConfig.class); + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(FileHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(ExtendedHiveMetastore.class) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueCatalogApiStats.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueCatalogApiStats.java index c35b852c38554..0e276a8234720 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueCatalogApiStats.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueCatalogApiStats.java @@ -13,8 +13,6 @@ */ package com.facebook.presto.hive.metastore.glue; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.handlers.AsyncHandler; import com.facebook.airlift.stats.CounterStat; import com.facebook.airlift.stats.TimeStat; import com.google.errorprone.annotations.ThreadSafe; @@ -24,6 +22,7 @@ import java.util.function.Supplier; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; @ThreadSafe public class GlueCatalogApiStats @@ -53,23 +52,12 @@ public void record(Runnable action) } } - public AsyncHandler metricsAsyncHandler() + public void recordAsync(long executionTimeNanos, boolean failed) { - return new AsyncHandler() { - private final TimeStat.BlockTimer timer = time.time(); - @Override - public void onError(Exception exception) - { - timer.close(); - recordException(exception); - } - - @Override - public void onSuccess(R request, T result) - { - timer.close(); - } - }; + time.add(executionTimeNanos, NANOSECONDS); + if (failed) { + totalFailures.update(1); + } } @Managed diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java index ef782cd889fac..da980eef676f5 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastore.java @@ -13,51 +13,6 @@ */ package com.facebook.presto.hive.metastore.glue; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.metrics.RequestMetricCollector; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.glue.AWSGlueAsync; -import com.amazonaws.services.glue.AWSGlueAsyncClientBuilder; -import com.amazonaws.services.glue.model.AlreadyExistsException; -import com.amazonaws.services.glue.model.BatchCreatePartitionRequest; -import com.amazonaws.services.glue.model.BatchCreatePartitionResult; -import com.amazonaws.services.glue.model.BatchGetPartitionRequest; -import com.amazonaws.services.glue.model.BatchGetPartitionResult; -import com.amazonaws.services.glue.model.CreateDatabaseRequest; -import com.amazonaws.services.glue.model.CreateTableRequest; -import com.amazonaws.services.glue.model.DatabaseInput; -import com.amazonaws.services.glue.model.DeleteDatabaseRequest; -import com.amazonaws.services.glue.model.DeletePartitionRequest; -import com.amazonaws.services.glue.model.DeleteTableRequest; -import com.amazonaws.services.glue.model.EntityNotFoundException; -import com.amazonaws.services.glue.model.ErrorDetail; -import com.amazonaws.services.glue.model.GetDatabaseRequest; -import com.amazonaws.services.glue.model.GetDatabaseResult; -import com.amazonaws.services.glue.model.GetDatabasesRequest; -import com.amazonaws.services.glue.model.GetDatabasesResult; -import com.amazonaws.services.glue.model.GetPartitionRequest; -import com.amazonaws.services.glue.model.GetPartitionResult; -import com.amazonaws.services.glue.model.GetPartitionsRequest; -import com.amazonaws.services.glue.model.GetPartitionsResult; -import com.amazonaws.services.glue.model.GetTableRequest; -import com.amazonaws.services.glue.model.GetTableResult; -import com.amazonaws.services.glue.model.GetTablesRequest; -import com.amazonaws.services.glue.model.GetTablesResult; -import com.amazonaws.services.glue.model.PartitionError; -import com.amazonaws.services.glue.model.PartitionInput; -import com.amazonaws.services.glue.model.PartitionValueList; -import com.amazonaws.services.glue.model.Segment; -import com.amazonaws.services.glue.model.TableInput; -import com.amazonaws.services.glue.model.UpdateDatabaseRequest; -import com.amazonaws.services.glue.model.UpdatePartitionRequest; -import com.amazonaws.services.glue.model.UpdateTableRequest; import com.facebook.airlift.units.Duration; import com.facebook.presto.common.predicate.Domain; import com.facebook.presto.common.type.Type; @@ -93,6 +48,7 @@ import com.facebook.presto.spi.security.PrestoPrincipal; import com.facebook.presto.spi.security.RoleGrant; import com.facebook.presto.spi.statistics.ColumnStatisticType; +import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -102,7 +58,59 @@ import org.apache.hadoop.fs.Path; import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; - +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.awscore.retry.AwsRetryStrategy; +import software.amazon.awssdk.core.async.SdkPublisher; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.metrics.MetricPublisher; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.services.glue.GlueAsyncClient; +import software.amazon.awssdk.services.glue.GlueAsyncClientBuilder; +import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.BatchCreatePartitionRequest; +import software.amazon.awssdk.services.glue.model.BatchCreatePartitionResponse; +import software.amazon.awssdk.services.glue.model.BatchGetPartitionRequest; +import software.amazon.awssdk.services.glue.model.BatchGetPartitionResponse; +import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest; +import software.amazon.awssdk.services.glue.model.CreateTableRequest; +import software.amazon.awssdk.services.glue.model.DatabaseInput; +import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest; +import software.amazon.awssdk.services.glue.model.DeletePartitionRequest; +import software.amazon.awssdk.services.glue.model.DeleteTableRequest; +import software.amazon.awssdk.services.glue.model.EntityNotFoundException; +import software.amazon.awssdk.services.glue.model.ErrorDetail; +import software.amazon.awssdk.services.glue.model.GetDatabaseRequest; +import software.amazon.awssdk.services.glue.model.GetDatabaseResponse; +import software.amazon.awssdk.services.glue.model.GetDatabasesRequest; +import software.amazon.awssdk.services.glue.model.GetPartitionRequest; +import software.amazon.awssdk.services.glue.model.GetPartitionResponse; +import software.amazon.awssdk.services.glue.model.GetPartitionsRequest; +import software.amazon.awssdk.services.glue.model.GetTableRequest; +import software.amazon.awssdk.services.glue.model.GetTableResponse; +import software.amazon.awssdk.services.glue.model.GetTablesRequest; +import software.amazon.awssdk.services.glue.model.GlueException; +import software.amazon.awssdk.services.glue.model.GlueResponse; +import software.amazon.awssdk.services.glue.model.PartitionError; +import software.amazon.awssdk.services.glue.model.PartitionInput; +import software.amazon.awssdk.services.glue.model.PartitionValueList; +import software.amazon.awssdk.services.glue.model.Segment; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.TableInput; +import software.amazon.awssdk.services.glue.model.UpdateDatabaseRequest; +import software.amazon.awssdk.services.glue.model.UpdatePartitionRequest; +import software.amazon.awssdk.services.glue.model.UpdateTableRequest; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +import java.net.URI; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -110,11 +118,14 @@ import java.util.Map.Entry; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.Future; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -143,6 +154,7 @@ import static com.google.common.collect.Comparators.lexicographical; import static java.util.Comparator.comparing; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.function.UnaryOperator.identity; import static java.util.stream.Collectors.toMap; @@ -177,7 +189,7 @@ public class GlueHiveMetastore private final GlueMetastoreStats stats = new GlueMetastoreStats(); private final HdfsEnvironment hdfsEnvironment; private final HdfsContext hdfsContext; - private final AWSGlueAsync glueClient; + private final GlueAsyncClient glueClient; private final Optional defaultDir; private final String catalogId; private final int partitionSegments; @@ -191,51 +203,71 @@ public GlueHiveMetastore( { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.hdfsContext = new HdfsContext(new ConnectorIdentity(DEFAULT_METASTORE_USER, Optional.empty(), Optional.empty())); - this.glueClient = createAsyncGlueClient(requireNonNull(glueConfig, "glueConfig is null"), stats.newRequestMetricsCollector()); + this.glueClient = createAsyncGlueClient(requireNonNull(glueConfig, "glueConfig is null"), stats.newRequestMetricPublisher()); this.defaultDir = glueConfig.getDefaultWarehouseDir(); this.catalogId = glueConfig.getCatalogId().orElse(null); this.partitionSegments = glueConfig.getPartitionSegments(); this.executor = requireNonNull(executor, "executor is null"); } - private static AWSGlueAsync createAsyncGlueClient(GlueHiveMetastoreConfig config, RequestMetricCollector metricsCollector) + private static GlueAsyncClient createAsyncGlueClient(GlueHiveMetastoreConfig config, MetricPublisher metricPublisher) { - ClientConfiguration clientConfig = new ClientConfiguration() - .withMaxConnections(config.getMaxGlueConnections()) - .withMaxErrorRetry(config.getMaxGlueErrorRetries()); - AWSGlueAsyncClientBuilder asyncGlueClientBuilder = AWSGlueAsyncClientBuilder.standard() - .withMetricsCollector(metricsCollector) - .withClientConfiguration(clientConfig); + NettyNioAsyncHttpClient.Builder nettyBuilder = NettyNioAsyncHttpClient.builder() + .maxConcurrency(config.getMaxGlueConnections()); + + StandardRetryStrategy strategy = AwsRetryStrategy.standardRetryStrategy() + .toBuilder() + .maxAttempts(config.getMaxGlueErrorRetries()) + .build(); + + ClientOverrideConfiguration.Builder overrideConfigBuilder = ClientOverrideConfiguration.builder() + .retryStrategy(strategy) + .addMetricPublisher(metricPublisher); + + GlueAsyncClientBuilder glueAsyncClientBuilder = GlueAsyncClient.builder() + .httpClientBuilder(nettyBuilder) + .overrideConfiguration(overrideConfigBuilder.build()); if (config.getGlueEndpointUrl().isPresent()) { checkArgument(config.getGlueRegion().isPresent(), "Glue region must be set when Glue endpoint URL is set"); - asyncGlueClientBuilder.setEndpointConfiguration(new EndpointConfiguration( - config.getGlueEndpointUrl().get(), - config.getGlueRegion().get())); + glueAsyncClientBuilder + .endpointOverride(URI.create(config.getGlueEndpointUrl().get())) + .region(Region.of(config.getGlueRegion().get())); } else if (config.getGlueRegion().isPresent()) { - asyncGlueClientBuilder.setRegion(config.getGlueRegion().get()); - } - else if (config.getPinGlueClientToCurrentRegion()) { - Region currentRegion = Regions.getCurrentRegion(); - if (currentRegion != null) { - asyncGlueClientBuilder.setRegion(currentRegion.getName()); - } + glueAsyncClientBuilder.region(Region.of(config.getGlueRegion().get())); } + AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create(); if (config.getAwsAccessKey().isPresent() && config.getAwsSecretKey().isPresent()) { - AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider( - new BasicAWSCredentials(config.getAwsAccessKey().get(), config.getAwsSecretKey().get())); - asyncGlueClientBuilder.setCredentials(credentialsProvider); + credentialsProvider = StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.getAwsAccessKey().get(), config.getAwsSecretKey().get())); } else if (config.getIamRole().isPresent()) { - AWSCredentialsProvider credentialsProvider = new STSAssumeRoleSessionCredentialsProvider - .Builder(config.getIamRole().get(), "roleSessionName") + StsClientBuilder stsClientBuilder = StsClient.builder() + .credentialsProvider(DefaultCredentialsProvider.create()); + + if (config.getGlueStsEndpointUrl().isPresent()) { + checkArgument(config.getGlueStsRegion().isPresent(), "Glue STS region must be set when Glue STS endpoint URL is set"); + stsClientBuilder + .endpointOverride(URI.create(config.getGlueStsEndpointUrl().get())) + .region(Region.of(config.getGlueStsRegion().get())); + } + else if (config.getGlueStsRegion().isPresent()) { + stsClientBuilder.region(Region.of(config.getGlueStsRegion().get())); + } + + credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(() -> AssumeRoleRequest.builder() + .roleArn(config.getIamRole().get()) + .roleSessionName("presto-session").build()) + .stsClient(stsClientBuilder.build()) .build(); - asyncGlueClientBuilder.setCredentials(credentialsProvider); } - return asyncGlueClientBuilder.build(); + glueAsyncClientBuilder.credentialsProvider(credentialsProvider); + + return glueAsyncClientBuilder.build(); } @Managed @@ -256,36 +288,35 @@ public int getPartitionCommitBatchSize() @Override public Optional getDatabase(MetastoreContext metastoreContext, String databaseName) { - return stats.getGetDatabase().record(() -> { - try { - GetDatabaseResult result = glueClient.getDatabase(new GetDatabaseRequest().withCatalogId(catalogId).withName(databaseName)); - return Optional.of(GlueToPrestoConverter.convertDatabase(result.getDatabase())); - } - catch (EntityNotFoundException e) { - return Optional.empty(); - } - catch (AmazonServiceException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - }); + try { + GetDatabaseResponse response = awsSyncRequest(glueClient::getDatabase, + GetDatabaseRequest.builder().catalogId(catalogId).name(databaseName).build(), + stats.getGetDatabase()); + + return Optional.of(GlueToPrestoConverter.convertDatabase(response.database())); + } + catch (EntityNotFoundException e) { + return Optional.empty(); + } + catch (AwsServiceException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } } @Override public List getAllDatabases(MetastoreContext metastoreContext) { try { - List databaseNames = new ArrayList<>(); - GetDatabasesRequest request = new GetDatabasesRequest().withCatalogId(catalogId); - do { - GetDatabasesResult result = stats.getGetDatabases().record(() -> glueClient.getDatabases(request)); - request.setNextToken(result.getNextToken()); - result.getDatabaseList().forEach(database -> databaseNames.add(database.getName())); - } - while (request.getNextToken() != null); + ImmutableList.Builder databaseNames = ImmutableList.builder(); + + awsSyncPaginatedRequest( + glueClient.getDatabasesPaginator(GetDatabasesRequest.builder().catalogId(catalogId).build()), + getDatabasesResponse -> getDatabasesResponse.databaseList().forEach(database -> databaseNames.add(database.name())), + stats.getGetDatabases()); - return databaseNames; + return databaseNames.build(); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -296,29 +327,28 @@ public Optional
getTable(MetastoreContext metastoreContext, String databa return getGlueTable(databaseName, tableName).map(table -> GlueToPrestoConverter.convertTable(table, databaseName)); } - private com.amazonaws.services.glue.model.Table getGlueTableOrElseThrow(String databaseName, String tableName) + private software.amazon.awssdk.services.glue.model.Table getGlueTableOrElseThrow(String databaseName, String tableName) { return getGlueTable(databaseName, tableName) .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); } - private Optional getGlueTable(String databaseName, String tableName) + private Optional getGlueTable(String databaseName, String tableName) { - return stats.getGetTable().record(() -> { - try { - GetTableResult result = glueClient.getTable(new GetTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withName(tableName)); - return Optional.of(result.getTable()); - } - catch (EntityNotFoundException e) { - return Optional.empty(); - } - catch (AmazonServiceException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - }); + try { + GetTableResponse response = awsSyncRequest( + glueClient::getTable, + GetTableRequest.builder().catalogId(catalogId).databaseName(databaseName).name(tableName).build(), + stats.getGetTable()); + + return Optional.of(response.table()); + } + catch (EntityNotFoundException e) { + return Optional.empty(); + } + catch (AwsServiceException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } } @Override @@ -367,17 +397,22 @@ public void updateTableStatistics(MetastoreContext metastoreContext, String data try { TableInput tableInput = GlueInputConverter.convertTable(table); - tableInput.setParameters(updateStatisticsParameters(table.getParameters(), updatedStatistics.getBasicStatistics())); - UpdateTableRequest request = new UpdateTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableInput(tableInput); - stats.getUpdateTable().record(() -> glueClient.updateTable(request)); + final Map statisticsParameters = + updateStatisticsParameters(table.getParameters(), updatedStatistics.getBasicStatistics()); + + awsSyncRequest( + glueClient::updateTable, + UpdateTableRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableInput(tableInput.toBuilder().parameters(statisticsParameters).build()) + .build(), + stats.getUpdateTable()); } catch (EntityNotFoundException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -399,18 +434,24 @@ public void updatePartitionStatistics(MetastoreContext metastoreContext, String .orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues)); try { PartitionInput partitionInput = GlueInputConverter.convertPartition(partition); - partitionInput.setParameters(updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics())); - stats.getUpdatePartition().record(() -> glueClient.updatePartition(new UpdatePartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionValueList(partition.getValues()) - .withPartitionInput(partitionInput))); + final Map statisticsParameters = + updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics()); + + awsSyncRequest( + glueClient::updatePartition, + UpdatePartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionValueList(partition.getValues()) + .partitionInput(partitionInput.toBuilder().parameters(statisticsParameters).build()) + .build(), + stats.getUpdatePartition()); } catch (EntityNotFoundException e) { throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -419,22 +460,24 @@ public void updatePartitionStatistics(MetastoreContext metastoreContext, String public Optional> getAllTables(MetastoreContext metastoreContext, String databaseName) { try { - List tableNames = new ArrayList<>(); - GetTablesRequest request = new GetTablesRequest().withCatalogId(catalogId).withDatabaseName(databaseName); - do { - GetTablesResult result = stats.getGetTables().record(() -> glueClient.getTables(request)); - request.setNextToken(result.getNextToken()); - result.getTableList().forEach(table -> tableNames.add(table.getName())); - } - while (request.getNextToken() != null); + ImmutableList.Builder tableNames = ImmutableList.builder(); + + awsSyncPaginatedRequest( + glueClient.getTablesPaginator(GetTablesRequest.builder().catalogId(catalogId).databaseName(databaseName).build()), + getTablesResponse -> { + getTablesResponse.tableList().stream() + .map(software.amazon.awssdk.services.glue.model.Table::name) + .forEach(tableNames::add); + }, + stats.getGetTables()); - return Optional.of(tableNames); + return Optional.of(tableNames.build()); } catch (EntityNotFoundException e) { // database does not exist return Optional.empty(); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -443,25 +486,25 @@ public Optional> getAllTables(MetastoreContext metastoreContext, St public Optional> getAllViews(MetastoreContext metastoreContext, String databaseName) { try { - List views = new ArrayList<>(); - GetTablesRequest request = new GetTablesRequest().withCatalogId(catalogId).withDatabaseName(databaseName); - - do { - GetTablesResult result = stats.getGetTables().record(() -> glueClient.getTables(request)); - request.setNextToken(result.getNextToken()); - result.getTableList().stream() - .filter(table -> VIRTUAL_VIEW.name().equals(table.getTableType())) - .forEach(table -> views.add(table.getName())); - } - while (request.getNextToken() != null); + ImmutableList.Builder viewNames = ImmutableList.builder(); - return Optional.of(views); + awsSyncPaginatedRequest( + glueClient.getTablesPaginator(GetTablesRequest.builder().catalogId(catalogId).databaseName(databaseName).build()), + getTablesResponse -> { + getTablesResponse.tableList().stream() + .filter(table -> VIRTUAL_VIEW.name().equals(table.tableType())) + .map(software.amazon.awssdk.services.glue.model.Table::name) + .forEach(viewNames::add); + }, + stats.getGetTables()); + + return Optional.of(viewNames.build()); } catch (EntityNotFoundException e) { // database does not exist return Optional.empty(); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -478,12 +521,15 @@ public void createDatabase(MetastoreContext metastoreContext, Database database) try { DatabaseInput databaseInput = GlueInputConverter.convertDatabase(database); - stats.getCreateDatabase().record(() -> glueClient.createDatabase(new CreateDatabaseRequest().withCatalogId(catalogId).withDatabaseInput(databaseInput))); + awsSyncRequest( + glueClient::createDatabase, + CreateDatabaseRequest.builder().catalogId(catalogId).databaseInput(databaseInput).build(), + stats.getCreateDatabase()); } catch (AlreadyExistsException e) { throw new SchemaAlreadyExistsException(database.getDatabaseName()); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } @@ -496,12 +542,15 @@ public void createDatabase(MetastoreContext metastoreContext, Database database) public void dropDatabase(MetastoreContext metastoreContext, String databaseName) { try { - stats.getDeleteDatabase().record(() -> glueClient.deleteDatabase(new DeleteDatabaseRequest().withCatalogId(catalogId).withName(databaseName))); + awsSyncRequest( + glueClient::deleteDatabase, + DeleteDatabaseRequest.builder().catalogId(catalogId).name(databaseName).build(), + stats.getDeleteDatabase()); } catch (EntityNotFoundException e) { throw new SchemaNotFoundException(databaseName); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -511,13 +560,18 @@ public void renameDatabase(MetastoreContext metastoreContext, String databaseNam { try { Database database = getDatabase(metastoreContext, databaseName).orElseThrow(() -> new SchemaNotFoundException(databaseName)); - DatabaseInput renamedDatabase = GlueInputConverter.convertDatabase(database).withName(newDatabaseName); - stats.getUpdateDatabase().record(() -> glueClient.updateDatabase(new UpdateDatabaseRequest() - .withCatalogId(catalogId) - .withName(databaseName) - .withDatabaseInput(renamedDatabase))); - } - catch (AmazonServiceException e) { + DatabaseInput renamedDatabase = GlueInputConverter.convertDatabase(database); + + awsSyncRequest( + glueClient::updateDatabase, + UpdateDatabaseRequest.builder() + .catalogId(catalogId) + .name(databaseName) + .databaseInput(renamedDatabase.toBuilder().name(newDatabaseName).build()) + .build(), + stats.getUpdateDatabase()); + } + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -531,10 +585,14 @@ public MetastoreOperationResult createTable(MetastoreContext metastoreContext, T try { TableInput input = GlueInputConverter.convertTable(table); - stats.getCreateTable().record(() -> glueClient.createTable(new CreateTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(table.getDatabaseName()) - .withTableInput(input))); + awsSyncRequest( + glueClient::createTable, + CreateTableRequest.builder() + .catalogId(catalogId) + .databaseName(table.getDatabaseName()) + .tableInput(input) + .build(), + stats.getCreateTable()); } catch (AlreadyExistsException e) { throw new TableAlreadyExistsException(new SchemaTableName(table.getDatabaseName(), table.getTableName())); @@ -542,7 +600,7 @@ public MetastoreOperationResult createTable(MetastoreContext metastoreContext, T catch (EntityNotFoundException e) { throw new SchemaNotFoundException(table.getDatabaseName()); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } @@ -555,12 +613,16 @@ public void dropTable(MetastoreContext metastoreContext, String databaseName, St Table table = getTableOrElseThrow(metastoreContext, databaseName, tableName); try { - stats.getDeleteTable().record(() -> glueClient.deleteTable(new DeleteTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withName(tableName))); - } - catch (AmazonServiceException e) { + awsSyncRequest( + glueClient::deleteTable, + DeleteTableRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .name(tableName) + .build(), + stats.getDeleteTable()); + } + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } @@ -575,17 +637,22 @@ public MetastoreOperationResult replaceTable(MetastoreContext metastoreContext, { try { TableInput newTableInput = GlueInputConverter.convertTable(newTable); - stats.getUpdateTable().record(() -> glueClient.updateTable(new UpdateTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableInput(newTableInput))); + + awsSyncRequest( + glueClient::updateTable, + UpdateTableRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableInput(newTableInput) + .build(), + stats.getUpdateTable()); return EMPTY_RESULT; } catch (EntityNotFoundException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -598,18 +665,24 @@ public MetastoreOperationResult persistTable(MetastoreContext metastoreContext, } try { TableInput newTableInput = GlueInputConverter.convertTable(newTable); - newTableInput.setParameters(updateStatisticsParameters(newTableInput.getParameters(), updatedStatistics.getBasicStatistics())); - stats.getUpdateTable().record(() -> glueClient.updateTable(new UpdateTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableInput(newTableInput))); + final Map statisticsParameters = + updateStatisticsParameters(newTableInput.parameters(), updatedStatistics.getBasicStatistics()); + + awsSyncRequest( + glueClient::updateTable, + UpdateTableRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableInput(newTableInput.toBuilder().parameters(statisticsParameters).build()) + .build(), + stats.getUpdateTable()); return EMPTY_RESULT; } catch (EntityNotFoundException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -623,36 +696,38 @@ public MetastoreOperationResult renameTable(MetastoreContext metastoreContext, S @Override public MetastoreOperationResult addColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) { - com.amazonaws.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); - ImmutableList.Builder newDataColumns = ImmutableList.builder(); - newDataColumns.addAll(table.getStorageDescriptor().getColumns()); + software.amazon.awssdk.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); + ImmutableList.Builder newDataColumns = ImmutableList.builder(); + newDataColumns.addAll(table.storageDescriptor().columns()); newDataColumns.add(convertColumn(new Column(columnName, columnType, Optional.ofNullable(columnComment), Optional.empty()))); - table.getStorageDescriptor().setColumns(newDataColumns.build()); - replaceGlueTable(databaseName, tableName, table); + StorageDescriptor newStorageDescriptor = table.storageDescriptor().toBuilder().columns(newDataColumns.build()).build(); + replaceGlueTable(databaseName, tableName, table.toBuilder().storageDescriptor(newStorageDescriptor).build()); return EMPTY_RESULT; } @Override public MetastoreOperationResult renameColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String oldColumnName, String newColumnName) { - com.amazonaws.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); - if (table.getPartitionKeys() != null && table.getPartitionKeys().stream().anyMatch(c -> c.getName().equals(oldColumnName))) { + software.amazon.awssdk.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); + if (table.partitionKeys() != null && table.partitionKeys().stream().anyMatch(c -> c.name().equals(oldColumnName))) { throw new PrestoException(NOT_SUPPORTED, "Renaming partition columns is not supported"); } - ImmutableList.Builder newDataColumns = ImmutableList.builder(); - for (com.amazonaws.services.glue.model.Column column : table.getStorageDescriptor().getColumns()) { - if (column.getName().equals(oldColumnName)) { - newDataColumns.add(new com.amazonaws.services.glue.model.Column() - .withName(newColumnName) - .withType(column.getType()) - .withComment(column.getComment())); + ImmutableList.Builder newDataColumns = ImmutableList.builder(); + for (software.amazon.awssdk.services.glue.model.Column column : table.storageDescriptor().columns()) { + if (column.name().equals(oldColumnName)) { + newDataColumns.add(software.amazon.awssdk.services.glue.model.Column.builder() + .name(newColumnName) + .type(column.type()) + .comment(column.comment()) + .build()); } else { newDataColumns.add(column); } } - table.getStorageDescriptor().setColumns(newDataColumns.build()); - replaceGlueTable(databaseName, tableName, table); + + StorageDescriptor newStorageDescriptor = table.storageDescriptor().toBuilder().columns(newDataColumns.build()).build(); + replaceGlueTable(databaseName, tableName, table.toBuilder().storageDescriptor(newStorageDescriptor).build()); return EMPTY_RESULT; } @@ -660,12 +735,12 @@ public MetastoreOperationResult renameColumn(MetastoreContext metastoreContext, public MetastoreOperationResult dropColumn(MetastoreContext metastoreContext, String databaseName, String tableName, String columnName) { verifyCanDropColumn(this, metastoreContext, databaseName, tableName, columnName); - com.amazonaws.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); + software.amazon.awssdk.services.glue.model.Table table = getGlueTableOrElseThrow(databaseName, tableName); - ImmutableList.Builder newDataColumns = ImmutableList.builder(); + ImmutableList.Builder newDataColumns = ImmutableList.builder(); boolean found = false; - for (com.amazonaws.services.glue.model.Column column : table.getStorageDescriptor().getColumns()) { - if (column.getName().equals(columnName)) { + for (software.amazon.awssdk.services.glue.model.Column column : table.storageDescriptor().columns()) { + if (column.name().equals(columnName)) { found = true; } else { @@ -678,24 +753,28 @@ public MetastoreOperationResult dropColumn(MetastoreContext metastoreContext, St throw new ColumnNotFoundException(name, columnName); } - table.getStorageDescriptor().setColumns(newDataColumns.build()); - replaceGlueTable(databaseName, tableName, table); + StorageDescriptor newStorageDescriptor = table.storageDescriptor().toBuilder().columns(newDataColumns.build()).build(); + replaceGlueTable(databaseName, tableName, table.toBuilder().storageDescriptor(newStorageDescriptor).build()); return EMPTY_RESULT; } - private void replaceGlueTable(String databaseName, String tableName, com.amazonaws.services.glue.model.Table newTable) + private void replaceGlueTable(String databaseName, String tableName, software.amazon.awssdk.services.glue.model.Table newTable) { try { - stats.getUpdateTable().record(() -> glueClient.updateTable(new UpdateTableRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableInput(toTableInput(newTable)))); + awsSyncRequest( + glueClient::updateTable, + UpdateTableRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableInput(toTableInput(newTable)) + .build(), + stats.getUpdateTable()); } catch (EntityNotFoundException e) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -703,22 +782,25 @@ private void replaceGlueTable(String databaseName, String tableName, com.amazona @Override public Optional getPartition(MetastoreContext metastoreContext, String databaseName, String tableName, List partitionValues) { - return stats.getGetPartition().record(() -> { - try { - GetPartitionResult result = glueClient.getPartition(new GetPartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionValues(partitionValues)); - return Optional.of(new GluePartitionConverter(databaseName, tableName).apply(result.getPartition())); - } - catch (EntityNotFoundException e) { - return Optional.empty(); - } - catch (AmazonServiceException e) { - throw new PrestoException(HIVE_METASTORE_ERROR, e); - } - }); + try { + GetPartitionResponse response = awsSyncRequest( + glueClient::getPartition, + GetPartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionValues(partitionValues) + .build(), + stats.getGetPartition()); + + return Optional.of(new GluePartitionConverter(databaseName, tableName).apply(response.partition())); + } + catch (EntityNotFoundException e) { + return Optional.empty(); + } + catch (AwsServiceException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, e); + } } @Override @@ -772,7 +854,7 @@ private List getPartitions(String databaseName, String tableName, Str // Do parallel partition fetch. CompletionService> completionService = new ExecutorCompletionService<>(executor); for (int i = 0; i < partitionSegments; i++) { - Segment segment = new Segment().withSegmentNumber(i).withTotalSegments(partitionSegments); + Segment segment = Segment.builder().segmentNumber(i).totalSegments(partitionSegments).build(); completionService.submit(() -> getPartitions(databaseName, tableName, expression, segment)); } @@ -798,28 +880,30 @@ private List getPartitions(String databaseName, String tableName, Str { try { GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName); - ArrayList partitions = new ArrayList<>(); - GetPartitionsRequest request = new GetPartitionsRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withExpression(expression) - .withSegment(segment) - .withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS); - - do { - GetPartitionsResult result = stats.getGetPartitions().record(() -> glueClient.getPartitions(request)); - request.setNextToken(result.getNextToken()); - partitions.ensureCapacity(partitions.size() + result.getPartitions().size()); - result.getPartitions().stream() - .map(converter) - .forEach(partitions::add); - } - while (request.getNextToken() != null); - return partitions; + ImmutableList.Builder partitionBuilder = ImmutableList.builder(); + + GetPartitionsRequest partitionsRequest = GetPartitionsRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .expression(expression) + .segment(segment) + .maxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS) + .build(); + + awsSyncPaginatedRequest( + glueClient.getPartitionsPaginator(partitionsRequest), + getPartitionsResponse -> { + getPartitionsResponse.partitions().stream() + .map(converter) + .forEach(partitionBuilder::add); + }, + stats.getGetPartitions()); + + return partitionBuilder.build(); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -865,28 +949,40 @@ public Map> getPartitionsByNames(MetastoreContext me private List batchGetPartition(String databaseName, String tableName, List partitionNames) { try { - List> batchGetPartitionFutures = new ArrayList<>(); + List> batchGetPartitionFutures = new ArrayList<>(); for (List partitionNamesBatch : Lists.partition(partitionNames, BATCH_GET_PARTITION_MAX_PAGE_SIZE)) { - List partitionValuesBatch = mappedCopy(partitionNamesBatch, partitionName -> new PartitionValueList().withValues(toPartitionValues(partitionName))); - batchGetPartitionFutures.add(glueClient.batchGetPartitionAsync(new BatchGetPartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionsToGet(partitionValuesBatch), stats.getBatchGetPartitions().metricsAsyncHandler())); + List partitionValuesBatch = mappedCopy(partitionNamesBatch, partitionName -> PartitionValueList.builder().values(toPartitionValues(partitionName)).build()); + + GlueStatsAsyncHandler asyncHandler = new GlueStatsAsyncHandler(stats.getBatchGetPartitions()); + + batchGetPartitionFutures.add(glueClient.batchGetPartition(BatchGetPartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionsToGet(partitionValuesBatch) + .build()) + .whenCompleteAsync((response, exception) -> { + if (response != null) { + asyncHandler.onSuccess(response); + } + else if (exception != null) { + asyncHandler.onError(exception); + } + })); } GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName); ImmutableList.Builder resultsBuilder = ImmutableList.builderWithExpectedSize(partitionNames.size()); - for (Future future : batchGetPartitionFutures) { - future.get().getPartitions().stream() + for (Future future : batchGetPartitionFutures) { + future.get().partitions().stream() .map(converter) .forEach(resultsBuilder::add); } return resultsBuilder.build(); } - catch (AmazonServiceException | InterruptedException | ExecutionException e) { + catch (AwsServiceException | InterruptedException | ExecutionException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -898,25 +994,37 @@ private List batchGetPartition(String databaseName, String tableName, public MetastoreOperationResult addPartitions(MetastoreContext metastoreContext, String databaseName, String tableName, List partitions) { try { - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (List partitionBatch : Lists.partition(partitions, BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)) { List partitionInputs = mappedCopy(partitionBatch, GlueInputConverter::convertPartition); - futures.add(glueClient.batchCreatePartitionAsync(new BatchCreatePartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionInputList(partitionInputs), stats.getBatchCreatePartitions().metricsAsyncHandler())); + + GlueStatsAsyncHandler asyncHandler = new GlueStatsAsyncHandler(stats.getBatchCreatePartitions()); + + futures.add(glueClient.batchCreatePartition(BatchCreatePartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionInputList(partitionInputs) + .build()) + .whenCompleteAsync((response, exception) -> { + if (response != null) { + asyncHandler.onSuccess(response); + } + else if (exception != null) { + asyncHandler.onError(exception); + } + })); } - for (Future future : futures) { - BatchCreatePartitionResult result = future.get(); - propagatePartitionErrorToPrestoException(databaseName, tableName, result.getErrors()); + for (Future future : futures) { + BatchCreatePartitionResponse result = future.get(); + propagatePartitionErrorToPrestoException(databaseName, tableName, result.errors()); } return EMPTY_RESULT; } - catch (AmazonServiceException | InterruptedException | ExecutionException e) { + catch (AwsServiceException | InterruptedException | ExecutionException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -927,16 +1035,16 @@ public MetastoreOperationResult addPartitions(MetastoreContext metastoreContext, private static void propagatePartitionErrorToPrestoException(String databaseName, String tableName, List partitionErrors) { if (partitionErrors != null && !partitionErrors.isEmpty()) { - ErrorDetail errorDetail = partitionErrors.get(0).getErrorDetail(); - String glueExceptionCode = errorDetail.getErrorCode(); + ErrorDetail errorDetail = partitionErrors.get(0).errorDetail(); + String glueExceptionCode = errorDetail.errorCode(); switch (glueExceptionCode) { case "AlreadyExistsException": - throw new PrestoException(ALREADY_EXISTS, errorDetail.getErrorMessage()); + throw new PrestoException(ALREADY_EXISTS, errorDetail.errorMessage()); case "EntityNotFoundException": - throw new TableNotFoundException(new SchemaTableName(databaseName, tableName), errorDetail.getErrorMessage()); + throw new TableNotFoundException(new SchemaTableName(databaseName, tableName), errorDetail.errorMessage()); default: - throw new PrestoException(HIVE_METASTORE_ERROR, errorDetail.getErrorCode() + ": " + errorDetail.getErrorMessage()); + throw new PrestoException(HIVE_METASTORE_ERROR, errorDetail.errorCode() + ": " + errorDetail.errorMessage()); } } } @@ -949,13 +1057,17 @@ public void dropPartition(MetastoreContext metastoreContext, String databaseName .orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), parts)); try { - stats.getDeletePartition().record(() -> glueClient.deletePartition(new DeletePartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionValues(parts))); - } - catch (AmazonServiceException e) { + awsSyncRequest( + glueClient::deletePartition, + DeletePartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionValues(parts) + .build(), + stats.getDeletePartition()); + } + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } @@ -970,19 +1082,24 @@ public MetastoreOperationResult alterPartition(MetastoreContext metastoreContext { try { PartitionInput newPartition = GlueInputConverter.convertPartition(partition); - stats.getUpdatePartition().record(() -> glueClient.updatePartition(new UpdatePartitionRequest() - .withCatalogId(catalogId) - .withDatabaseName(databaseName) - .withTableName(tableName) - .withPartitionInput(newPartition) - .withPartitionValueList(partition.getPartition().getValues()))); + + awsSyncRequest( + glueClient::updatePartition, + UpdatePartitionRequest.builder() + .catalogId(catalogId) + .databaseName(databaseName) + .tableName(tableName) + .partitionInput(newPartition) + .partitionValueList(partition.getPartition().getValues()) + .build(), + stats.getUpdatePartition()); return EMPTY_RESULT; } catch (EntityNotFoundException e) { throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partition.getPartition().getValues()); } - catch (AmazonServiceException e) { + catch (AwsServiceException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } } @@ -1071,4 +1188,80 @@ public MetastoreOperationResult addConstraint(MetastoreContext metastoreContext, { throw new PrestoException(NOT_SUPPORTED, "addConstraint is not supported by Glue"); } + + public static T awsSyncRequest( + Function> submission, + R request, + GlueCatalogApiStats stats) + { + requireNonNull(submission, "submission is null"); + requireNonNull(request, "request is null"); + + try { + if (stats != null) { + return stats.record(() -> submission.apply(request).join()); + } + + return submission.apply(request).join(); + } + catch (CompletionException e) { + if (e.getCause() instanceof GlueException) { + throw (GlueException) e.getCause(); + } + throw new PrestoException(HIVE_METASTORE_ERROR, e.getCause()); + } + } + + private static void awsSyncPaginatedRequest( + SdkPublisher paginator, + Consumer resultConsumer, + GlueCatalogApiStats stats) + { + requireNonNull(paginator, "paginator is null"); + requireNonNull(resultConsumer, "resultConsumer is null"); + + // Single join point so exception handling is consistent, and stats (when present) + // cover the full wall-clock time of the paginated request including completion. + Runnable paginationTask = () -> paginator.subscribe(resultConsumer).join(); + + try { + if (stats != null) { + stats.record(() -> { + paginationTask.run(); + return null; + }); + } + else { + paginationTask.run(); + } + } + catch (CompletionException e) { + if (e.getCause() instanceof GlueException) { + throw (GlueException) e.getCause(); + } + throw new PrestoException(HIVE_METASTORE_ERROR, e.getCause()); + } + } + + static class GlueStatsAsyncHandler + { + private final GlueCatalogApiStats stats; + private final Stopwatch stopwatch; + + public GlueStatsAsyncHandler(GlueCatalogApiStats stats) + { + this.stats = requireNonNull(stats, "stats is null"); + this.stopwatch = Stopwatch.createStarted(); + } + + public void onError(Throwable e) + { + stats.recordAsync(stopwatch.elapsed(NANOSECONDS), true); + } + + public void onSuccess(GlueResponse response) + { + stats.recordAsync(stopwatch.elapsed(NANOSECONDS), false); + } + } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java index 9c2869261b16e..06aa90aef674b 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueHiveMetastoreConfig.java @@ -16,16 +16,19 @@ import com.facebook.airlift.configuration.Config; import com.facebook.airlift.configuration.ConfigDescription; import com.facebook.airlift.configuration.ConfigSecuritySensitive; +import com.facebook.airlift.configuration.DefunctConfig; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import java.util.Optional; +@DefunctConfig("hive.metastore.glue.pin-client-to-current-region") public class GlueHiveMetastoreConfig { private Optional glueRegion = Optional.empty(); private Optional glueEndpointUrl = Optional.empty(); - private boolean pinGlueClientToCurrentRegion; + private Optional glueStsRegion = Optional.empty(); + private Optional glueStsEndpointUrl = Optional.empty(); private int maxGlueErrorRetries = 10; private int maxGlueConnections = 50; private Optional defaultWarehouseDir = Optional.empty(); @@ -62,16 +65,29 @@ public GlueHiveMetastoreConfig setGlueEndpointUrl(String glueEndpointUrl) return this; } - public boolean getPinGlueClientToCurrentRegion() + public Optional getGlueStsRegion() { - return pinGlueClientToCurrentRegion; + return glueStsRegion; } - @Config("hive.metastore.glue.pin-client-to-current-region") - @ConfigDescription("Should the Glue client be pinned to the current EC2 region") - public GlueHiveMetastoreConfig setPinGlueClientToCurrentRegion(boolean pinGlueClientToCurrentRegion) + @Config("hive.metastore.glue.sts.region") + @ConfigDescription("AWS STS region for Glue authentication") + public GlueHiveMetastoreConfig setGlueStsRegion(String region) { - this.pinGlueClientToCurrentRegion = pinGlueClientToCurrentRegion; + this.glueStsRegion = Optional.ofNullable(region); + return this; + } + + public Optional getGlueStsEndpointUrl() + { + return glueStsEndpointUrl; + } + + @Config("hive.metastore.glue.sts.endpoint-url") + @ConfigDescription("AWS STS endpoint URL for Glue authentication") + public GlueHiveMetastoreConfig setGlueStsEndpointUrl(String glueStsEndpointUrl) + { + this.glueStsEndpointUrl = Optional.ofNullable(glueStsEndpointUrl); return this; } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java index 0bc9bee6b81fc..607165178c5c6 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreModule.java @@ -19,6 +19,7 @@ import com.facebook.presto.hive.HiveCommonClientConfig; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.google.inject.Binder; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -51,6 +52,7 @@ public void setup(Binder binder) checkArgument(buildConfigObject(HiveCommonClientConfig.class).getCatalogName() == null, "'hive.metastore.catalog.name' should not be set for glue metastore"); configBinder(binder).bindConfig(GlueHiveMetastoreConfig.class); binder.bind(GlueHiveMetastore.class).in(Scopes.SINGLETON); + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(GlueHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(ExtendedHiveMetastore.class) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreStats.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreStats.java index 7c0e99595d34a..b0779572f46be 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreStats.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/GlueMetastoreStats.java @@ -13,15 +13,11 @@ */ package com.facebook.presto.hive.metastore.glue; -import com.facebook.airlift.stats.CounterStat; -import com.facebook.airlift.stats.TimeStat; -import com.facebook.airlift.units.Duration; -import com.facebook.presto.hive.aws.AbstractSdkMetricsCollector; +import com.facebook.presto.hive.aws.metrics.AwsSdkClientStats; +import org.weakref.jmx.Flatten; import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; - -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import software.amazon.awssdk.metrics.MetricPublisher; public class GlueMetastoreStats { @@ -42,13 +38,7 @@ public class GlueMetastoreStats private final GlueCatalogApiStats getPartitions = new GlueCatalogApiStats(); private final GlueCatalogApiStats getPartition = new GlueCatalogApiStats(); - // see AWSRequestMetrics - private final CounterStat awsRequestCount = new CounterStat(); - private final CounterStat awsRetryCount = new CounterStat(); - private final CounterStat awsThrottleExceptions = new CounterStat(); - private final TimeStat awsRequestTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientExecuteTime = new TimeStat(MILLISECONDS); - private final TimeStat awsClientRetryPauseTime = new TimeStat(MILLISECONDS); + private final AwsSdkClientStats awsSdkClientStats = new AwsSdkClientStats(); @Managed @Nested @@ -163,96 +153,14 @@ public GlueCatalogApiStats getGetPartition() } @Managed - @Nested - public CounterStat getAwsRequestCount() - { - return awsRequestCount; - } - - @Managed - @Nested - public CounterStat getAwsRetryCount() + @Flatten + public AwsSdkClientStats getAwsSdkClientStats() { - return awsRetryCount; + return awsSdkClientStats; } - @Managed - @Nested - public CounterStat getAwsThrottleExceptions() - { - return awsThrottleExceptions; - } - - @Managed - @Nested - public TimeStat getAwsRequestTime() - { - return awsRequestTime; - } - - @Managed - @Nested - public TimeStat getAwsClientExecuteTime() + public MetricPublisher newRequestMetricPublisher() { - return awsClientExecuteTime; - } - - @Managed - @Nested - public TimeStat getAwsClientRetryPauseTime() - { - return awsClientRetryPauseTime; - } - - public GlueSdkClientMetricsCollector newRequestMetricsCollector() - { - return new GlueSdkClientMetricsCollector(this); - } - - public static class GlueSdkClientMetricsCollector - extends AbstractSdkMetricsCollector - { - private final GlueMetastoreStats stats; - - public GlueSdkClientMetricsCollector(GlueMetastoreStats stats) - { - this.stats = requireNonNull(stats, "stats is null"); - } - - @Override - protected void recordRequestCount(long count) - { - stats.awsRequestCount.update(count); - } - - @Override - protected void recordRetryCount(long count) - { - stats.awsRetryCount.update(count); - } - - @Override - protected void recordThrottleExceptionCount(long count) - { - stats.awsThrottleExceptions.update(count); - } - - @Override - protected void recordHttpRequestTime(Duration duration) - { - stats.awsRequestTime.add(duration); - } - - @Override - protected void recordClientExecutionTime(Duration duration) - { - stats.awsClientExecuteTime.add(duration); - } - - @Override - protected void recordRetryPauseTime(Duration duration) - { - stats.awsClientRetryPauseTime.add(duration); - } + return awsSdkClientStats.newRequestMetricsPublisher(); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java index edf2b7d17fdf5..7c4cf09f3746f 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueInputConverter.java @@ -13,12 +13,6 @@ */ package com.facebook.presto.hive.metastore.glue.converter; -import com.amazonaws.services.glue.model.DatabaseInput; -import com.amazonaws.services.glue.model.Order; -import com.amazonaws.services.glue.model.PartitionInput; -import com.amazonaws.services.glue.model.SerDeInfo; -import com.amazonaws.services.glue.model.StorageDescriptor; -import com.amazonaws.services.glue.model.TableInput; import com.facebook.presto.hive.HiveBucketProperty; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; @@ -29,6 +23,12 @@ import com.facebook.presto.hive.metastore.Table; import com.facebook.presto.spi.PrestoException; import com.google.common.collect.ImmutableMap; +import software.amazon.awssdk.services.glue.model.DatabaseInput; +import software.amazon.awssdk.services.glue.model.Order; +import software.amazon.awssdk.services.glue.model.PartitionInput; +import software.amazon.awssdk.services.glue.model.SerDeInfo; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.TableInput; import java.util.EnumSet; import java.util.List; @@ -41,7 +41,6 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.stream.Collectors.toList; public final class GlueInputConverter { @@ -49,41 +48,42 @@ private GlueInputConverter() {} public static DatabaseInput convertDatabase(Database database) { - DatabaseInput input = new DatabaseInput(); - input.setName(database.getDatabaseName()); - input.setParameters(database.getParameters()); - database.getComment().ifPresent(input::setDescription); - database.getLocation().ifPresent(input::setLocationUri); - return input; + return DatabaseInput.builder() + .name(database.getDatabaseName()) + .parameters(database.getParameters()) + .applyMutation(builder -> database.getComment().ifPresent(builder::description)) + .applyMutation(builder -> database.getLocation().ifPresent(builder::locationUri)) + .build(); } public static TableInput convertTable(Table table) { - TableInput input = new TableInput(); - input.setName(table.getTableName()); - input.setOwner(table.getOwner()); checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType()); - input.setTableType(table.getTableType().toString()); - input.setStorageDescriptor(convertStorage(table.getStorage(), table.getDataColumns())); - input.setPartitionKeys(table.getPartitionColumns().stream().map(GlueInputConverter::convertColumn).collect(toList())); - input.setParameters(table.getParameters()); - table.getViewOriginalText().ifPresent(input::setViewOriginalText); - table.getViewExpandedText().ifPresent(input::setViewExpandedText); - return input; + + return TableInput.builder() + .name(table.getTableName()) + .owner(table.getOwner()) + .tableType(table.getTableType().toString()) + .storageDescriptor(convertStorage(table.getStorage(), table.getDataColumns())) + .partitionKeys(table.getPartitionColumns().stream().map(GlueInputConverter::convertColumn).collect(toImmutableList())) + .parameters(table.getParameters()) + .applyMutation(builder -> table.getViewOriginalText().ifPresent(builder::viewOriginalText)) + .applyMutation(builder -> table.getViewExpandedText().ifPresent(builder::viewExpandedText)) + .build(); } - public static TableInput toTableInput(com.amazonaws.services.glue.model.Table table) + public static TableInput toTableInput(software.amazon.awssdk.services.glue.model.Table table) { - TableInput input = new TableInput(); - input.setName(table.getName()); - input.setOwner(table.getOwner()); - input.setTableType(table.getTableType()); - input.setStorageDescriptor(table.getStorageDescriptor()); - input.setPartitionKeys(table.getPartitionKeys()); - input.setParameters(table.getParameters()); - input.setViewOriginalText(table.getViewOriginalText()); - input.setViewExpandedText(table.getViewExpandedText()); - return input; + return TableInput.builder() + .name(table.name()) + .owner(table.owner()) + .tableType(table.tableType()) + .storageDescriptor(table.storageDescriptor()) + .partitionKeys(table.partitionKeys()) + .parameters(table.parameters()) + .viewOriginalText(table.viewOriginalText()) + .viewExpandedText(table.viewExpandedText()) + .build(); } public static PartitionInput convertPartition(PartitionWithStatistics partitionWithStatistics) @@ -93,17 +93,17 @@ public static PartitionInput convertPartition(PartitionWithStatistics partitionW if (!statistics.getColumnStatistics().isEmpty()) { throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics"); } - input.setParameters(updateStatisticsParameters(input.getParameters(), statistics.getBasicStatistics())); - return input; + return input.toBuilder().parameters(updateStatisticsParameters(input.parameters(), statistics.getBasicStatistics())) + .build(); } public static PartitionInput convertPartition(Partition partition) { - PartitionInput input = new PartitionInput(); - input.setValues(partition.getValues()); - input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns())); - input.setParameters(partition.getParameters()); - return input; + return PartitionInput.builder() + .values(partition.getValues()) + .storageDescriptor(convertStorage(partition.getStorage(), partition.getColumns())) + .parameters(partition.getParameters()) + .build(); } private static StorageDescriptor convertStorage(Storage storage, List columns) @@ -111,37 +111,39 @@ private static StorageDescriptor convertStorage(Storage storage, List co if (storage.isSkewed()) { throw new IllegalArgumentException("Writing to skewed table/partition is not supported"); } - SerDeInfo serdeInfo = new SerDeInfo() - .withSerializationLibrary(storage.getStorageFormat().getSerDeNullable()) - .withParameters(storage.getSerdeParameters()); + SerDeInfo serDeInfo = SerDeInfo.builder() + .serializationLibrary(storage.getStorageFormat().getSerDeNullable()) + .parameters(storage.getSerdeParameters()) + .build(); - StorageDescriptor sd = new StorageDescriptor(); - sd.setLocation(storage.getLocation()); - sd.setColumns(columns.stream().map(GlueInputConverter::convertColumn).collect(toList())); - sd.setSerdeInfo(serdeInfo); - sd.setInputFormat(storage.getStorageFormat().getInputFormatNullable()); - sd.setOutputFormat(storage.getStorageFormat().getOutputFormatNullable()); - sd.setParameters(ImmutableMap.of()); + StorageDescriptor.Builder sd = StorageDescriptor.builder() + .location(storage.getLocation()) + .columns(columns.stream().map(GlueInputConverter::convertColumn).collect(toImmutableList())) + .serdeInfo(serDeInfo) + .inputFormat(storage.getStorageFormat().getInputFormatNullable()) + .outputFormat(storage.getStorageFormat().getOutputFormatNullable()) + .parameters(ImmutableMap.of()); Optional bucketProperty = storage.getBucketProperty(); if (bucketProperty.isPresent()) { - sd.setNumberOfBuckets(bucketProperty.get().getBucketCount()); - sd.setBucketColumns(bucketProperty.get().getBucketedBy()); + sd.numberOfBuckets(bucketProperty.get().getBucketCount()); + sd.bucketColumns(bucketProperty.get().getBucketedBy()); if (!bucketProperty.get().getSortedBy().isEmpty()) { - sd.setSortColumns(bucketProperty.get().getSortedBy().stream() - .map(column -> new Order().withColumn(column.getColumnName()).withSortOrder(column.getOrder().getHiveOrder())) + sd.sortColumns(bucketProperty.get().getSortedBy().stream() + .map(column -> Order.builder().column(column.getColumnName()).sortOrder(column.getOrder().getHiveOrder()).build()) .collect(toImmutableList())); } } - return sd; + return sd.build(); } - public static com.amazonaws.services.glue.model.Column convertColumn(Column prestoColumn) + public static software.amazon.awssdk.services.glue.model.Column convertColumn(Column prestoColumn) { - return new com.amazonaws.services.glue.model.Column() - .withName(prestoColumn.getName()) - .withType(prestoColumn.getType().toString()) - .withComment(prestoColumn.getComment().orElse(null)); + return software.amazon.awssdk.services.glue.model.Column.builder() + .name(prestoColumn.getName()) + .type(prestoColumn.getType().toString()) + .comment(prestoColumn.getComment().orElse(null)) + .build(); } } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java index 83de7d0150fc7..947bddbb80043 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/glue/converter/GlueToPrestoConverter.java @@ -13,8 +13,6 @@ */ package com.facebook.presto.hive.metastore.glue.converter; -import com.amazonaws.services.glue.model.SerDeInfo; -import com.amazonaws.services.glue.model.StorageDescriptor; import com.facebook.presto.hive.HiveBucketProperty; import com.facebook.presto.hive.HiveStorageFormat; import com.facebook.presto.hive.HiveType; @@ -31,6 +29,8 @@ import com.facebook.presto.spi.security.PrincipalType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import software.amazon.awssdk.services.glue.model.SerDeInfo; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; import java.util.List; import java.util.Locale; @@ -57,47 +57,47 @@ public final class GlueToPrestoConverter private GlueToPrestoConverter() {} - public static Database convertDatabase(com.amazonaws.services.glue.model.Database glueDb) + public static Database convertDatabase(software.amazon.awssdk.services.glue.model.Database glueDb) { return Database.builder() - .setDatabaseName(glueDb.getName()) - .setLocation(Optional.ofNullable(glueDb.getLocationUri())) - .setComment(Optional.ofNullable(glueDb.getDescription())) - .setParameters(convertParameters(glueDb.getParameters())) + .setDatabaseName(glueDb.name()) + .setLocation(Optional.ofNullable(glueDb.locationUri())) + .setComment(Optional.ofNullable(glueDb.description())) + .setParameters(convertParameters(glueDb.parameters())) .setOwnerName(PUBLIC_OWNER) .setOwnerType(PrincipalType.ROLE) .build(); } - public static Table convertTable(com.amazonaws.services.glue.model.Table glueTable, String dbName) + public static Table convertTable(software.amazon.awssdk.services.glue.model.Table glueTable, String dbName) { - Map tableParameters = convertParameters(glueTable.getParameters()); + Map tableParameters = convertParameters(glueTable.parameters()); Table.Builder tableBuilder = Table.builder() .setDatabaseName(dbName) - .setTableName(glueTable.getName()) - .setOwner(nullToEmpty(glueTable.getOwner())) + .setTableName(glueTable.name()) + .setOwner(nullToEmpty(glueTable.owner())) // Athena treats missing table type as EXTERNAL_TABLE. - .setTableType(PrestoTableType.optionalValueOf(glueTable.getTableType()).orElse(EXTERNAL_TABLE)) + .setTableType(PrestoTableType.optionalValueOf(glueTable.tableType()).orElse(EXTERNAL_TABLE)) .setParameters(tableParameters) - .setViewOriginalText(Optional.ofNullable(glueTable.getViewOriginalText())) - .setViewExpandedText(Optional.ofNullable(glueTable.getViewExpandedText())); + .setViewOriginalText(Optional.ofNullable(glueTable.viewOriginalText())) + .setViewExpandedText(Optional.ofNullable(glueTable.viewExpandedText())); - StorageDescriptor sd = glueTable.getStorageDescriptor(); + StorageDescriptor sd = glueTable.storageDescriptor(); if (isIcebergTable(tableParameters) || (sd == null && isDeltaLakeTable(tableParameters))) { // Iceberg and Delta Lake tables do not use the StorageDescriptor field, but we need to return a Table so the caller can check that // the table is an Iceberg/Delta table and decide whether to redirect or fail. tableBuilder.setDataColumns(ImmutableList.of(new Column("dummy", HIVE_INT, Optional.empty(), Optional.empty()))); tableBuilder.getStorageBuilder().setStorageFormat(StorageFormat.fromHiveStorageFormat(HiveStorageFormat.PARQUET)); - tableBuilder.getStorageBuilder().setLocation(sd == null ? "" : sd.getLocation()); + tableBuilder.getStorageBuilder().setLocation(sd == null ? "" : sd.location()); } else { if (sd == null) { - throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Table StorageDescriptor is null for table %s.%s (%s)", dbName, glueTable.getName(), glueTable)); + throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Table StorageDescriptor is null for table %s.%s (%s)", dbName, glueTable.name(), glueTable)); } - tableBuilder.setDataColumns(convertColumns(sd.getColumns())); - if (glueTable.getPartitionKeys() != null) { - tableBuilder.setPartitionColumns(convertColumns(glueTable.getPartitionKeys())); + tableBuilder.setDataColumns(convertColumns(sd.columns())); + if (glueTable.partitionKeys() != null) { + tableBuilder.setPartitionColumns(convertColumns(glueTable.partitionKeys())); } else { tableBuilder.setPartitionColumns(ImmutableList.of()); @@ -109,12 +109,12 @@ public static Table convertTable(com.amazonaws.services.glue.model.Table glueTab return tableBuilder.build(); } - private static Column convertColumn(com.amazonaws.services.glue.model.Column glueColumn) + private static Column convertColumn(software.amazon.awssdk.services.glue.model.Column glueColumn) { - return new Column(glueColumn.getName(), HiveType.valueOf(glueColumn.getType().toLowerCase(Locale.ENGLISH)), Optional.ofNullable(glueColumn.getComment()), Optional.empty()); + return new Column(glueColumn.name(), HiveType.valueOf(glueColumn.type().toLowerCase(Locale.ENGLISH)), Optional.ofNullable(glueColumn.comment()), Optional.empty()); } - private static List convertColumns(List glueColumns) + private static List convertColumns(List glueColumns) { return mappedCopy(glueColumns, GlueToPrestoConverter::convertColumn); } @@ -138,9 +138,9 @@ private static boolean isNullOrEmpty(List list) } public static final class GluePartitionConverter - implements Function + implements Function { - private final Function, List> columnsConverter = memoizeLast(GlueToPrestoConverter::convertColumns); + private final Function, List> columnsConverter = memoizeLast(GlueToPrestoConverter::convertColumns); private final Function, Map> parametersConverter = parametersConverter(); private final StorageConverter storageConverter = new StorageConverter(); private final String databaseName; @@ -153,25 +153,25 @@ public GluePartitionConverter(String databaseName, String tableName) } @Override - public Partition apply(com.amazonaws.services.glue.model.Partition gluePartition) + public Partition apply(software.amazon.awssdk.services.glue.model.Partition gluePartition) { - requireNonNull(gluePartition.getStorageDescriptor(), "Partition StorageDescriptor is null"); - StorageDescriptor sd = gluePartition.getStorageDescriptor(); + requireNonNull(gluePartition.storageDescriptor(), "Partition StorageDescriptor is null"); + StorageDescriptor sd = gluePartition.storageDescriptor(); - if (!databaseName.equals(gluePartition.getDatabaseName())) { - throw new IllegalArgumentException(format("Unexpected databaseName, expected: %s, but found: %s", databaseName, gluePartition.getDatabaseName())); + if (!databaseName.equals(gluePartition.databaseName())) { + throw new IllegalArgumentException(format("Unexpected databaseName, expected: %s, but found: %s", databaseName, gluePartition.databaseName())); } - if (!tableName.equals(gluePartition.getTableName())) { - throw new IllegalArgumentException(format("Unexpected tableName, expected: %s, but found: %s", tableName, gluePartition.getTableName())); + if (!tableName.equals(gluePartition.tableName())) { + throw new IllegalArgumentException(format("Unexpected tableName, expected: %s, but found: %s", tableName, gluePartition.tableName())); } Partition.Builder partitionBuilder = Partition.builder() .setCatalogName(Optional.empty()) .setDatabaseName(databaseName) .setTableName(tableName) - .setValues(gluePartition.getValues()) // No memoization benefit - .setColumns(columnsConverter.apply(sd.getColumns())) - .setParameters(parametersConverter.apply(gluePartition.getParameters())); + .setValues(gluePartition.values()) // No memoization benefit + .setColumns(columnsConverter.apply(sd.columns())) + .setParameters(parametersConverter.apply(gluePartition.parameters())); storageConverter.setConvertedStorage(sd, partitionBuilder.getStorageBuilder()); @@ -182,7 +182,7 @@ public Partition apply(com.amazonaws.services.glue.model.Partition gluePartition private static final class StorageConverter { private final Function, List> bucketColumns = memoizeLast(ImmutableList::copyOf); - private final Function, List> sortColumns = memoizeLast(StorageConverter::createSortingColumns); + private final Function, List> sortColumns = memoizeLast(StorageConverter::createSortingColumns); private final UnaryOperator> bucketProperty = memoizeLast(); private final Function, Map> serdeParametersConverter = parametersConverter(); private final Function, Map> partitionParametersConverter = parametersConverter(); @@ -190,36 +190,36 @@ private static final class StorageConverter public void setConvertedStorage(StorageDescriptor sd, Storage.Builder storageBuilder) { - requireNonNull(sd.getSerdeInfo(), "StorageDescriptor SerDeInfo is null"); - SerDeInfo serdeInfo = sd.getSerdeInfo(); + requireNonNull(sd.serdeInfo(), "StorageDescriptor SerDeInfo is null"); + SerDeInfo serdeInfo = sd.serdeInfo(); - storageBuilder.setLocation(nullToEmpty(sd.getLocation())) + storageBuilder.setLocation(nullToEmpty(sd.location())) .setBucketProperty(createBucketProperty(sd)) - .setSkewed(sd.getSkewedInfo() != null && !isNullOrEmpty(sd.getSkewedInfo().getSkewedColumnNames())) - .setSerdeParameters(serdeParametersConverter.apply(serdeInfo.getParameters())) - .setParameters(partitionParametersConverter.apply(sd.getParameters())) + .setSkewed(sd.skewedInfo() != null && !isNullOrEmpty(sd.skewedInfo().skewedColumnNames())) + .setSerdeParameters(serdeParametersConverter.apply(serdeInfo.parameters())) + .setParameters(partitionParametersConverter.apply(sd.parameters())) .setStorageFormat(storageFormatConverter.createStorageFormat(serdeInfo, sd)); } private Optional createBucketProperty(StorageDescriptor sd) { - if (sd.getNumberOfBuckets() > 0) { - if (isNullOrEmpty(sd.getBucketColumns())) { + if (sd.numberOfBuckets() > 0) { + if (isNullOrEmpty(sd.bucketColumns())) { throw new PrestoException(HIVE_INVALID_METADATA, "Table/partition metadata has 'numBuckets' set, but 'bucketCols' is not set"); } - List bucketColumns = this.bucketColumns.apply(sd.getBucketColumns()); - List sortedBy = this.sortColumns.apply(sd.getSortColumns()); - return bucketProperty.apply(Optional.of(new HiveBucketProperty(bucketColumns, sd.getNumberOfBuckets(), sortedBy, HIVE_COMPATIBLE, Optional.empty()))); + List bucketColumns = this.bucketColumns.apply(sd.bucketColumns()); + List sortedBy = this.sortColumns.apply(sd.sortColumns()); + return bucketProperty.apply(Optional.of(new HiveBucketProperty(bucketColumns, sd.numberOfBuckets(), sortedBy, HIVE_COMPATIBLE, Optional.empty()))); } return Optional.empty(); } - private static List createSortingColumns(List sortColumns) + private static List createSortingColumns(List sortColumns) { if (isNullOrEmpty(sortColumns)) { return ImmutableList.of(); } - return mappedCopy(sortColumns, column -> new SortingColumn(column.getColumn(), Order.fromMetastoreApiOrder(column.getSortOrder(), "unknown"))); + return mappedCopy(sortColumns, column -> new SortingColumn(column.column(), Order.fromMetastoreApiOrder(column.sortOrder(), "unknown"))); } } @@ -234,9 +234,9 @@ private static final class StorageFormatConverter public StorageFormat createStorageFormat(SerDeInfo serdeInfo, StorageDescriptor storageDescriptor) { - String serializationLib = this.serializationLib.apply(serdeInfo.getSerializationLibrary()); - String inputFormat = this.inputFormat.apply(storageDescriptor.getInputFormat()); - String outputFormat = this.outputFormat.apply(storageDescriptor.getOutputFormat()); + String serializationLib = this.serializationLib.apply(serdeInfo.serializationLibrary()); + String inputFormat = this.inputFormat.apply(storageDescriptor.inputFormat()); + String outputFormat = this.outputFormat.apply(storageDescriptor.outputFormat()); if (serializationLib == null && inputFormat == null && outputFormat == null) { return ALL_NULLS; } diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java index 6893f1f1dbd28..094dd9d448c73 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java @@ -19,6 +19,7 @@ import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.RecordingHiveMetastore; import com.facebook.presto.spi.ConnectorId; import com.google.inject.Binder; @@ -69,6 +70,7 @@ protected void setup(Binder binder) .in(Scopes.SINGLETON); } + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(HiveMetastore.class) .as(generatedNameOf(ThriftHiveMetastore.class, connectorId)); diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java index 3c8e64dc48bba..6f2630b47e9d2 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/TestMetastoreClientConfig.java @@ -20,11 +20,15 @@ import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; +import com.google.inject.ConfigurationException; import org.testng.annotations.Test; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.expectThrows; + public class TestMetastoreClientConfig { @Test @@ -35,8 +39,12 @@ public void testDefaults() .setMetastoreTimeout(new Duration(10, TimeUnit.SECONDS)) .setVerifyChecksum(true) .setRequireHadoopNative(true) - .setMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS)) - .setMetastoreRefreshInterval(new Duration(0, TimeUnit.SECONDS)) + .setEnabledCaches(null) + .setDisabledCaches(null) + .setDefaultMetastoreCacheTtl(new Duration(0, TimeUnit.SECONDS)) + .setDefaultMetastoreCacheRefreshInterval(new Duration(0, TimeUnit.SECONDS)) + .setMetastoreCacheTtlByType(null) + .setMetastoreCacheRefreshIntervalByType(null) .setMetastoreCacheMaximumSize(10000) .setPerTransactionMetastoreCacheMaximumSize(1000) .setMaxMetastoreRefreshThreads(100) @@ -61,8 +69,12 @@ public void testExplicitPropertyMappings() .put("hive.metastore-timeout", "20s") .put("hive.dfs.verify-checksum", "false") .put("hive.dfs.require-hadoop-native", "false") - .put("hive.metastore-cache-ttl", "2h") - .put("hive.metastore-refresh-interval", "30m") + .put("hive.metastore.cache.enabled-caches", "TABLE,TABLE_NAMES") + .put("hive.metastore.cache.disabled-caches", "TABLE,TABLE_NAMES") + .put("hive.metastore.cache.ttl.default", "2h") + .put("hive.metastore.cache.refresh-interval.default", "30m") + .put("hive.metastore.cache.ttl-by-type", "TABLE:10m") + .put("hive.metastore.cache.refresh-interval-by-type", "TABLE:5m") .put("hive.metastore-cache-maximum-size", "5000") .put("hive.per-transaction-metastore-cache-maximum-size", "500") .put("hive.metastore-refresh-max-threads", "2500") @@ -84,8 +96,12 @@ public void testExplicitPropertyMappings() .setMetastoreTimeout(new Duration(20, TimeUnit.SECONDS)) .setVerifyChecksum(false) .setRequireHadoopNative(false) - .setMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) - .setMetastoreRefreshInterval(new Duration(30, TimeUnit.MINUTES)) + .setEnabledCaches("TABLE,TABLE_NAMES") + .setDisabledCaches("TABLE,TABLE_NAMES") + .setDefaultMetastoreCacheTtl(new Duration(2, TimeUnit.HOURS)) + .setDefaultMetastoreCacheRefreshInterval(new Duration(30, TimeUnit.MINUTES)) + .setMetastoreCacheTtlByType("TABLE:10m") + .setMetastoreCacheRefreshIntervalByType("TABLE:5m") .setMetastoreCacheMaximumSize(5000) .setPerTransactionMetastoreCacheMaximumSize(500) .setMaxMetastoreRefreshThreads(2500) @@ -103,4 +119,20 @@ public void testExplicitPropertyMappings() ConfigAssertions.assertFullMapping(properties, expected); } + + @Test + public void testInvalidConfiguration() + { + MetastoreClientConfig config = new MetastoreClientConfig(); + config.setEnabledCaches("TABLE,TABLE_NAMES"); + config.setDisabledCaches("TABLE,TABLE_NAMES"); + + ConfigurationException exception = expectThrows( + ConfigurationException.class, + config::isBothEnabledAndDisabledConfigured); + + assertEquals(exception.getErrorMessages().iterator().next().getMessage(), + "Only one of 'hive.metastore.cache.enabled-caches' or 'hive.metastore.cache.disabled-caches' can be set. " + + "These configs are mutually exclusive."); + } } diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java index fbc22461393ef..cea988de5c96b 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueHiveMetastoreConfig.java @@ -30,7 +30,8 @@ public void testDefaults() assertRecordedDefaults(recordDefaults(GlueHiveMetastoreConfig.class) .setGlueRegion(null) .setGlueEndpointUrl(null) - .setPinGlueClientToCurrentRegion(false) + .setGlueStsRegion(null) + .setGlueStsEndpointUrl(null) .setMaxGlueConnections(50) .setMaxGlueErrorRetries(10) .setDefaultWarehouseDir(null) @@ -48,7 +49,8 @@ public void testExplicitPropertyMapping() Map properties = new ImmutableMap.Builder() .put("hive.metastore.glue.region", "us-east-1") .put("hive.metastore.glue.endpoint-url", "http://foo.bar") - .put("hive.metastore.glue.pin-client-to-current-region", "true") + .put("hive.metastore.glue.sts.region", "us-east-1") + .put("hive.metastore.glue.sts.endpoint-url", "http://foo.bar") .put("hive.metastore.glue.max-connections", "10") .put("hive.metastore.glue.max-error-retries", "20") .put("hive.metastore.glue.default-warehouse-dir", "/location") @@ -63,7 +65,8 @@ public void testExplicitPropertyMapping() GlueHiveMetastoreConfig expected = new GlueHiveMetastoreConfig() .setGlueRegion("us-east-1") .setGlueEndpointUrl("http://foo.bar") - .setPinGlueClientToCurrentRegion(true) + .setGlueStsRegion("us-east-1") + .setGlueStsEndpointUrl("http://foo.bar") .setMaxGlueConnections(10) .setMaxGlueErrorRetries(20) .setDefaultWarehouseDir("/location") diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index bc26803c8ec4d..fb3181c672e94 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -30,6 +30,18 @@ + + + + software.amazon.awssdk + utils + + + + software.amazon.awssdk + glue + + com.facebook.airlift http-client @@ -218,11 +230,6 @@ aws-java-sdk-core - - com.amazonaws - aws-java-sdk-glue - - com.amazonaws aws-java-sdk-s3 @@ -546,7 +553,8 @@ com.fasterxml.jackson.core:jackson-core - com.amazonaws:aws-java-sdk-glue + software.amazon.awssdk:glue + software.amazon.awssdk:utils diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java index a68d371ee0712..7053ea0610d41 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java @@ -21,6 +21,7 @@ import com.facebook.presto.cache.CachingModule; import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.hive.authentication.HiveAuthenticationModule; import com.facebook.presto.hive.gcs.HiveGcsModule; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index f9cab97400b3e..f806e1e696eaa 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -261,9 +261,12 @@ import static com.facebook.presto.hive.HiveTableProperties.SKIP_HEADER_LINE_COUNT; import static com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY; +import static com.facebook.presto.hive.HiveTableProperties.TEXTFILE_COLLECTION_DELIM; +import static com.facebook.presto.hive.HiveTableProperties.TEXTFILE_ESCAPE_DELIM; +import static com.facebook.presto.hive.HiveTableProperties.TEXTFILE_FIELD_DELIM; +import static com.facebook.presto.hive.HiveTableProperties.TEXTFILE_MAPKEY_DELIM; import static com.facebook.presto.hive.HiveTableProperties.getAvroSchemaUrl; import static com.facebook.presto.hive.HiveTableProperties.getBucketProperty; -import static com.facebook.presto.hive.HiveTableProperties.getCsvProperty; import static com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionAlgorithm; import static com.facebook.presto.hive.HiveTableProperties.getDwrfEncryptionProvider; import static com.facebook.presto.hive.HiveTableProperties.getEncryptColumns; @@ -276,6 +279,7 @@ import static com.facebook.presto.hive.HiveTableProperties.getOrcBloomFilterFpp; import static com.facebook.presto.hive.HiveTableProperties.getPartitionedBy; import static com.facebook.presto.hive.HiveTableProperties.getPreferredOrderingColumns; +import static com.facebook.presto.hive.HiveTableProperties.getSingleCharacterProperty; import static com.facebook.presto.hive.HiveTableProperties.isExternalTable; import static com.facebook.presto.hive.HiveType.HIVE_BINARY; import static com.facebook.presto.hive.HiveType.toHiveType; @@ -418,6 +422,17 @@ public class HiveMetadata private static final String CSV_QUOTE_KEY = OpenCSVSerde.QUOTECHAR; private static final String CSV_ESCAPE_KEY = OpenCSVSerde.ESCAPECHAR; + private static final String TEXTFILE_FIELD_DELIM_KEY = "field.delim"; + private static final String TEXTFILE_ESCAPE_DELIM_KEY = "escape.delim"; + private static final String TEXTFILE_COLLECTION_DELIM_KEY = "collection.delim"; + private static final String TEXTFILE_MAPKEY_DELIM_KEY = "mapkey.delim"; + + private static final Set TEXTFILE_SERDE_KEYS = ImmutableSet.of( + TEXTFILE_FIELD_DELIM_KEY, + TEXTFILE_ESCAPE_DELIM_KEY, + TEXTFILE_COLLECTION_DELIM_KEY, + TEXTFILE_MAPKEY_DELIM_KEY); + public static final String SKIP_HEADER_COUNT_KEY = "skip.header.line.count"; public static final String SKIP_FOOTER_COUNT_KEY = "skip.footer.line.count"; @@ -768,6 +783,16 @@ private ConnectorTableMetadata getTableMetadata(Optional
table, SchemaTab getSerdeProperty(table.get(), SKIP_FOOTER_COUNT_KEY) .ifPresent(skipFooterCount -> properties.put(SKIP_FOOTER_LINE_COUNT, Integer.valueOf(skipFooterCount))); + // Textfile specific properties + getSerdeProperty(table.get(), TEXTFILE_FIELD_DELIM_KEY) + .ifPresent(fieldDelim -> properties.put(TEXTFILE_FIELD_DELIM, fieldDelim)); + getSerdeProperty(table.get(), TEXTFILE_ESCAPE_DELIM_KEY) + .ifPresent(escapeDelim -> properties.put(TEXTFILE_ESCAPE_DELIM, escapeDelim)); + getSerdeProperty(table.get(), TEXTFILE_COLLECTION_DELIM_KEY) + .ifPresent(textCollectionDelim -> properties.put(TEXTFILE_COLLECTION_DELIM, textCollectionDelim)); + getSerdeProperty(table.get(), TEXTFILE_MAPKEY_DELIM_KEY) + .ifPresent(textMapKeyDelim -> properties.put(TEXTFILE_MAPKEY_DELIM, textMapKeyDelim)); + // CSV specific property getCsvSerdeProperty(table.get(), CSV_SEPARATOR_KEY) .ifPresent(csvSeparator -> properties.put(CSV_SEPARATOR, csvSeparator)); @@ -1335,22 +1360,45 @@ private Map getEmptyTableProperties( }); // CSV specific properties - getCsvProperty(tableMetadata.getProperties(), CSV_ESCAPE) + getSingleCharacterProperty(tableMetadata.getProperties(), CSV_ESCAPE) .ifPresent(escape -> { checkFormatForProperty(hiveStorageFormat, CSV, CSV_ESCAPE); tableProperties.put(CSV_ESCAPE_KEY, escape.toString()); }); - getCsvProperty(tableMetadata.getProperties(), CSV_QUOTE) + getSingleCharacterProperty(tableMetadata.getProperties(), CSV_QUOTE) .ifPresent(quote -> { checkFormatForProperty(hiveStorageFormat, CSV, CSV_QUOTE); tableProperties.put(CSV_QUOTE_KEY, quote.toString()); }); - getCsvProperty(tableMetadata.getProperties(), CSV_SEPARATOR) + getSingleCharacterProperty(tableMetadata.getProperties(), CSV_SEPARATOR) .ifPresent(separator -> { checkFormatForProperty(hiveStorageFormat, CSV, CSV_SEPARATOR); tableProperties.put(CSV_SEPARATOR_KEY, separator.toString()); }); + // TEXT specific properties + getSingleCharacterProperty(tableMetadata.getProperties(), TEXTFILE_FIELD_DELIM) + .ifPresent(fieldDelim -> { + checkFormatForProperty(hiveStorageFormat, TEXTFILE, TEXTFILE_FIELD_DELIM_KEY); + tableProperties.put(TEXTFILE_FIELD_DELIM_KEY, fieldDelim.toString()); + }); + getSingleCharacterProperty(tableMetadata.getProperties(), TEXTFILE_ESCAPE_DELIM) + .ifPresent(escapeDelim -> { + checkFormatForProperty(hiveStorageFormat, TEXTFILE, TEXTFILE_ESCAPE_DELIM_KEY); + tableProperties.put(TEXTFILE_ESCAPE_DELIM_KEY, escapeDelim.toString()); + }); + getSingleCharacterProperty(tableMetadata.getProperties(), TEXTFILE_COLLECTION_DELIM) + .ifPresent(collectionDelim -> { + checkFormatForProperty(hiveStorageFormat, TEXTFILE, TEXTFILE_COLLECTION_DELIM_KEY); + tableProperties.put(TEXTFILE_COLLECTION_DELIM_KEY, collectionDelim.toString()); + }); + + getSingleCharacterProperty(tableMetadata.getProperties(), TEXTFILE_MAPKEY_DELIM) + .ifPresent(mapKeyDelim -> { + checkFormatForProperty(hiveStorageFormat, TEXTFILE, TEXTFILE_MAPKEY_DELIM_KEY); + tableProperties.put(TEXTFILE_MAPKEY_DELIM_KEY, mapKeyDelim.toString()); + }); + // Table comment property tableMetadata.getComment().ifPresent(value -> tableProperties.put(TABLE_COMMENT, value)); @@ -1461,10 +1509,14 @@ private static Table buildTableObject( } } + Map serdeParameters = extractSerdeParameters(additionalTableParameters); + ImmutableMap.Builder tableParameters = ImmutableMap.builder() .put(PRESTO_VERSION_NAME, prestoVersion) .put(PRESTO_QUERY_ID_NAME, queryId) - .putAll(additionalTableParameters); + .putAll(additionalTableParameters.entrySet().stream() + .filter(entry -> !serdeParameters.containsKey(entry.getKey())) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))); if (tableType.equals(EXTERNAL_TABLE)) { tableParameters.put("EXTERNAL", "TRUE"); @@ -1483,6 +1535,7 @@ private static Table buildTableObject( .setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)) .setBucketProperty(bucketProperty) .setParameters(ImmutableMap.of(PREFERRED_ORDERING_COLUMNS, encodePreferredOrderingColumns(preferredOrderingColumns))) + .setSerdeParameters(serdeParameters) .setLocation(targetPath.toString()); return tableBuilder.build(); @@ -3509,6 +3562,13 @@ private static HiveStorageFormat extractHiveStorageFormat(Table table) throw new PrestoException(HIVE_UNSUPPORTED_FORMAT, format("Output format %s with SerDe %s is not supported", outputFormat, serde)); } + private static Map extractSerdeParameters(Map tableParameters) + { + return tableParameters.entrySet().stream() + .filter(entry -> TEXTFILE_SERDE_KEYS.contains(entry.getKey())) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + } + @VisibleForTesting static String encodePreferredOrderingColumns(List preferredOrderingColumns) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java index 0829404178f31..b62b9e62e513f 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePartitionManager.java @@ -207,7 +207,7 @@ private List getPartitionListFromPartitionNames( { if (isOptimizeParsingOfPartitionValues(session) && partitionNames.size() >= getOptimizeParsingOfPartitionValuesThreshold(session)) { List partitionList = partitionNames.stream() - .map(partitionNameWithVersion -> parsePartition(tableName, partitionNameWithVersion, partitionColumns, partitionTypes, timeZone)) + .map(partitionNameWithVersion -> parsePartition(tableName, partitionNameWithVersion, partitionColumns, partitionTypes)) .collect(toImmutableList()); Map domains = constraint.getSummary().getDomains().get(); @@ -425,6 +425,7 @@ public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastor Table table = getTable(session, metastore, hiveTableHandle, isOfflineDataDebugModeEnabled(session)); List partitionColumns = getPartitionKeyColumnHandles(table); + List partitionColumnTypes = partitionColumns.stream() .map(column -> typeManager.getType(column.getTypeSignature())) .collect(toImmutableList()); @@ -455,7 +456,7 @@ private Optional parseValuesAndFilterPartition( List partitionColumnTypes, Constraint constraint) { - HivePartition partition = parsePartition(tableName, partitionNameWithVersion, partitionColumns, partitionColumnTypes, timeZone); + HivePartition partition = parsePartition(tableName, partitionNameWithVersion, partitionColumns, partitionColumnTypes); Map domains = constraint.getSummary().getDomains().get(); for (HiveColumnHandle column : partitionColumns) { @@ -512,12 +513,11 @@ private List getAllPartitionNames(ConnectorSession ses .orElseThrow(() -> new TableNotFoundException(hiveTableHandle.getSchemaTableName())); } - public static HivePartition parsePartition( + public HivePartition parsePartition( SchemaTableName tableName, PartitionNameWithVersion partitionNameWithVersion, List partitionColumns, - List partitionColumnTypes, - DateTimeZone timeZone) + List partitionColumnTypes) { List partitionColumnNames = partitionColumns.stream() .map(HiveColumnHandle::getName) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 4879d5cb0d4ad..c51b15ced860a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -139,6 +139,8 @@ public final class HiveSessionProperties public static final String NATIVE_STATS_BASED_FILTER_REORDER_DISABLED = "native_stats_based_filter_reorder_disabled"; + public static final String NATIVE_MAX_TARGET_FILE_SIZE = "native_max_target_file_size"; + private final List> sessionProperties; @Inject @@ -676,7 +678,12 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon NATIVE_STATS_BASED_FILTER_REORDER_DISABLED, "Native Execution only. Disable stats based filter reordering.", false, - true)); + true), + dataSizeSessionProperty( + NATIVE_MAX_TARGET_FILE_SIZE, + "Native Execution only. Maximum target file size. When a file exceeds this size during writing, the writer will close the current file and start writing to a new file. Zero means no limit.", + new DataSize(0, DataSize.Unit.BYTE), + false)); } public List> getSessionProperties() diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java index 5c7eb8153a034..6a2d96ea4914a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableProperties.java @@ -57,6 +57,11 @@ public class HiveTableProperties public static final String CSV_SEPARATOR = "csv_separator"; public static final String CSV_QUOTE = "csv_quote"; public static final String CSV_ESCAPE = "csv_escape"; + public static final String TEXTFILE_FIELD_DELIM = "textfile_field_delim"; + public static final String TEXTFILE_MAPKEY_DELIM = "textfile_mapkey_delim"; + public static final String TEXTFILE_COLLECTION_DELIM = "textfile_collection_delim"; + public static final String TEXTFILE_ESCAPE_DELIM = "textfile_escape_delim"; + public static final String SKIP_HEADER_LINE_COUNT = "skip_header_line_count"; public static final String SKIP_FOOTER_LINE_COUNT = "skip_footer_line_count"; @@ -157,6 +162,10 @@ public HiveTableProperties(TypeManager typeManager, HiveClientConfig config) stringProperty(CSV_SEPARATOR, "CSV separator character", null, false), stringProperty(CSV_QUOTE, "CSV quote character", null, false), stringProperty(CSV_ESCAPE, "CSV escape character", null, false), + stringProperty(TEXTFILE_FIELD_DELIM, "Textfile field delimiter character", null, false), + stringProperty(TEXTFILE_ESCAPE_DELIM, "Textfile escape delimiter character", null, false), + stringProperty(TEXTFILE_COLLECTION_DELIM, "Textfile collection delimiter character", null, false), + stringProperty(TEXTFILE_MAPKEY_DELIM, "Textfile map key delimiter character", null, false), integerProperty(SKIP_HEADER_LINE_COUNT, "Number of header lines", null, false), integerProperty(SKIP_FOOTER_LINE_COUNT, "Number of footer lines", null, false), new PropertyMetadata<>( @@ -248,17 +257,17 @@ public static Double getOrcBloomFilterFpp(Map tableProperties) return (Double) tableProperties.get(ORC_BLOOM_FILTER_FPP); } - public static Optional getCsvProperty(Map tableProperties, String key) + public static Optional getSingleCharacterProperty(Map tableProperties, String key) { Object value = tableProperties.get(key); if (value == null) { return Optional.empty(); } - String csvValue = (String) value; - if (csvValue.length() != 1) { - throw new PrestoException(INVALID_TABLE_PROPERTY, format("%s must be a single character string, but was: '%s'", key, csvValue)); + String stringValue = (String) value; + if (stringValue.length() != 1) { + throw new PrestoException(INVALID_TABLE_PROPERTY, format("%s must be a single character string, but was: '%s'", key, stringValue)); } - return Optional.of(csvValue.charAt(0)); + return Optional.of(stringValue.charAt(0)); } @SuppressWarnings("unchecked") diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java index 3ccb4db413a69..d52fd54fc4e4a 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/LegacyAccessControl.java @@ -301,6 +301,16 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con { } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java index b230ffc0d8249..3171f56dbe691 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/SqlStandardAccessControl.java @@ -57,9 +57,11 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddColumn; import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateRole; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -269,6 +271,42 @@ public void checkCanDropBranch(ConnectorTransactionHandle transaction, Connector } } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transaction, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + MetastoreContext metastoreContext = new MetastoreContext( + identity, context.getQueryId().getId(), + context.getClientInfo(), + context.getClientTags(), + context.getSource(), + Optional.empty(), + false, + HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER, + context.getWarningCollector(), + context.getRuntimeStats()); + if (!isTableOwner(transaction, identity, metastoreContext, tableName)) { + denyCreateBranch(tableName.toString()); + } + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transaction, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + MetastoreContext metastoreContext = new MetastoreContext( + identity, context.getQueryId().getId(), + context.getClientInfo(), + context.getClientTags(), + context.getSource(), + Optional.empty(), + false, + HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER, + context.getWarningCollector(), + context.getRuntimeStats()); + if (!isTableOwner(transaction, identity, metastoreContext, tableName)) { + denyCreateTag(tableName.toString()); + } + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transaction, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/security/SystemTableAwareAccessControl.java b/presto-hive/src/main/java/com/facebook/presto/hive/security/SystemTableAwareAccessControl.java index f22bb65c01c30..7dfb340b124e7 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/security/SystemTableAwareAccessControl.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/security/SystemTableAwareAccessControl.java @@ -300,6 +300,18 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con delegate.checkCanDropBranch(transactionHandle, identity, context, tableName); } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + delegate.checkCanCreateBranch(transactionHandle, identity, context, tableName); + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + delegate.checkCanCreateTag(transactionHandle, identity, context, tableName); + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 792818f338678..9d8dc8d05985d 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -44,7 +44,6 @@ import com.facebook.presto.hive.LocationService.WriteInfo; import com.facebook.presto.hive.authentication.NoHdfsAuthentication; import com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory; -import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveColumnStatistics; @@ -52,6 +51,7 @@ import com.facebook.presto.hive.metastore.HivePrivilegeInfo; import com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.MetastoreContext; import com.facebook.presto.hive.metastore.Partition; import com.facebook.presto.hive.metastore.PartitionStatistics; @@ -284,6 +284,7 @@ import static com.facebook.presto.hive.HiveType.toHiveType; import static com.facebook.presto.hive.HiveUtil.columnExtraInfo; import static com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ALL; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; import static com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; @@ -985,6 +986,12 @@ protected final void setup(String host, int port, String databaseName, String ti HiveClientConfig hiveClientConfig = getHiveClientConfig(); CacheConfig cacheConfig = getCacheConfig(); MetastoreClientConfig metastoreClientConfig = getMetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(Duration.valueOf("1m")); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(Duration.valueOf("15s")); + metastoreClientConfig.setMetastoreCacheMaximumSize(10000); + metastoreClientConfig.setEnabledCaches(ALL.name()); + ThriftHiveMetastoreConfig thriftHiveMetastoreConfig = getThriftHiveMetastoreConfig(); hiveClientConfig.setTimeZone(timeZone); String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy"); @@ -998,14 +1005,12 @@ protected final void setup(String host, int port, String databaseName, String ti new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig, hdfsEnvironment), new HivePartitionMutator()), executor, false, - Duration.valueOf("1m"), - Duration.valueOf("15s"), 10000, false, - MetastoreCacheScope.ALL, 0.0, metastoreClientConfig.getPartitionCacheColumnCountLimit(), - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); setup(databaseName, hiveClientConfig, cacheConfig, metastoreClientConfig, metastore); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java index 60ebbf2a978ad..f2636f98ace9a 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveFileSystem.java @@ -26,6 +26,7 @@ import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HivePartitionMutator; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.MetastoreContext; import com.facebook.presto.hive.metastore.MetastoreOperationResult; import com.facebook.presto.hive.metastore.PrincipalPrivileges; @@ -508,7 +509,7 @@ public static class TestingHiveMetastore public TestingHiveMetastore(ExtendedHiveMetastore delegate, ExecutorService executor, MetastoreClientConfig metastoreClientConfig, Path basePath, HdfsEnvironment hdfsEnvironment) { - super(delegate, executor, NOOP_METASTORE_CACHE_STATS, metastoreClientConfig); + super(delegate, executor, NOOP_METASTORE_CACHE_STATS, metastoreClientConfig, new MetastoreCacheSpecProvider(metastoreClientConfig)); this.basePath = basePath; this.hdfsEnvironment = hdfsEnvironment; } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java index 4bed0b39e8b36..a8aa552763d44 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java @@ -343,11 +343,11 @@ public void testHistoryBasedStatsCalculatorCTE() .setSystemProperty(CTE_PARTITIONING_PROVIDER_CATALOG, "hive") .build(); // CBO Statistics - assertPlan(cteMaterialization, sql, anyTree(node(ProjectNode.class, anyTree(any())).withOutputRowCount(Double.NaN))); + assertPlan(cteMaterialization, sql, anyTree(node(ProjectNode.class, anyTree(any())).withOutputRowCount(0D))); // HBO Statistics executeAndTrackHistory(sql, cteMaterialization); - assertPlan(cteMaterialization, sql, anyTree(node(ProjectNode.class, anyTree(any())).withOutputRowCount(3))); + assertPlan(cteMaterialization, sql, anyTree(node(ProjectNode.class, anyTree(any())).withOutputRowCount(3D))); } @Test diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index fe8e7f6a225e6..7a7b8de866c99 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; import com.facebook.presto.spi.plan.MarkDistinctNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.SelectedRole; @@ -38,7 +39,6 @@ import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.ColumnConstraint; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.FormattedDomain; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.FormattedMarker; @@ -7090,6 +7090,157 @@ public void testInsertTableWithHeaderAndFooterForCsv() assertUpdate("DROP TABLE csv_table_skip_header"); } + @Test + public void testSerdeParametersForTextfileRead() + throws Exception + { + File tempDir = createTempDir(); + File dataFile = new File(tempDir, "custom-delim.txt"); + Files.write( + "1001" + + "|he\u0001|llo" + + "|true" + + "|88.5" + + "|alpha;beta;gamma" + + "|size:large;color:blue" + + "|42;1.1:2.2:3.3;20\u0004bar:10\u0004foo\n", dataFile, UTF_8); + + String catalog = getSession().getCatalog().get(); + String schema = getSession().getSchema().get(); + String table = "test_textfile_custom_delim"; + String path = new Path(tempDir.toURI().toASCIIString()).toString(); + + String createTableWithCustomSerdeFormat = + "CREATE TABLE %s.%s.%s (\n" + + " %s bigint,\n" + + " %s varchar,\n" + + " %s boolean,\n" + + " %s double,\n" + + " %s array(varchar),\n" + + " %s map(varchar, varchar),\n" + + " %s row(%s integer, %s array(real), %s map(smallint, varchar))\n" + + ")\n" + + "WITH (\n" + + " external_location = '%s',\n" + + " format = 'TEXTFILE',\n" + + " textfile_collection_delim = ';',\n" + + " textfile_escape_delim = %s,\n" + + " textfile_field_delim = '|',\n" + + " textfile_mapkey_delim = ':'\n" + + ")"; + + @Language("SQL") String createTableSql = format( + createTableWithCustomSerdeFormat, + catalog, schema, table, + "c1", "c2", "c3", "c4", "c5", "c6", "c7", + "s_int", "s_arr", "s_map", + path, + "'\u0001'"); + + String expectedCreateTableSql = format( + createTableWithCustomSerdeFormat, + catalog, schema, table, + "\"c1\"", "\"c2\"", "\"c3\"", "\"c4\"", "\"c5\"", "\"c6\"", "\"c7\"", + "\"s_int\"", "\"s_arr\"", "\"s_map\"", + path, + "U&'\\0001'"); + + try { + assertUpdate(createTableSql); + + MaterializedResult actualCreateTableSql = computeActual(format("SHOW CREATE TABLE %s.%s.%s", catalog, schema, table)); + assertEquals(actualCreateTableSql.getOnlyValue(), expectedCreateTableSql); + + assertQuery( + format( + "SELECT\n" + + "c1, c2, c3, c4, c5, \n" + + "element_at(c6, 'size'), element_at(c6, 'color'), \n" + + "c7.s_arr, element_at(c7.s_map, 10), element_at(c7.s_map, 20) FROM %s.%s.%s", catalog, schema, table), + "VALUES(" + + "1001, 'he|llo', true, 88.5, \n" + + "ARRAY['alpha', 'beta', 'gamma'], \n" + + "'large', 'blue', \n" + + "ARRAY[CAST(1.1 AS REAL), CAST(2.2 AS REAL), CAST(3.3 AS REAL)], 'foo', 'bar')"); + } + finally { + assertUpdate(format("DROP TABLE IF EXISTS %s.%s.%s", catalog, schema, table)); + deleteRecursively(tempDir.toPath(), ALLOW_INSECURE); + } + } + + @Test + public void testSerdeParametersForTextfileWrite() + { + String catalog = getSession().getCatalog().get(); + String schema = getSession().getSchema().get(); + String table = "test_textfile_custom_delim"; + + String createTableWithCustomSerdeFormat = + "CREATE TABLE %s.%s.%s (\n" + + " %s bigint,\n" + + " %s varchar,\n" + + " %s boolean,\n" + + " %s double,\n" + + " %s array(varchar),\n" + + " %s map(varchar, varchar),\n" + + " %s row(%s integer, %s array(real), %s map(smallint, varchar))\n" + + ")\n" + + "WITH (\n" + + " format = 'TEXTFILE',\n" + + " textfile_collection_delim = ';',\n" + + " textfile_escape_delim = %s,\n" + + " textfile_field_delim = '|',\n" + + " textfile_mapkey_delim = ':'\n" + + ")"; + + @Language("SQL") String createTableSql = format( + createTableWithCustomSerdeFormat, + catalog, schema, table, + "c1", "c2", "c3", "c4", "c5", "c6", "c7", + "s_int", "s_arr", "s_map", + "'\u0001'"); + + String expectedCreateTableSql = format( + createTableWithCustomSerdeFormat, + catalog, schema, table, + "\"c1\"", "\"c2\"", "\"c3\"", "\"c4\"", "\"c5\"", "\"c6\"", "\"c7\"", + "\"s_int\"", "\"s_arr\"", "\"s_map\"", + "U&'\\0001'"); + + try { + assertUpdate(createTableSql); + + MaterializedResult actualCreateTableSql = computeActual(format("SHOW CREATE TABLE %s.%s.%s", catalog, schema, table)); + assertEquals(actualCreateTableSql.getOnlyValue(), expectedCreateTableSql); + + assertUpdate(format( + "INSERT INTO %s.%s.%s VALUES (" + + "1001, " + + "'he|llo', " + + "true, " + + "88.5, " + + "ARRAY['alpha','beta', 'gamma'], " + + "MAP(ARRAY['size', 'color'], ARRAY['large', 'blue']), " + + "ROW(42, ARRAY[REAL '1.1', REAL '2.2',REAL '3.3'], MAP(ARRAY[SMALLINT '10', SMALLINT '20'], ARRAY['foo', 'bar'])))", catalog, schema, table), 1); + + assertQuery( + format( + "SELECT\n" + + "c1, c2, c3, c4, c5, \n" + + "element_at(c6, 'size'), element_at(c6, 'color'), \n" + + "c7.s_arr, element_at(c7.s_map, 10), element_at(c7.s_map, 20) FROM %s.%s.%s", catalog, schema, table), + "VALUES(" + + "1001, 'he|llo', true, 88.5, \n" + + "ARRAY['alpha', 'beta', 'gamma'], \n" + + "'large', 'blue', \n" + + "ARRAY[CAST(1.1 AS REAL), CAST(2.2 AS REAL), CAST(3.3 AS REAL)], 'foo', 'bar')"); + } + finally { + assertUpdate(format("DROP TABLE IF EXISTS %s.%s.%s", catalog, schema, table)); + } + } + protected String retentionDays(int days) { return ""; diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index 534beafe5a0b6..3fec92f66e088 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -84,6 +84,7 @@ import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_METADATA_QUERIES_IGNORE_STATS; import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_DEREFERENCE_ENABLED; import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_SUBFIELDS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_SUBFIELDS_FOR_CARDINALITY; import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_SUBFIELDS_FOR_MAP_FUNCTIONS; import static com.facebook.presto.SystemSessionProperties.UTILIZE_UNIQUE_PROPERTY_IN_QUERY_PLANNING; import static com.facebook.presto.common.function.OperatorType.EQUAL; @@ -1637,6 +1638,56 @@ public void testPushdownSubfieldsForMapFilter() assertUpdate("DROP TABLE test_pushdown_map_subfields"); } + @Test + public void testPushdownSubfieldsForCardinality() + { + Session cardinalityPushdown = Session.builder(getSession()) + .setSystemProperty(PUSHDOWN_SUBFIELDS_FOR_CARDINALITY, "true") + .build(); + + // Test simple cardinality pushdown for MAP + assertUpdate("CREATE TABLE test_pushdown_cardinality_map(id integer, x map(integer, double))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT t.id, cardinality(x) FROM test_pushdown_cardinality_map t", "test_pushdown_cardinality_map", + ImmutableMap.of("x", toSubfields("x[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_map"); + + // Test cardinality pushdown for ARRAY + assertUpdate("CREATE TABLE test_pushdown_cardinality_array(id integer, arr array(bigint))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT t.id, cardinality(arr) FROM test_pushdown_cardinality_array t", "test_pushdown_cardinality_array", + ImmutableMap.of("arr", toSubfields("arr[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_array"); + + // Test cardinality in WHERE clause + assertUpdate("CREATE TABLE test_pushdown_cardinality_where(id integer, features map(varchar, double))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT t.id FROM test_pushdown_cardinality_where t WHERE cardinality(features) > 10", "test_pushdown_cardinality_where", + ImmutableMap.of("features", toSubfields("features[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_where"); + + // Test cardinality in aggregation + assertUpdate("CREATE TABLE test_pushdown_cardinality_agg(id integer, data map(integer, varchar))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT AVG(cardinality(data)) FROM test_pushdown_cardinality_agg", "test_pushdown_cardinality_agg", + ImmutableMap.of("data", toSubfields("data[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_agg"); + + // Test multiple cardinalities + assertUpdate("CREATE TABLE test_pushdown_cardinality_multi(id integer, map1 map(integer, double), map2 map(varchar, integer))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT cardinality(map1), cardinality(map2) FROM test_pushdown_cardinality_multi", "test_pushdown_cardinality_multi", + ImmutableMap.of("map1", toSubfields("map1[$]"), "map2", toSubfields("map2[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_multi"); + + // Test cardinality with complex expression + assertUpdate("CREATE TABLE test_pushdown_cardinality_expr(id integer, tags map(varchar, varchar))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT cardinality(tags) * 2 FROM test_pushdown_cardinality_expr", "test_pushdown_cardinality_expr", + ImmutableMap.of("tags", toSubfields("tags[$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_expr"); + + // Test cardinality on ARRAY of maps + assertUpdate("CREATE TABLE test_pushdown_cardinality_nested(id integer, arr_of_maps array(map(integer, varchar)))"); + assertPushdownSubfields(cardinalityPushdown, "SELECT transform(arr_of_maps, m -> cardinality(m)) FROM test_pushdown_cardinality_nested", "test_pushdown_cardinality_nested", + ImmutableMap.of("arr_of_maps", toSubfields("arr_of_maps[*][$]"))); + assertUpdate("DROP TABLE test_pushdown_cardinality_nested"); + } + @Test public void testPushdownSubfieldsAssorted() { diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java index 6767fc4d96b9c..8a1f9ea62a3c9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMaterializedViewLogicalPlanner.java @@ -2169,6 +2169,11 @@ public void testMaterializedViewOptimizationWithUnsupportedFunctionSubquery() assertPlan(queryOptimizationWithMaterializedView, baseQuery, anyTree( node(JoinNode.class, + exchange( + anyTree( + constrainedTableScan(table, + ImmutableMap.of(), + ImmutableMap.of()))), anyTree( exchange( anyTree( @@ -2178,12 +2183,7 @@ public void testMaterializedViewOptimizationWithUnsupportedFunctionSubquery() anyTree( constrainedTableScan(view2, ImmutableMap.of(), - ImmutableMap.of("ds_42", "ds", "orderkey_41", "orderkey"))))), - exchange( - anyTree( - constrainedTableScan(table, - ImmutableMap.of(), - ImmutableMap.of())))))); + ImmutableMap.of("ds_42", "ds", "orderkey_41", "orderkey")))))))); } finally { queryRunner.execute("DROP MATERIALIZED VIEW IF EXISTS " + view2); @@ -2921,6 +2921,7 @@ public void testInsertBySelectingFromMaterializedView() String table1 = "orders_partitioned_source"; String table2 = "orders_partitioned_target"; String table3 = "orders_from_mv"; + String table4 = "orders_from_refreshed_mv"; String view = "test_orders_view"; try { queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds']) AS " + @@ -2935,18 +2936,31 @@ public void testInsertBySelectingFromMaterializedView() assertUpdate(format("CREATE TABLE %s AS SELECT * FROM %s WHERE 1=0", table2, table1), 0); assertTrue(getQueryRunner().tableExists(getSession(), table2)); - assertQueryFails(format("CREATE TABLE %s AS SELECT * FROM %s", table3, view), - ".*CreateTableAsSelect by selecting from a materialized view \\w+ is not supported.*"); + // CTAS from a materialized view should succeed (MV is not yet refreshed so storage is empty) + assertUpdate(format("CREATE TABLE %s AS SELECT * FROM %s WHERE ds = '2020-01-01'", table3, view), 0); + assertTrue(getQueryRunner().tableExists(getSession(), table3)); + + // Refresh the MV so it has data, then CTAS should read from the refreshed MV + assertUpdate(format("REFRESH MATERIALIZED VIEW %s WHERE ds = '2020-01-01'", view), 255); + assertUpdate(format("CREATE TABLE %s AS SELECT * FROM %s WHERE ds = '2020-01-01'", table4, view), 255); + assertTrue(getQueryRunner().tableExists(getSession(), table4)); assertUpdate(format("INSERT INTO %s VALUES(99999, '1-URGENT', '2019-01-02')", table2), 1); assertUpdate(format("INSERT INTO %s SELECT * FROM %s WHERE ds = '2020-01-01'", table2, table1), 255); - assertQueryFails(format("INSERT INTO %s SELECT * FROM %s WHERE ds = '2020-01-01'", table2, view), - ".*Insert by selecting from a materialized view \\w+ is not supported.*"); + + // INSERT from MV into a non-base-table should succeed + assertUpdate(format("INSERT INTO %s SELECT * FROM %s WHERE ds = '2020-01-01'", table2, view), 255); + + // INSERT from MV into one of its base tables should fail (circular dependency) + assertQueryFails(format("INSERT INTO %s SELECT * FROM %s WHERE ds = '2020-01-01'", table1, view), + ".*INSERT into table .* by selecting from materialized view .* is not supported because .* is a base table of the materialized view.*"); } finally { queryRunner.execute("DROP MATERIALIZED VIEW IF EXISTS " + view); queryRunner.execute("DROP TABLE IF EXISTS " + table1); queryRunner.execute("DROP TABLE IF EXISTS " + table2); + queryRunner.execute("DROP TABLE IF EXISTS " + table3); + queryRunner.execute("DROP TABLE IF EXISTS " + table4); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestInMemoryCachingHiveMetastore.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestInMemoryCachingHiveMetastore.java index 9e4c4be57add6..f3fe7dee3b7c4 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestInMemoryCachingHiveMetastore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/TestInMemoryCachingHiveMetastore.java @@ -18,7 +18,6 @@ import com.facebook.presto.hive.MockHiveMetastore; import com.facebook.presto.hive.PartitionMutator; import com.facebook.presto.hive.PartitionNameWithVersion; -import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; import com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore; import com.facebook.presto.hive.metastore.thrift.HiveCluster; import com.facebook.presto.hive.metastore.thrift.HiveMetastoreClient; @@ -45,6 +44,10 @@ import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; import static com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ALL; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.PARTITION_STATISTICS; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE; import static com.facebook.presto.hive.metastore.NoopMetastoreCacheStats.NOOP_METASTORE_CACHE_STATS; import static com.facebook.presto.hive.metastore.Partition.Builder; import static com.facebook.presto.hive.metastore.thrift.MockHiveMetastoreClient.BAD_DATABASE; @@ -77,7 +80,8 @@ public class TestInMemoryCachingHiveMetastore private static final ImmutableList EXPECTED_PARTITIONS = ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2); private MockHiveMetastoreClient mockClient; - private InMemoryCachingHiveMetastore metastore; + private InMemoryCachingHiveMetastore metastoreWithAllCachesEnabled; + private InMemoryCachingHiveMetastore metastoreWithSelectiveCachesEnabled; private ThriftHiveMetastoreStats stats; @BeforeMethod @@ -87,20 +91,44 @@ public void setUp() MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient); ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s"))); MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfig.setMetastoreCacheMaximumSize(1000); + metastoreClientConfig.setEnabledCaches(ALL.name()); + ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(mockHiveCluster, metastoreClientConfig, HDFS_ENVIRONMENT); PartitionMutator hivePartitionMutator = new HivePartitionMutator(); - metastore = new InMemoryCachingHiveMetastore( + metastoreWithAllCachesEnabled = new InMemoryCachingHiveMetastore( new BridgingHiveMetastore(thriftHiveMetastore, hivePartitionMutator), executor, false, - new Duration(5, TimeUnit.MINUTES), - new Duration(1, TimeUnit.MINUTES), 1000, false, - MetastoreCacheScope.ALL, 0.0, metastoreClientConfig.getPartitionCacheColumnCountLimit(), - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); + + MetastoreClientConfig metastoreClientConfigWithSelectiveCaching = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfigWithSelectiveCaching.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfigWithSelectiveCaching.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfigWithSelectiveCaching.setMetastoreCacheMaximumSize(1000); + metastoreClientConfigWithSelectiveCaching.setDisabledCaches(TABLE.name()); + + ThriftHiveMetastore thriftHiveMetastoreWithSelectiveCaching = new ThriftHiveMetastore(mockHiveCluster, metastoreClientConfigWithSelectiveCaching, HDFS_ENVIRONMENT); + metastoreWithSelectiveCachesEnabled = new InMemoryCachingHiveMetastore( + new BridgingHiveMetastore(thriftHiveMetastoreWithSelectiveCaching, hivePartitionMutator), + executor, + false, + 1000, + false, + 0.0, + metastoreClientConfigWithSelectiveCaching.getPartitionCacheColumnCountLimit(), + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfigWithSelectiveCaching)); + stats = thriftHiveMetastore.getStats(); } @@ -108,19 +136,19 @@ public void setUp() public void testGetAllDatabases() { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertEquals(metastore.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); + assertEquals(metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 2); // Test invalidate a specific database - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); - assertEquals(metastore.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); + assertEquals(metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT), ImmutableList.of(TEST_DATABASE)); assertEquals(mockClient.getAccessCount(), 3); } @@ -128,69 +156,94 @@ public void testGetAllDatabases() public void testGetAllTable() { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 2); // Test invalidate a specific database which will also invalidate all table caches mapped to that database - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 3); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 3); // Test invalidate a specific database.table which also invalidates the tablesNamesCache for that database - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); - assertEquals(metastore.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); + assertEquals(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 4); } + @Test + public void testGetAllTableWithSelectiveCaching() + { + assertEquals(mockClient.getAccessCount(), 0); + assertEquals(metastoreWithSelectiveCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(mockClient.getAccessCount(), 1); + assertEquals(metastoreWithSelectiveCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(mockClient.getAccessCount(), 1); + + metastoreWithSelectiveCachesEnabled.invalidateAll(); + + assertEquals(metastoreWithSelectiveCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, TEST_DATABASE).get(), ImmutableList.of(TEST_TABLE, TEST_TABLE_WITH_CONSTRAINTS)); + assertEquals(mockClient.getAccessCount(), 2); + } + public void testInvalidDbGetAllTAbles() { - assertFalse(metastore.getAllTables(TEST_METASTORE_CONTEXT, BAD_DATABASE).isPresent()); + assertFalse(metastoreWithAllCachesEnabled.getAllTables(TEST_METASTORE_CONTEXT, BAD_DATABASE).isPresent()); } @Test public void testGetTable() { assertEquals(mockClient.getAccessCount(), 0); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 2); // Test invalidate a specific database which will also invalidate all table caches mapped to that database - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 3); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 3); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 4); // Test invalidate a specific table - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS)); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS)); assertEquals(mockClient.getAccessCount(), 4); - assertNotNull(metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertNotNull(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); assertEquals(mockClient.getAccessCount(), 5); } + @Test + public void testGetTableWithSelectiveCaching() + { + assertEquals(mockClient.getAccessCount(), 0); + assertNotNull(metastoreWithSelectiveCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertEquals(mockClient.getAccessCount(), 1); + assertNotNull(metastoreWithSelectiveCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE)); + assertEquals(mockClient.getAccessCount(), 2); + } + public void testInvalidDbGetTable() { - assertFalse(metastore.getTable(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE).isPresent()); + assertFalse(metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE).isPresent()); assertEquals(stats.getGetTable().getThriftExceptions().getTotalCount(), 0); assertEquals(stats.getGetTable().getTotalFailures().getTotalCount(), 0); @@ -202,33 +255,33 @@ public void testGetPartitionNames() { ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION_NAME_WITHOUT_VERSION1, TEST_PARTITION_NAME_WITHOUT_VERSION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); // Test invalidate the database which will also invalidate all linked table and partition caches - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 3); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 3); // Test invalidate a specific table which will also invalidate all linked partition caches - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 4); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 4); // Test invalidate a specific partition - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of("testpartition1")); - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of("testpartition1")); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE).get(), expectedPartitions); assertEquals(mockClient.getAccessCount(), 5); } @@ -236,27 +289,27 @@ public void testGetPartitionNames() public void testInvalidInvalidateCache() { // Test invalidate cache with null/empty database name - assertThatThrownBy(() -> metastore.invalidateCache(TEST_METASTORE_CONTEXT, null)) + assertThatThrownBy(() -> metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("databaseName cannot be null or empty"); // Test invalidate cache with null/empty table name - assertThatThrownBy(() -> metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, null)) + assertThatThrownBy(() -> metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("tableName cannot be null or empty"); // Test invalidate cache with invalid/empty partition columns list - assertThatThrownBy(() -> metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(), ImmutableList.of())) + assertThatThrownBy(() -> metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(), ImmutableList.of())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("partitionColumnNames cannot be null or empty"); // Test invalidate cache with invalid/empty partition values list - assertThatThrownBy(() -> metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of())) + assertThatThrownBy(() -> metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("partitionValues cannot be null or empty"); // Test invalidate cache with mismatched partition columns and values list - assertThatThrownBy(() -> metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key1", "key2"), ImmutableList.of("testpartition1"))) + assertThatThrownBy(() -> metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key1", "key2"), ImmutableList.of("testpartition1"))) .isInstanceOf(IllegalArgumentException.class) .hasMessage("partitionColumnNames and partitionValues should be of same length"); } @@ -264,7 +317,7 @@ public void testInvalidInvalidateCache() @Test public void testInvalidGetPartitionNames() { - assertEquals(metastore.getPartitionNames(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNames(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE).get(), ImmutableList.of()); } @Test @@ -273,14 +326,14 @@ public void testGetPartitionNamesByParts() ImmutableList expectedPartitions = ImmutableList.of(TEST_PARTITION_NAME_WITHOUT_VERSION1, TEST_PARTITION_NAME_WITHOUT_VERSION2); assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertEquals(metastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); + assertEquals(metastoreWithAllCachesEnabled.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), expectedPartitions); assertEquals(mockClient.getAccessCount(), 2); } @@ -311,18 +364,23 @@ public void testCachingWithPartitionVersioning() ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s"))); MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster); PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfig.setMetastoreCacheMaximumSize(1000); + metastoreClientConfig.setEnabledCaches(String.join(",", PARTITION.name(), PARTITION_STATISTICS.name())); + InMemoryCachingHiveMetastore partitionCachingEnabledmetastore = new InMemoryCachingHiveMetastore( new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, - new Duration(5, TimeUnit.MINUTES), - new Duration(1, TimeUnit.MINUTES), 1000, true, - MetastoreCacheScope.PARTITION, 0.0, 10_000, - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); assertEquals(mockClient.getAccessCount(), 0); assertEquals(partitionCachingEnabledmetastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableMap.of()), EXPECTED_PARTITIONS); @@ -361,18 +419,23 @@ private void assertInvalidateCache(MockPartitionMutator partitionMutator, Functi MockHiveCluster mockHiveCluster = new MockHiveCluster(mockClient); ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s"))); MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfig.setMetastoreCacheMaximumSize(1000); + metastoreClientConfig.setEnabledCaches(String.join(",", PARTITION.name(), PARTITION_STATISTICS.name())); + InMemoryCachingHiveMetastore partitionCachingEnabledmetastore = new InMemoryCachingHiveMetastore( new BridgingHiveMetastore(mockHiveMetastore, partitionMutator), executor, false, - new Duration(5, TimeUnit.MINUTES), - new Duration(1, TimeUnit.MINUTES), 1000, true, - MetastoreCacheScope.PARTITION, 0.0, 10_000, - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); int clientAccessCount = 0; for (int i = 0; i < 100; i++) { @@ -388,7 +451,7 @@ private void assertInvalidateCache(MockPartitionMutator partitionMutator, Functi public void testInvalidGetPartitionNamesByParts() { - assertTrue(metastore.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE, ImmutableMap.of()).isEmpty()); + assertTrue(metastoreWithAllCachesEnabled.getPartitionNamesByFilter(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE, ImmutableMap.of()).isEmpty()); } @Test @@ -399,18 +462,23 @@ public void testPartitionCacheValidation() ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s"))); MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster); PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfig.setMetastoreCacheMaximumSize(1000); + metastoreClientConfig.setEnabledCaches(String.join(",", PARTITION.name(), PARTITION_STATISTICS.name())); + InMemoryCachingHiveMetastore partitionCacheVerificationEnabledMetastore = new InMemoryCachingHiveMetastore( new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, - new Duration(5, TimeUnit.MINUTES), - new Duration(1, TimeUnit.MINUTES), 1000, true, - MetastoreCacheScope.PARTITION, 100.0, 10_000, - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); // Warmup the cache partitionCacheVerificationEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)); @@ -430,19 +498,24 @@ public void testPartitionCacheColumnCountLimit() ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("partition-versioning-test-%s"))); MockHiveMetastore mockHiveMetastore = new MockHiveMetastore(mockHiveCluster); PartitionMutator mockPartitionMutator = new MockPartitionMutator(identity()); + MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig(); + // Configure Metastore Cache + metastoreClientConfig.setDefaultMetastoreCacheTtl(new Duration(5, TimeUnit.MINUTES)); + metastoreClientConfig.setDefaultMetastoreCacheRefreshInterval(new Duration(1, TimeUnit.MINUTES)); + metastoreClientConfig.setMetastoreCacheMaximumSize(1000); + metastoreClientConfig.setEnabledCaches(String.join(",", PARTITION.name(), PARTITION_STATISTICS.name())); + InMemoryCachingHiveMetastore partitionCachingEnabledMetastore = new InMemoryCachingHiveMetastore( new BridgingHiveMetastore(mockHiveMetastore, mockPartitionMutator), executor, false, - new Duration(5, TimeUnit.MINUTES), - new Duration(1, TimeUnit.MINUTES), 1000, true, - MetastoreCacheScope.PARTITION, 0.0, // set the cached partition column count limit as 1 for testing purpose 1, - NOOP_METASTORE_CACHE_STATS); + NOOP_METASTORE_CACHE_STATS, + new MetastoreCacheSpecProvider(metastoreClientConfig)); // Select all of the available partitions. Normally they would have been loaded into the cache. But because of column count limit, they will not be cached assertEquals(partitionCachingEnabledMetastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); @@ -461,43 +534,43 @@ public void testPartitionCacheColumnCountLimit() public void testGetPartitionsByNames() { assertEquals(mockClient.getAccessCount(), 0); - metastore.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); + metastoreWithAllCachesEnabled.getTable(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); assertEquals(mockClient.getAccessCount(), 1); // Select half of the available partitions and load them into the cache - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); assertEquals(mockClient.getAccessCount(), 2); // Now select all of the partitions - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); // There should be one more access to fetch the remaining partition assertEquals(mockClient.getAccessCount(), 3); // Now if we fetch any or both of them, they should not hit the client - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION2)).size(), 1); - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION2)).size(), 1); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); assertEquals(mockClient.getAccessCount(), 3); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); // Fetching both should only result in one batched access - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); assertEquals(mockClient.getAccessCount(), 4); // Test invalidate a specific partition - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of("testpartition1")); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of("key"), ImmutableList.of("testpartition1")); // This should still be a cache hit - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION2)).size(), 1); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION2)).size(), 1); assertEquals(mockClient.getAccessCount(), 4); // This should be a cache miss - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)).size(), 1); assertEquals(mockClient.getAccessCount(), 5); // This should be a cache hit - assertEquals(metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); + assertEquals(metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1, TEST_PARTITION_NAME_WITH_VERSION2)).size(), 2); assertEquals(mockClient.getAccessCount(), 5); } @@ -507,31 +580,31 @@ public void testListRoles() { assertEquals(mockClient.getAccessCount(), 0); - assertEquals(metastore.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); + assertEquals(metastoreWithAllCachesEnabled.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); assertEquals(mockClient.getAccessCount(), 1); - assertEquals(metastore.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); + assertEquals(metastoreWithAllCachesEnabled.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); assertEquals(mockClient.getAccessCount(), 1); - metastore.invalidateAll(); + metastoreWithAllCachesEnabled.invalidateAll(); - assertEquals(metastore.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); + assertEquals(metastoreWithAllCachesEnabled.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); assertEquals(mockClient.getAccessCount(), 2); - metastore.createRole(TEST_METASTORE_CONTEXT, "role", "grantor"); + metastoreWithAllCachesEnabled.createRole(TEST_METASTORE_CONTEXT, "role", "grantor"); - assertEquals(metastore.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); + assertEquals(metastoreWithAllCachesEnabled.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); assertEquals(mockClient.getAccessCount(), 3); - metastore.dropRole(TEST_METASTORE_CONTEXT, "testrole"); + metastoreWithAllCachesEnabled.dropRole(TEST_METASTORE_CONTEXT, "testrole"); - assertEquals(metastore.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); + assertEquals(metastoreWithAllCachesEnabled.listRoles(TEST_METASTORE_CONTEXT), TEST_ROLES); assertEquals(mockClient.getAccessCount(), 4); } public void testInvalidGetPartitionsByNames() { - Map> partitionsByNames = metastore.getPartitionsByNames(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)); + Map> partitionsByNames = metastoreWithAllCachesEnabled.getPartitionsByNames(TEST_METASTORE_CONTEXT, BAD_DATABASE, TEST_TABLE, ImmutableList.of(TEST_PARTITION_NAME_WITH_VERSION1)); assertEquals(partitionsByNames.size(), 1); Optional onlyElement = Iterables.getOnlyElement(partitionsByNames.values()); assertFalse(onlyElement.isPresent()); @@ -543,7 +616,7 @@ public void testNoCacheExceptions() // Throw exceptions on usage mockClient.setThrowException(true); try { - metastore.getAllDatabases(TEST_METASTORE_CONTEXT); + metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT); } catch (RuntimeException ignored) { } @@ -551,7 +624,7 @@ public void testNoCacheExceptions() // Second try should hit the client again try { - metastore.getAllDatabases(TEST_METASTORE_CONTEXT); + metastoreWithAllCachesEnabled.getAllDatabases(TEST_METASTORE_CONTEXT); } catch (RuntimeException ignored) { } @@ -562,25 +635,25 @@ public void testNoCacheExceptions() public void testTableConstraints() { assertEquals(mockClient.getAccessCount(), 0); - List> tableConstraints = metastore.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + List> tableConstraints = metastoreWithAllCachesEnabled.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); assertEquals(tableConstraints.get(0), new PrimaryKeyConstraint<>(Optional.of("pk"), new LinkedHashSet<>(ImmutableList.of("c1")), true, true, false)); assertEquals(tableConstraints.get(1), new UniqueConstraint<>(Optional.of("uk"), new LinkedHashSet<>(ImmutableList.of("c2")), true, true, false)); assertEquals(tableConstraints.get(2), new NotNullConstraint<>("c3")); assertEquals(mockClient.getAccessCount(), 3); - metastore.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + metastoreWithAllCachesEnabled.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); assertEquals(mockClient.getAccessCount(), 3); - metastore.invalidateAll(); - metastore.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + metastoreWithAllCachesEnabled.invalidateAll(); + metastoreWithAllCachesEnabled.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); assertEquals(mockClient.getAccessCount(), 6); // Test invalidate TEST_TABLE, which should not affect any entries linked to TEST_TABLE_WITH_CONSTRAINTS - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); - metastore.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE); + metastoreWithAllCachesEnabled.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); assertEquals(mockClient.getAccessCount(), 6); // Test invalidate TEST_TABLE_WITH_CONSTRAINTS - metastore.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); - metastore.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + metastoreWithAllCachesEnabled.invalidateCache(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); + metastoreWithAllCachesEnabled.getTableConstraints(TEST_METASTORE_CONTEXT, TEST_DATABASE, TEST_TABLE_WITH_CONSTRAINTS); assertEquals(mockClient.getAccessCount(), 9); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueInputConverter.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueInputConverter.java index ef5d5213bd9f2..967908b8c83c8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueInputConverter.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestGlueInputConverter.java @@ -13,10 +13,6 @@ */ package com.facebook.presto.hive.metastore.glue; -import com.amazonaws.services.glue.model.DatabaseInput; -import com.amazonaws.services.glue.model.PartitionInput; -import com.amazonaws.services.glue.model.StorageDescriptor; -import com.amazonaws.services.glue.model.TableInput; import com.facebook.presto.hive.HiveBucketProperty; import com.facebook.presto.hive.metastore.Column; import com.facebook.presto.hive.metastore.Database; @@ -26,6 +22,10 @@ import com.facebook.presto.hive.metastore.glue.converter.GlueInputConverter; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; +import software.amazon.awssdk.services.glue.model.DatabaseInput; +import software.amazon.awssdk.services.glue.model.PartitionInput; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.TableInput; import java.util.List; @@ -46,10 +46,10 @@ public void testConvertDatabase() { DatabaseInput dbInput = GlueInputConverter.convertDatabase(testDb); - assertEquals(dbInput.getName(), testDb.getDatabaseName()); - assertEquals(dbInput.getDescription(), testDb.getComment().get()); - assertEquals(dbInput.getLocationUri(), testDb.getLocation().get()); - assertEquals(dbInput.getParameters(), testDb.getParameters()); + assertEquals(dbInput.name(), testDb.getDatabaseName()); + assertEquals(dbInput.description(), testDb.getComment().get()); + assertEquals(dbInput.locationUri(), testDb.getLocation().get()); + assertEquals(dbInput.parameters(), testDb.getParameters()); } @Test @@ -57,15 +57,15 @@ public void testConvertTable() { TableInput tblInput = GlueInputConverter.convertTable(testTbl); - assertEquals(tblInput.getName(), testTbl.getTableName()); - assertEquals(tblInput.getOwner(), testTbl.getOwner()); - assertEquals(tblInput.getTableType(), testTbl.getTableType().toString()); - assertEquals(tblInput.getParameters(), testTbl.getParameters()); - assertColumnList(tblInput.getStorageDescriptor().getColumns(), testTbl.getDataColumns()); - assertColumnList(tblInput.getPartitionKeys(), testTbl.getPartitionColumns()); - assertStorage(tblInput.getStorageDescriptor(), testTbl.getStorage()); - assertEquals(tblInput.getViewExpandedText(), testTbl.getViewExpandedText().get()); - assertEquals(tblInput.getViewOriginalText(), testTbl.getViewOriginalText().get()); + assertEquals(tblInput.name(), testTbl.getTableName()); + assertEquals(tblInput.owner(), testTbl.getOwner()); + assertEquals(tblInput.tableType(), testTbl.getTableType().toString()); + assertEquals(tblInput.parameters(), testTbl.getParameters()); + assertColumnList(tblInput.storageDescriptor().columns(), testTbl.getDataColumns()); + assertColumnList(tblInput.partitionKeys(), testTbl.getPartitionColumns()); + assertStorage(tblInput.storageDescriptor(), testTbl.getStorage()); + assertEquals(tblInput.viewExpandedText(), testTbl.getViewExpandedText().get()); + assertEquals(tblInput.viewOriginalText(), testTbl.getViewOriginalText().get()); } @Test @@ -73,12 +73,12 @@ public void testConvertPartition() { PartitionInput partitionInput = GlueInputConverter.convertPartition(testPartition); - assertEquals(partitionInput.getParameters(), testPartition.getParameters()); - assertStorage(partitionInput.getStorageDescriptor(), testPartition.getStorage()); - assertEquals(partitionInput.getValues(), testPartition.getValues()); + assertEquals(partitionInput.parameters(), testPartition.getParameters()); + assertStorage(partitionInput.storageDescriptor(), testPartition.getStorage()); + assertEquals(partitionInput.values(), testPartition.getValues()); } - private static void assertColumnList(List actual, List expected) + private static void assertColumnList(List actual, List expected) { if (expected == null) { assertNull(actual); @@ -90,24 +90,24 @@ private static void assertColumnList(List) null).build(); + com.facebook.presto.hive.metastore.Table prestoTbl = GlueToPrestoConverter.convertTable(testTbl, testDb.name()); assertTrue(prestoTbl.getPartitionColumns().isEmpty()); } @Test public void testConvertTableUppercaseColumnType() { - com.amazonaws.services.glue.model.Column uppercaseCol = getGlueTestColumn().withType("String"); - testTbl.getStorageDescriptor().setColumns(ImmutableList.of(uppercaseCol)); - GlueToPrestoConverter.convertTable(testTbl, testDb.getName()); + software.amazon.awssdk.services.glue.model.Column uppercaseCol = getGlueTestColumn().toBuilder().type("String").build(); + + StorageDescriptor sd = testTbl.storageDescriptor(); + testTbl = testTbl.toBuilder().storageDescriptor(sd.toBuilder().columns(ImmutableList.of(uppercaseCol)).build()).build(); + GlueToPrestoConverter.convertTable(testTbl, testDb.name()); } @Test public void testConvertPartition() { - GluePartitionConverter converter = new GluePartitionConverter(testPartition.getDatabaseName(), testPartition.getTableName()); + GluePartitionConverter converter = new GluePartitionConverter(testPartition.databaseName(), testPartition.tableName()); com.facebook.presto.hive.metastore.Partition prestoPartition = converter.apply(testPartition); - assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName()); - assertEquals(prestoPartition.getTableName(), testPartition.getTableName()); - assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns()); - assertEquals(prestoPartition.getValues(), testPartition.getValues()); - assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor()); - assertEquals(prestoPartition.getParameters(), testPartition.getParameters()); + assertEquals(prestoPartition.getDatabaseName(), testPartition.databaseName()); + assertEquals(prestoPartition.getTableName(), testPartition.tableName()); + assertColumnList(prestoPartition.getColumns(), testPartition.storageDescriptor().columns()); + assertEquals(prestoPartition.getValues(), testPartition.values()); + assertStorage(prestoPartition.getStorage(), testPartition.storageDescriptor()); + assertEquals(prestoPartition.getParameters(), testPartition.parameters()); } @Test public void testPartitionConversionMemoization() { String fakeS3Location = "s3://some-fake-location"; - testPartition.getStorageDescriptor().setLocation(fakeS3Location); + + StorageDescriptor sdPartition = testPartition.storageDescriptor(); + testPartition = testPartition.toBuilder().storageDescriptor(sdPartition.toBuilder().location(fakeS3Location).build()).build(); + // Second partition to convert with equal (but not aliased) values - Partition partitionTwo = getGlueTestPartition(testPartition.getDatabaseName(), testPartition.getTableName(), new ArrayList<>(testPartition.getValues())); + Partition partitionTwo = getGlueTestPartition(testPartition.databaseName(), testPartition.tableName(), new ArrayList<>(testPartition.values())); // Ensure storage fields are equal but not aliased as well - partitionTwo.getStorageDescriptor().setColumns(new ArrayList<>(testPartition.getStorageDescriptor().getColumns())); - partitionTwo.getStorageDescriptor().setBucketColumns(new ArrayList<>(testPartition.getStorageDescriptor().getBucketColumns())); - partitionTwo.getStorageDescriptor().setLocation("" + fakeS3Location); - partitionTwo.getStorageDescriptor().setInputFormat("" + testPartition.getStorageDescriptor().getInputFormat()); - partitionTwo.getStorageDescriptor().setOutputFormat("" + testPartition.getStorageDescriptor().getOutputFormat()); - partitionTwo.getStorageDescriptor().setParameters(new HashMap<>(testPartition.getStorageDescriptor().getParameters())); - - GluePartitionConverter converter = new GluePartitionConverter(testDb.getName(), testTbl.getName()); + StorageDescriptor sdPartitionTwo = partitionTwo.storageDescriptor(); + partitionTwo = partitionTwo.toBuilder().storageDescriptor( + sdPartitionTwo.toBuilder() + .columns(new ArrayList<>(testPartition.storageDescriptor().columns())) + .bucketColumns(new ArrayList<>(testPartition.storageDescriptor().bucketColumns())) + .location("" + fakeS3Location) + .inputFormat("" + testPartition.storageDescriptor().inputFormat()) + .outputFormat("" + testPartition.storageDescriptor().outputFormat()) + .parameters(new HashMap<>(testPartition.storageDescriptor().parameters())) + .build()).build(); + + GluePartitionConverter converter = new GluePartitionConverter(testDb.name(), testTbl.name()); com.facebook.presto.hive.metastore.Partition prestoPartition = converter.apply(testPartition); com.facebook.presto.hive.metastore.Partition prestoPartition2 = converter.apply(partitionTwo); @@ -161,16 +172,20 @@ public void testPartitionConversionMemoization() @Test public void testDatabaseNullParameters() { - testDb.setParameters(null); + testDb = testDb.toBuilder().parameters(null).build(); assertNotNull(GlueToPrestoConverter.convertDatabase(testDb).getParameters()); } @Test public void testTableNullParameters() { - testTbl.setParameters(null); - testTbl.getStorageDescriptor().getSerdeInfo().setParameters(null); - com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.getName()); + StorageDescriptor sd = testTbl.storageDescriptor(); + SerDeInfo serDeInfo = sd.serdeInfo(); + testTbl = testTbl.toBuilder() + .parameters(null) + .storageDescriptor(sd.toBuilder().serdeInfo(serDeInfo.toBuilder().parameters(null).build()).build()) + .build(); + com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.name()); assertNotNull(prestoTable.getParameters()); assertNotNull(prestoTable.getStorage().getSerdeParameters()); } @@ -178,38 +193,37 @@ public void testTableNullParameters() @Test public void testPartitionNullParameters() { - testPartition.setParameters(null); - assertNotNull(new GluePartitionConverter(testDb.getName(), testTbl.getName()).apply(testPartition).getParameters()); + testPartition = testPartition.toBuilder().parameters(null).build(); + assertNotNull(new GluePartitionConverter(testDb.name(), testTbl.name()).apply(testPartition).getParameters()); } @Test public void testConvertTableWithoutTableType() { - Table table = getGlueTestTable(testDb.getName()); - table.setTableType(null); - com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(table, testDb.getName()); + Table table = getGlueTestTable(testDb.name()).toBuilder().tableType(null).build(); + com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(table, testDb.name()); assertEquals(prestoTable.getTableType(), EXTERNAL_TABLE); } @Test public void testIcebergTableNonNullStorageDescriptor() { - testTbl.setParameters(ImmutableMap.of(ICEBERG_TABLE_TYPE_NAME, ICEBERG_TABLE_TYPE_VALUE)); - assertNotNull(testTbl.getStorageDescriptor()); - com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.getName()); + testTbl = testTbl.toBuilder().parameters(ImmutableMap.of(ICEBERG_TABLE_TYPE_NAME, ICEBERG_TABLE_TYPE_VALUE)).build(); + assertNotNull(testTbl.storageDescriptor()); + com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.name()); assertEquals(prestoTable.getDataColumns().size(), 1); } @Test public void testDeltaTableNonNullStorageDescriptor() { - testTbl.setParameters(ImmutableMap.of(SPARK_TABLE_PROVIDER_KEY, DELTA_LAKE_PROVIDER)); - assertNotNull(testTbl.getStorageDescriptor()); - com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.getName()); + testTbl = testTbl.toBuilder().parameters(ImmutableMap.of(SPARK_TABLE_PROVIDER_KEY, DELTA_LAKE_PROVIDER)).build(); + assertNotNull(testTbl.storageDescriptor()); + com.facebook.presto.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTbl, testDb.name()); assertEquals(prestoTable.getDataColumns().size(), 1); } - private static void assertColumnList(List actual, List expected) + private static void assertColumnList(List actual, List expected) { if (expected == null) { assertNull(actual); @@ -221,23 +235,23 @@ private static void assertColumnList(List actual, List getMetastoreClient().getTable(metastoreContext, table.getSchemaName(), table.getTableName())) .hasMessageStartingWith("Table StorageDescriptor is null for table"); - glueClient.deleteTable(deleteTableRequest); + awsSyncRequest(glueClient::deleteTable, deleteTableRequest, null); // Iceberg table - tableInput = tableInput.withParameters(ImmutableMap.of(ICEBERG_TABLE_TYPE_NAME, ICEBERG_TABLE_TYPE_VALUE)); - glueClient.createTable(new CreateTableRequest() - .withDatabaseName(database) - .withTableInput(tableInput)); + tableInput = tableInput.toBuilder().parameters(ImmutableMap.of(ICEBERG_TABLE_TYPE_NAME, ICEBERG_TABLE_TYPE_VALUE)).build(); + awsSyncRequest( + glueClient::createTable, + CreateTableRequest.builder() + .databaseName(database) + .tableInput(tableInput) + .build(), + null); assertTrue(isIcebergTable(getMetastoreClient().getTable(metastoreContext, table.getSchemaName(), table.getTableName()).orElseThrow(() -> new NoSuchElementException()))); - glueClient.deleteTable(deleteTableRequest); + awsSyncRequest(glueClient::deleteTable, deleteTableRequest, null); // Delta Lake table - tableInput = tableInput.withParameters(ImmutableMap.of(SPARK_TABLE_PROVIDER_KEY, DELTA_LAKE_PROVIDER)); - glueClient.createTable(new CreateTableRequest() - .withDatabaseName(database) - .withTableInput(tableInput)); + tableInput = tableInput.toBuilder().parameters(ImmutableMap.of(SPARK_TABLE_PROVIDER_KEY, DELTA_LAKE_PROVIDER)).build(); + awsSyncRequest( + glueClient::createTable, + CreateTableRequest.builder() + .databaseName(database) + .tableInput(tableInput) + .build(), + null); assertTrue(isDeltaLakeTable(getMetastoreClient().getTable(metastoreContext, table.getSchemaName(), table.getTableName()).orElseThrow(() -> new NoSuchElementException()))); } finally { // Table cannot be dropped through HiveMetastore since a TableHandle cannot be created - glueClient.deleteTable(new DeleteTableRequest() - .withDatabaseName(table.getSchemaName()) - .withName(table.getTableName())); + awsSyncRequest(glueClient::deleteTable, deleteTableRequest, null); } } @@ -351,12 +366,16 @@ public void testGetPartitionsWithFilterUsingReservedKeywordsAsColumnName() .addBigintValues(regularColumnPartitionName, 2L) .build(); - List partitionNames = metastoreClient.getPartitionNamesByFilter( + List partitionNamesWithVersion = metastoreClient.getPartitionNamesByFilter( METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), predicates); + List partitionNames = partitionNamesWithVersion.stream() + .map(PartitionNameWithVersion::getPartitionName) + .collect(toImmutableList()); + assertFalse(partitionNames.isEmpty()); assertEquals(partitionNames, ImmutableList.of("key=value2/int_partition=2")); @@ -366,11 +385,16 @@ public void testGetPartitionsWithFilterUsingReservedKeywordsAsColumnName() .addStringValues(reservedKeywordPartitionColumnName, "value1") .build(); - partitionNames = metastoreClient.getPartitionNamesByFilter( + partitionNamesWithVersion = metastoreClient.getPartitionNamesByFilter( METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), predicates); + + partitionNames = partitionNamesWithVersion.stream() + .map(PartitionNameWithVersion::getPartitionName) + .collect(toImmutableList()); + assertFalse(partitionNames.isEmpty()); assertEquals(partitionNames, ImmutableList.of("key=value1/int_partition=1", "key=value2/int_partition=2")); } @@ -898,11 +922,16 @@ private void doGetPartitionsFilterTest( .map(expectedPartitionValues -> makePartName(partitionColumnNames, expectedPartitionValues.getValues())) .collect(toImmutableList()); - List partitionNames = metastoreClient.getPartitionNamesByFilter( + List partitionNamesWithVersion = metastoreClient.getPartitionNamesByFilter( METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), filter); + + List partitionNames = partitionNamesWithVersion.stream() + .map(PartitionNameWithVersion::getPartitionName) + .collect(toImmutableList()); + assertEquals( partitionNames, expectedResults, diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java index be540ec76279a..c18057870b95f 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/metastore/glue/TestingMetastoreObjects.java @@ -13,18 +13,18 @@ */ package com.facebook.presto.hive.metastore.glue; -import com.amazonaws.services.glue.model.Column; -import com.amazonaws.services.glue.model.Database; -import com.amazonaws.services.glue.model.Partition; -import com.amazonaws.services.glue.model.SerDeInfo; -import com.amazonaws.services.glue.model.StorageDescriptor; -import com.amazonaws.services.glue.model.Table; import com.facebook.presto.hive.HiveType; import com.facebook.presto.hive.metastore.Storage; import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.spi.security.PrincipalType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.Database; +import software.amazon.awssdk.services.glue.model.Partition; +import software.amazon.awssdk.services.glue.model.SerDeInfo; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.Table; import java.util.List; import java.util.Optional; @@ -41,58 +41,64 @@ private TestingMetastoreObjects() {} public static Database getGlueTestDatabase() { - return new Database() - .withName("test-db" + generateRandom()) - .withDescription("database desc") - .withLocationUri("/db") - .withParameters(ImmutableMap.of()); + return Database.builder() + .name("test-db" + generateRandom()) + .description("database desc") + .locationUri("/db") + .parameters(ImmutableMap.of()) + .build(); } public static Table getGlueTestTable(String dbName) { - return new Table() - .withDatabaseName(dbName) - .withName("test-tbl" + generateRandom()) - .withOwner("owner") - .withParameters(ImmutableMap.of()) - .withPartitionKeys(ImmutableList.of(getGlueTestColumn())) - .withStorageDescriptor(getGlueTestStorageDescriptor()) - .withTableType(EXTERNAL_TABLE.name()) - .withViewOriginalText("originalText") - .withViewExpandedText("expandedText"); + return Table.builder() + .databaseName(dbName) + .name("test-tbl" + generateRandom()) + .owner("owner") + .parameters(ImmutableMap.of()) + .partitionKeys(ImmutableList.of(getGlueTestColumn())) + .storageDescriptor(getGlueTestStorageDescriptor()) + .tableType(EXTERNAL_TABLE.name()) + .viewOriginalText("originalText") + .viewExpandedText("expandedText") + .build(); } public static Column getGlueTestColumn() { - return new Column() - .withName("test-col" + generateRandom()) - .withType("string") - .withComment("column comment"); + return Column.builder() + .name("test-col" + generateRandom()) + .type("string") + .comment("column comment") + .build(); } public static StorageDescriptor getGlueTestStorageDescriptor() { - return new StorageDescriptor() - .withBucketColumns(ImmutableList.of("test-bucket-col")) - .withColumns(ImmutableList.of(getGlueTestColumn())) - .withParameters(ImmutableMap.of()) - .withSerdeInfo(new SerDeInfo() - .withSerializationLibrary("SerdeLib") - .withParameters(ImmutableMap.of())) - .withInputFormat("InputFormat") - .withOutputFormat("OutputFormat") - .withLocation("/test-tbl") - .withNumberOfBuckets(1); + return StorageDescriptor.builder() + .bucketColumns(ImmutableList.of("test-bucket-col")) + .columns(ImmutableList.of(getGlueTestColumn())) + .parameters(ImmutableMap.of()) + .serdeInfo(SerDeInfo.builder() + .serializationLibrary("SerdeLib") + .parameters(ImmutableMap.of()) + .build()) + .inputFormat("InputFormat") + .outputFormat("OutputFormat") + .location("/test-tbl") + .numberOfBuckets(1) + .build(); } public static Partition getGlueTestPartition(String dbName, String tblName, List values) { - return new Partition() - .withDatabaseName(dbName) - .withTableName(tblName) - .withValues(values) - .withParameters(ImmutableMap.of()) - .withStorageDescriptor(getGlueTestStorageDescriptor()); + return Partition.builder() + .databaseName(dbName) + .tableName(tblName) + .values(values) + .parameters(ImmutableMap.of()) + .storageDescriptor(getGlueTestStorageDescriptor()) + .build(); } // --------------- Presto Objects --------------- diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java index b9dcdf6979db1..d2c5edd9bbefa 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/statistics/TestMetastoreHiveStatisticsProvider.java @@ -16,11 +16,13 @@ import com.facebook.presto.cache.CacheConfig; import com.facebook.presto.common.predicate.NullableValue; import com.facebook.presto.common.type.DecimalType; +import com.facebook.presto.common.type.TestingTypeManager; import com.facebook.presto.common.type.Type; import com.facebook.presto.hive.HiveBasicStatistics; import com.facebook.presto.hive.HiveClientConfig; import com.facebook.presto.hive.HiveColumnHandle; import com.facebook.presto.hive.HivePartition; +import com.facebook.presto.hive.HivePartitionManager; import com.facebook.presto.hive.HiveSessionProperties; import com.facebook.presto.hive.NamenodeStats; import com.facebook.presto.hive.OrcFileWriterConfig; @@ -65,7 +67,6 @@ import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CORRUPTED_COLUMN_STATISTICS; import static com.facebook.presto.hive.HivePartition.UNPARTITIONED_ID; -import static com.facebook.presto.hive.HivePartitionManager.parsePartition; import static com.facebook.presto.hive.HiveTestUtils.DO_NOTHING_DIRECTORY_LISTER; import static com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static com.facebook.presto.hive.HiveType.HIVE_LONG; @@ -110,6 +111,8 @@ public class TestMetastoreHiveStatisticsProvider private static final QuickStatsProvider quickStatsProvider = new QuickStatsProvider(new TestingExtendedHiveMetastore(), HDFS_ENVIRONMENT, DO_NOTHING_DIRECTORY_LISTER, new HiveClientConfig(), new NamenodeStats(), ImmutableList.of()); + private final HivePartitionManager hivePartitionManager = new HivePartitionManager(new TestingTypeManager(), new HiveClientConfig()); + @Test public void testGetPartitionsSample() { @@ -825,9 +828,9 @@ private static String invalidColumnStatistics(String message) return format("Corrupted partition statistics (Table: %s Partition: [%s] Column: %s): %s", TABLE, PARTITION, COLUMN, message); } - private static HivePartition partition(String name) + private HivePartition partition(String name) { - return parsePartition(TABLE, new PartitionNameWithVersion(name, Optional.empty()), ImmutableList.of(PARTITION_COLUMN_1, PARTITION_COLUMN_2), ImmutableList.of(VARCHAR, BIGINT), DateTimeZone.getDefault()); + return hivePartitionManager.parsePartition(TABLE, new PartitionNameWithVersion(name, Optional.empty()), ImmutableList.of(PARTITION_COLUMN_1, PARTITION_COLUMN_2), ImmutableList.of(VARCHAR, BIGINT)); } private static PartitionStatistics rowsCount(long rowsCount) diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index 34a1f855e1643..459b23998558f 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -14,7 +14,6 @@ ${project.parent.basedir} 17 - 1.8.1 0.103.0 true @@ -88,6 +87,22 @@ + + org.apache.parquet + parquet-hadoop + ${dep.parquet.version} + + + org.apache.yetus + audience-annotations + + + org.apache.hadoop + hadoop-client + + + + com.facebook.presto presto-expressions @@ -390,6 +405,12 @@ runtime + + software.amazon.awssdk + kms + runtime + + org.apache.iceberg iceberg-parquet @@ -644,7 +665,7 @@ org.apache.httpcomponents.core5 httpcore5 - 5.3.1 + 5.3.4 diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index 913c501b5795d..9d1c79c3e029b 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -19,6 +19,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.RuntimeStats; import com.facebook.presto.common.Subfield; +import com.facebook.presto.common.predicate.Domain; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.BigintType; import com.facebook.presto.common.type.SqlTimestampWithTimeZone; @@ -35,6 +36,8 @@ import com.facebook.presto.iceberg.changelog.ChangelogOperation; import com.facebook.presto.iceberg.changelog.ChangelogUtil; import com.facebook.presto.iceberg.statistics.StatisticsFileCache; +import com.facebook.presto.iceberg.transaction.IcebergTransactionContext; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorDeleteTableHandle; @@ -64,11 +67,12 @@ import com.facebook.presto.spi.SchemaTablePrefix; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.TableNotFoundException; -import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorCommitHandle; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.facebook.presto.spi.connector.ConnectorTableVersion; import com.facebook.presto.spi.connector.ConnectorTableVersion.VersionOperator; import com.facebook.presto.spi.connector.ConnectorTableVersion.VersionType; +import com.facebook.presto.spi.connector.EmptyConnectorCommitHandle; import com.facebook.presto.spi.connector.RowChangeParadigm; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.plan.FilterStatsCalculatorService; @@ -99,10 +103,14 @@ import org.apache.iceberg.DeleteFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; -import org.apache.iceberg.IsolationLevel; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.IncrementalAppendScan; +import org.apache.iceberg.ManageSnapshots; +import org.apache.iceberg.ManifestFile; import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.MetricsConfig; import org.apache.iceberg.MetricsModes.None; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowDelta; import org.apache.iceberg.RowLevelOperationMode; @@ -110,14 +118,16 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; import org.apache.iceberg.Transaction; import org.apache.iceberg.UpdatePartitionSpec; import org.apache.iceberg.UpdateProperties; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -126,18 +136,19 @@ import org.apache.iceberg.util.CharSequenceSet; import org.apache.iceberg.view.View; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -166,6 +177,7 @@ import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_FORMAT_VERSION; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_MATERIALIZED_VIEW; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_SNAPSHOT_ID; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_TRANSACTION_CONFLICT_ERROR; import static com.facebook.presto.iceberg.IcebergMaterializedViewProperties.getRefreshType; import static com.facebook.presto.iceberg.IcebergMaterializedViewProperties.getStaleReadBehavior; import static com.facebook.presto.iceberg.IcebergMaterializedViewProperties.getStalenessWindow; @@ -180,6 +192,7 @@ import static com.facebook.presto.iceberg.IcebergMetadataColumn.UPDATE_ROW_DATA; import static com.facebook.presto.iceberg.IcebergPartitionType.ALL; import static com.facebook.presto.iceberg.IcebergSessionProperties.getCompressionCodec; +import static com.facebook.presto.iceberg.IcebergSessionProperties.getMaterializedViewMaxChangedPartitions; import static com.facebook.presto.iceberg.IcebergSessionProperties.getMaterializedViewStoragePrefix; import static com.facebook.presto.iceberg.IcebergSessionProperties.isPushdownFilterEnabled; import static com.facebook.presto.iceberg.IcebergTableProperties.LOCATION_PROPERTY; @@ -188,7 +201,9 @@ import static com.facebook.presto.iceberg.IcebergTableType.CHANGELOG; import static com.facebook.presto.iceberg.IcebergTableType.DATA; import static com.facebook.presto.iceberg.IcebergTableType.EQUALITY_DELETES; +import static com.facebook.presto.iceberg.IcebergUtil.MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS; import static com.facebook.presto.iceberg.IcebergUtil.MIN_FORMAT_VERSION_FOR_DELETE; +import static com.facebook.presto.iceberg.IcebergUtil.createDomainFromIcebergPartitionValue; import static com.facebook.presto.iceberg.IcebergUtil.getColumns; import static com.facebook.presto.iceberg.IcebergUtil.getColumnsForWrite; import static com.facebook.presto.iceberg.IcebergUtil.getDeleteMode; @@ -200,12 +215,15 @@ import static com.facebook.presto.iceberg.IcebergUtil.getSnapshotIdTimeOperator; import static com.facebook.presto.iceberg.IcebergUtil.getSortFields; import static com.facebook.presto.iceberg.IcebergUtil.getTableComment; -import static com.facebook.presto.iceberg.IcebergUtil.getViewComment; +import static com.facebook.presto.iceberg.IcebergUtil.opsFromTable; import static com.facebook.presto.iceberg.IcebergUtil.resolveSnapshotIdByName; import static com.facebook.presto.iceberg.IcebergUtil.toHiveColumns; import static com.facebook.presto.iceberg.IcebergUtil.tryGetLocation; import static com.facebook.presto.iceberg.IcebergUtil.tryGetProperties; import static com.facebook.presto.iceberg.IcebergUtil.tryGetSchema; +import static com.facebook.presto.iceberg.IcebergUtil.validateBranchExists; +import static com.facebook.presto.iceberg.IcebergUtil.validateNoBranchInBaseTables; +import static com.facebook.presto.iceberg.IcebergUtil.validateNoBranchSpecified; import static com.facebook.presto.iceberg.IcebergUtil.validateTableMode; import static com.facebook.presto.iceberg.IcebergWarningCode.SORT_COLUMN_TRANSFORM_NOT_SUPPORTED_WARNING; import static com.facebook.presto.iceberg.IcebergWarningCode.USE_OF_DEPRECATED_TABLE_PROPERTY; @@ -230,11 +248,12 @@ import static com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.NOT_MATERIALIZED; import static com.facebook.presto.spi.MaterializedViewStatus.MaterializedViewState.PARTIALLY_MATERIALIZED; import static com.facebook.presto.spi.StandardErrorCode.ALREADY_EXISTS; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_VIEW; import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.connector.RowChangeParadigm.DELETE_ROW_AND_INSERT_ROW; import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; -import static com.google.common.base.Preconditions.checkState; +import static com.facebook.presto.spi.transaction.IsolationLevel.SERIALIZABLE; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Verify.verify; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -243,21 +262,22 @@ import static com.google.common.collect.Maps.transformValues; import static java.lang.Long.parseLong; import static java.lang.String.format; +import static java.time.Duration.ofDays; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; +import static org.apache.iceberg.DataOperations.APPEND; +import static org.apache.iceberg.DataOperations.REPLACE; import static org.apache.iceberg.MetadataColumns.ROW_POSITION; import static org.apache.iceberg.MetadataColumns.SPEC_ID; import static org.apache.iceberg.RowLevelOperationMode.MERGE_ON_READ; import static org.apache.iceberg.SnapshotSummary.DELETED_RECORDS_PROP; import static org.apache.iceberg.SnapshotSummary.REMOVED_EQ_DELETES_PROP; import static org.apache.iceberg.SnapshotSummary.REMOVED_POS_DELETES_PROP; -import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL; -import static org.apache.iceberg.TableProperties.DELETE_ISOLATION_LEVEL_DEFAULT; import static org.apache.iceberg.TableProperties.WRITE_DATA_LOCATION; import static org.apache.iceberg.expressions.Expressions.alwaysTrue; public abstract class IcebergAbstractMetadata - implements ConnectorMetadata + implements IcebergTransactionMetadata { private static final Logger log = Logger.get(IcebergAbstractMetadata.class); protected static final String INFORMATION_SCHEMA = "information_schema"; @@ -288,12 +308,10 @@ public abstract class IcebergAbstractMetadata protected final RowExpressionService rowExpressionService; protected final FilterStatsCalculatorService filterStatsCalculatorService; protected Optional procedureContext = Optional.empty(); - protected Transaction transaction; + protected final IcebergTransactionContext transactionContext; protected final StatisticsFileCache statisticsFileCache; protected final IcebergTableProperties tableProperties; - private final StandardFunctionResolution functionResolution; - private final ConcurrentMap icebergTables = new ConcurrentHashMap<>(); public IcebergAbstractMetadata( TypeManager typeManager, @@ -306,7 +324,9 @@ public IcebergAbstractMetadata( NodeVersion nodeVersion, FilterStatsCalculatorService filterStatsCalculatorService, StatisticsFileCache statisticsFileCache, - IcebergTableProperties tableProperties) + IcebergTableProperties tableProperties, + com.facebook.presto.spi.transaction.IsolationLevel isolationLevel, + boolean autoCommitContext) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.procedureRegistry = requireNonNull(procedureRegistry, "procedureRegistry is null"); @@ -319,18 +339,17 @@ public IcebergAbstractMetadata( this.filterStatsCalculatorService = requireNonNull(filterStatsCalculatorService, "filterStatsCalculatorService is null"); this.statisticsFileCache = requireNonNull(statisticsFileCache, "statisticsFileCache is null"); this.tableProperties = requireNonNull(tableProperties, "tableProperties is null"); + this.transactionContext = new IcebergTransactionContext(isolationLevel, autoCommitContext); } protected final Table getIcebergTable(ConnectorSession session, SchemaTableName schemaTableName) { - return icebergTables.computeIfAbsent( - schemaTableName, - ignored -> getRawIcebergTable(session, schemaTableName)); + return this.transactionContext.getIcebergTable(schemaTableName, ignored -> getRawIcebergTable(session, schemaTableName)); } protected abstract Table getRawIcebergTable(ConnectorSession session, SchemaTableName schemaTableName); - protected abstract View getIcebergView(ConnectorSession session, SchemaTableName schemaTableName); + protected abstract Optional getViewMetadata(ConnectorSession session, SchemaTableName viewName); protected abstract void createIcebergView( ConnectorSession session, @@ -357,6 +376,48 @@ public Optional getProcedureContext() return this.procedureContext; } + protected static void validateTableForPresto(BaseTable table, Optional tableSnapshotId) + { + Snapshot snapshot; + try { + snapshot = tableSnapshotId + .map(table::snapshot) + .orElse(table.currentSnapshot()); + } + catch (RuntimeException e) { + // If the snapshot cannot be retrieved (e.g. metadata is missing), we cannot validate the table. + // Returning here allows operations that do not strictly require the snapshot (like DROP TABLE) to proceed. + return; + } + + if (snapshot == null) { + // empty table, nothing to validate + return; + } + + TableMetadata metadata = table.operations().current(); + if (metadata.formatVersion() < 3) { + return; + } + + Schema schema = metadata.schemasById().get(snapshot.schemaId()); + if (schema == null) { + schema = metadata.schema(); + } + + // Reject schema default values (initial-default / write-default) + for (Types.NestedField field : schema.columns()) { + if (field.initialDefault() != null || field.writeDefault() != null) { + throw new PrestoException(NOT_SUPPORTED, "Iceberg v3 column default values are not supported"); + } + } + + // Reject Iceberg table encryption + if (!metadata.encryptionKeys().isEmpty() || snapshot.keyId() != null || metadata.properties().containsKey("encryption.key-id")) { + throw new PrestoException(NOT_SUPPORTED, "Iceberg table encryption is not supported"); + } + } + /** * This class implements the default implementation for getTableLayoutForConstraint which will be used in the case of a Java Worker */ @@ -550,13 +611,9 @@ protected ConnectorTableMetadata getTableOrViewMetadata(ConnectorSession session // Considering that the Iceberg library does not provide an efficient way to determine whether // it's a view or a table without loading it, we first try to load it as a table directly, and then // try to load it as a view when getting an `NoSuchTableException`. This will be more efficient. - try { - View icebergView = getIcebergView(session, schemaTableName); - return new ConnectorTableMetadata(table, getColumnMetadata(session, icebergView), createViewMetadataProperties(icebergView), getViewComment(icebergView)); - } - catch (NoSuchViewException noSuchViewException) { - throw new TableNotFoundException(schemaTableName); - } + return getViewMetadata(session, schemaTableName) + .map(IcebergViewMetadata::getTableMetadata) + .orElseThrow(() -> new TableNotFoundException(schemaTableName)); } } @@ -598,6 +655,7 @@ public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTable @Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) { + shouldRunInAutoCommitTransaction("CREATE TABLE"); Optional layout = getNewTableLayout(session, tableMetadata); finishCreateTable(session, beginCreateTable(session, tableMetadata, layout), ImmutableList.of(), ImmutableList.of()); } @@ -610,8 +668,7 @@ public Optional finishCreateTable(ConnectorSession sess protected ConnectorInsertTableHandle beginIcebergTableInsert(ConnectorSession session, IcebergTableHandle table, Table icebergTable) { - transaction = icebergTable.newTransaction(); - + validateBranchExists(table, icebergTable); return new IcebergInsertTableHandle( table.getSchemaName(), table.getIcebergTableName(), @@ -659,7 +716,7 @@ public Optional finishInsert(ConnectorSession session, private Optional finishInsert(ConnectorSession session, IcebergWritableTableHandle writableTableHandle, Collection fragments) { if (fragments.isEmpty()) { - transaction.commitTransaction(); + transactionContext.commit(); return Optional.empty(); } @@ -667,8 +724,11 @@ private Optional finishInsert(ConnectorSession session, .map(slice -> commitTaskCodec.fromJson(slice.getBytes())) .collect(toImmutableList()); - Table icebergTable = transaction.table(); - AppendFiles appendFiles = transaction.newAppend(); + SchemaTableName schemaTableName = new SchemaTableName(writableTableHandle.getSchemaName(), writableTableHandle.getTableName().getTableName()); + Table icebergTable = getIcebergTable(session, schemaTableName); + AppendFiles appendFiles = icebergTable.newAppend(); + Optional branchName = writableTableHandle.getTableName().getBranchName(); + branchName.ifPresent(appendFiles::toBranch); ImmutableSet.Builder writtenFiles = ImmutableSet.builder(); commitTasks.forEach(task -> handleInsertTask(task, icebergTable, appendFiles, writtenFiles)); @@ -676,7 +736,6 @@ private Optional finishInsert(ConnectorSession session, try { appendFiles.set(PRESTO_QUERY_ID, session.getQueryId()); appendFiles.commit(); - transaction.commitTransaction(); } catch (ValidationException e) { log.error(e, "ValidationException in finishWrite"); @@ -688,29 +747,31 @@ private Optional finishInsert(ConnectorSession session, .collect(toImmutableList()), icebergTable.location()))); } - private Optional finishWrite(ConnectorSession session, IcebergWritableTableHandle writableTableHandle, Collection fragments, ChangelogOperation operationType) + private Optional finishWrite(ConnectorSession session, SchemaTableName tableName, IcebergWritableTableHandle writableTableHandle, Collection fragments, ChangelogOperation operationType) { if (fragments.isEmpty()) { - transaction.commitTransaction(); return Optional.empty(); } - Table icebergTable = transaction.table(); + Table icebergTable = getIcebergTable(session, tableName); List commitTasks = fragments.stream() .map(slice -> commitTaskCodec.fromJson(slice.getBytes())) .collect(toImmutableList()); - RowDelta rowDelta = transaction.newRowDelta(); + RowDelta rowDelta = icebergTable.newRowDelta(); writableTableHandle.getTableName().getSnapshotId().map(icebergTable::snapshot).ifPresent(s -> rowDelta.validateFromSnapshot(s.snapshotId())); - IsolationLevel isolationLevel = IsolationLevel.fromName(icebergTable.properties().getOrDefault(DELETE_ISOLATION_LEVEL, DELETE_ISOLATION_LEVEL_DEFAULT)); + Optional branchName = writableTableHandle.getTableName().getBranchName(); + if (branchName.isPresent()) { + rowDelta.toBranch(branchName.get()); + } ImmutableSet.Builder writtenFiles = ImmutableSet.builder(); ImmutableSet.Builder referencedDataFiles = ImmutableSet.builder(); commitTasks.forEach(task -> handleTask(task, icebergTable, rowDelta, writtenFiles, referencedDataFiles)); rowDelta.validateDataFilesExist(referencedDataFiles.build()); - if (isolationLevel == IsolationLevel.SERIALIZABLE) { + if (this.transactionContext.getIsolationLevel() == SERIALIZABLE) { rowDelta.validateNoConflictingDataFiles(); } @@ -723,7 +784,6 @@ private Optional finishWrite(ConnectorSession session, try { rowDelta.set(PRESTO_QUERY_ID, session.getQueryId()); rowDelta.commit(); - transaction.commitTransaction(); } catch (ValidationException e) { log.error(e, "ValidationException in finishWrite"); @@ -831,11 +891,18 @@ public ColumnHandle getMergeTargetTableRowIdColumnHandle(ConnectorSession sessio @Override public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorTableHandle tableHandle) { + shouldRunInAutoCommitTransaction("MERGE INTO"); IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can have data merged"); Table icebergTable = getIcebergTable(session, icebergTableHandle.getSchemaTableName()); + validateBranchExists(icebergTableHandle, icebergTable); int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + if (formatVersion > MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS) { + throw new PrestoException(NOT_SUPPORTED, + format("Iceberg table updates for format version %s are not supported yet", formatVersion)); + } + if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE || !Optional.ofNullable(icebergTable.properties().get(TableProperties.UPDATE_MODE)) .map(mode -> mode.equals(MERGE_ON_READ.modeName())) @@ -844,7 +911,6 @@ public ConnectorMergeTableHandle beginMerge(ConnectorSession session, ConnectorT "Iceberg table updates require at least format version 2 and update mode must be merge-on-read"); } validateTableMode(session, icebergTable); - transaction = icebergTable.newTransaction(); IcebergInsertTableHandle insertHandle = new IcebergInsertTableHandle( icebergTableHandle.getSchemaName(), @@ -874,7 +940,9 @@ public void finishMerge( IcebergWritableTableHandle insertTableHandle = ((IcebergMergeTableHandle) tableHandle).getInsertTableHandle(); - finishWrite(session, insertTableHandle, fragments, UPDATE_AFTER); + finishWrite(session, + new SchemaTableName(insertTableHandle.getSchemaName(), insertTableHandle.getTableName().getTableName()), + insertTableHandle, fragments, UPDATE_AFTER); } @Override @@ -927,7 +995,7 @@ protected ImmutableMap createMetadataProperties(Table icebergTab ImmutableMap.Builder properties = ImmutableMap.builder(); properties.put(TableProperties.DEFAULT_FILE_FORMAT, getFileFormat(icebergTable)); - int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + int formatVersion = opsFromTable(icebergTable).current().formatVersion(); properties.put(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)); if (!icebergTable.spec().fields().isEmpty()) { @@ -1014,6 +1082,7 @@ public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession @Override public ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle) { + shouldRunInAutoCommitTransaction("ANALYZE"); return tableHandle; } @@ -1025,9 +1094,15 @@ public void finishStatisticsCollection(ConnectorSession session, ConnectorTableH TableStatisticsMaker.writeTableStatistics(nodeVersion, typeManager, icebergTableHandle, icebergTable, session, computedStatistics); } + public ConnectorCommitHandle commit() + { + transactionContext.commit(); + return EmptyConnectorCommitHandle.INSTANCE; + } + public void rollback() { - // TODO: cleanup open transaction + transactionContext.rollback(); } @Override @@ -1046,6 +1121,90 @@ public void dropBranch(ConnectorSession session, ConnectorTableHandle tableHandl } } + @Override + public void createBranch( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; + verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can have branch created"); + Table icebergTable = getIcebergTable(session, icebergTableHandle.getSchemaTableName()); + + boolean branchExists = icebergTable.refs().containsKey(branchName); + if (ifNotExists && branchExists) { + return; + } + long targetSnapshotId = tableVersion.map(version -> getSnapshotIdForTableVersion(icebergTable, version)) + .orElseGet(() -> { + if (icebergTable.currentSnapshot() == null) { + throw new PrestoException(NOT_FOUND, format("Table %s has no current snapshot", icebergTableHandle.getSchemaTableName().getTableName())); + } + return icebergTable.currentSnapshot().snapshotId(); + }); + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + if (replace && branchExists) { + manageSnapshots.replaceBranch(branchName, targetSnapshotId); + } + else if (!branchExists) { + manageSnapshots.createBranch(branchName, targetSnapshotId); + } + else { + throw new PrestoException(ALREADY_EXISTS, format("Branch %s already exists in table %s", branchName, icebergTableHandle.getSchemaTableName().getTableName())); + } + // Apply retention policies if specified + retainDays.ifPresent(retainDs -> manageSnapshots.setMaxRefAgeMs(branchName, ofDays(retainDs).toMillis())); + minSnapshotsToKeep.ifPresent(minSnapshots -> manageSnapshots.setMinSnapshotsToKeep(branchName, minSnapshots)); + maxSnapshotAgeDays.ifPresent(maxAgeDays -> manageSnapshots.setMaxSnapshotAgeMs(branchName, ofDays(maxAgeDays).toMillis())); + manageSnapshots.commit(); + } + + @Override + public void createTag( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays) + { + IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; + verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can have tag created"); + Table icebergTable = getIcebergTable(session, icebergTableHandle.getSchemaTableName()); + + boolean tagExists = icebergTable.refs().containsKey(tagName); + if (ifNotExists && tagExists) { + return; + } + long targetSnapshotId = tableVersion.map(version -> getSnapshotIdForTableVersion(icebergTable, version)) + .orElseGet(() -> { + if (icebergTable.currentSnapshot() == null) { + throw new PrestoException(NOT_FOUND, format("Table %s has no current snapshot", icebergTableHandle.getSchemaTableName().getTableName())); + } + return icebergTable.currentSnapshot().snapshotId(); + }); + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + if (replace && tagExists) { + manageSnapshots.replaceTag(tagName, targetSnapshotId); + } + else if (!tagExists) { + manageSnapshots.createTag(tagName, targetSnapshotId); + } + else { + throw new PrestoException(ALREADY_EXISTS, format("Tag %s already exists in table %s", tagName, icebergTableHandle.getSchemaTableName().getTableName())); + } + // Apply retention policies if specified + retainDays.ifPresent(retainDs -> manageSnapshots.setMaxRefAgeMs(tagName, ofDays(retainDs).toMillis())); + manageSnapshots.commit(); + } + @Override public void dropTag(ConnectorSession session, ConnectorTableHandle tableHandle, String tagName, boolean tagExists) { @@ -1073,19 +1232,18 @@ public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle IcebergTableHandle handle = (IcebergTableHandle) tableHandle; verify(handle.getIcebergTableName().getTableType() == DATA, "only the data table can have columns added"); + validateNoBranchSpecified(handle, "ADD COLUMN"); Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); - Transaction transaction = icebergTable.newTransaction(); - transaction.updateSchema().addColumn(column.getName(), columnType, column.getComment().orElse(null)).commit(); + icebergTable.updateSchema().addColumn(column.getName(), columnType, column.getComment().orElse(null)).commit(); if (column.getProperties().containsKey(PARTITIONING_PROPERTY)) { List partitioningTransform = (List) column.getProperties().get(PARTITIONING_PROPERTY); - UpdatePartitionSpec updatePartitionSpec = transaction.updateSpec(); + UpdatePartitionSpec updatePartitionSpec = icebergTable.updateSpec(); for (String transform : partitioningTransform) { updatePartitionSpec = updatePartitionSpec.addField(getPartitionColumnName(column.getName(), transform), getTransformTerm(column.getName(), transform)); } updatePartitionSpec.commit(); } - transaction.commitTransaction(); } @Override @@ -1094,6 +1252,7 @@ public void dropColumn(ConnectorSession session, ConnectorTableHandle tableHandl IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; IcebergColumnHandle handle = (IcebergColumnHandle) column; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can have columns dropped"); + validateNoBranchSpecified(icebergTableHandle, "DROP COLUMN"); Table icebergTable = getIcebergTable(session, icebergTableHandle.getSchemaTableName()); // Currently drop partition column used in any partition specs of a table would introduce some problems in Iceberg. @@ -1114,17 +1273,16 @@ public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHan { IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can have columns renamed"); + validateNoBranchSpecified(icebergTableHandle, "RENAME COLUMN"); IcebergColumnHandle columnHandle = (IcebergColumnHandle) source; Table icebergTable = getIcebergTable(session, icebergTableHandle.getSchemaTableName()); - Transaction transaction = icebergTable.newTransaction(); - transaction.updateSchema().renameColumn(columnHandle.getName(), target).commit(); + icebergTable.updateSchema().renameColumn(columnHandle.getName(), target).commit(); icebergTable.spec().fields().stream() .filter(field -> field.sourceId() == columnHandle.getId()) .forEach(field -> { String transform = field.transform().toString(); - transaction.updateSpec().renameField(field.name(), getPartitionColumnName(target, transform)).commit(); + icebergTable.updateSpec().renameField(field.name(), getPartitionColumnName(target, transform)).commit(); }); - transaction.commitTransaction(); } @Override @@ -1196,7 +1354,7 @@ public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableNa return new IcebergTableHandle( storageTableName.getSchemaName(), - new IcebergTableName(storageTableName.getTableName(), name.getTableType(), Optional.empty(), Optional.empty()), + new IcebergTableName(storageTableName.getTableName(), name.getTableType(), Optional.empty(), Optional.empty(), Optional.empty()), name.getSnapshotId().isPresent(), tryGetLocation(storageTable), tryGetProperties(storageTable), @@ -1221,6 +1379,12 @@ public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableNa }) .orElseGet(() -> resolveSnapshotIdByName(table, name)); + // Validate unsupported v3 features (column defaults, encryption) before + // proceeding + if (table instanceof BaseTable) { + validateTableForPresto((BaseTable) table, tableSnapshotId); + } + // Get Iceberg tables schema, properties, and location with missing // filesystem metadata will fail. // See https://github.com/prestodb/presto/pull/21181 @@ -1229,7 +1393,7 @@ public IcebergTableHandle getTableHandle(ConnectorSession session, SchemaTableNa return new IcebergTableHandle( tableNameToLoad.getSchemaName(), - new IcebergTableName(tableNameToLoad.getTableName(), name.getTableType(), tableSnapshotId, name.getChangelogEndSnapshot()), + new IcebergTableName(tableNameToLoad.getTableName(), name.getTableType(), tableSnapshotId, name.getBranchName(), name.getChangelogEndSnapshot()), name.getSnapshotId().isPresent(), tryGetLocation(table), tryGetProperties(table), @@ -1265,9 +1429,10 @@ public Optional getSystemTable(ConnectorSession session, SchemaTabl @Override public void truncateTable(ConnectorSession session, ConnectorTableHandle tableHandle) { + shouldRunInAutoCommitTransaction("TRUNCATE TABLE"); IcebergTableHandle handle = (IcebergTableHandle) tableHandle; Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); - removeScanFiles(icebergTable, TupleDomain.all()); + removeScanFiles(handle, icebergTable, TupleDomain.all()); } @Override @@ -1284,14 +1449,13 @@ public ConnectorDistributedProcedureHandle beginCallDistributedProcedure( throw new PrestoException(NOT_SUPPORTED, "This connector do not allow table execute at specified snapshot"); } - transaction = icebergTable.newTransaction(); BaseProcedure procedure = procedureRegistry.resolve( new ConnectorId(procedureName.getCatalogName()), new SchemaTableName( procedureName.getSchemaName(), procedureName.getObjectName())); verify(procedure instanceof DistributedProcedure, "procedure must be DistributedProcedure"); - procedureContext = Optional.of((IcebergProcedureContext) ((DistributedProcedure) procedure).createContext(icebergTable, transaction)); + procedureContext = Optional.of((IcebergProcedureContext) ((DistributedProcedure) procedure).createContext(icebergTable, icebergTable.newTransaction())); return ((DistributedProcedure) procedure).begin(session, procedureContext.get(), tableLayoutHandle, arguments); } @@ -1306,7 +1470,6 @@ public void finishCallDistributedProcedure(ConnectorSession session, ConnectorDi verify(procedure instanceof DistributedProcedure, "procedure must be DistributedProcedure"); verify(procedureContext.isPresent(), "procedure context must be present"); ((DistributedProcedure) procedure).finish(session, procedureContext.get(), procedureHandle, fragments); - transaction.commitTransaction(); procedureContext = Optional.empty(); } @@ -1319,16 +1482,20 @@ public ConnectorDeleteTableHandle beginDelete(ConnectorSession session, Connecto if (handle.isSnapshotSpecified()) { throw new PrestoException(NOT_SUPPORTED, "This connector do not allow delete data at specified snapshot"); } + validateBranchExists(handle, icebergTable); - int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + int formatVersion = opsFromTable(icebergTable).current().formatVersion(); if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE) { throw new PrestoException(NOT_SUPPORTED, format("This connector only supports delete where one or more partitions are deleted entirely for table versions older than %d", MIN_FORMAT_VERSION_FOR_DELETE)); } + if (formatVersion > MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS) { + throw new PrestoException(NOT_SUPPORTED, + format("Iceberg table updates for format version %s are not supported yet", formatVersion)); + } if (getDeleteMode(icebergTable) == RowLevelOperationMode.COPY_ON_WRITE) { throw new PrestoException(NOT_SUPPORTED, "This connector only supports delete where one or more partitions are deleted entirely. Configure write.delete.mode table property to allow row level deletions."); } validateTableMode(session, icebergTable); - transaction = icebergTable.newTransaction(); return handle; } @@ -1339,8 +1506,12 @@ public Optional finishDeleteWithOutput(ConnectorSession IcebergTableHandle handle = (IcebergTableHandle) tableHandle; Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); - RowDelta rowDelta = transaction.newRowDelta(); + RowDelta rowDelta = icebergTable.newRowDelta(); + Optional branchName = handle.getIcebergTableName().getBranchName(); + if (branchName.isPresent()) { + rowDelta.toBranch(branchName.get()); + } List commitTasks = fragments.stream() .map(slice -> commitTaskCodec.fromJson(slice.getBytes())) .collect(toImmutableList()); @@ -1376,7 +1547,6 @@ public Optional finishDeleteWithOutput(ConnectorSession } rowDelta.commit(); - transaction.commitTransaction(); return Optional.empty(); } @@ -1445,17 +1615,17 @@ public OptionalLong metadataDelete(ConnectorSession session, ConnectorTableHandl } TupleDomain domainPredicate = layoutHandle.getValidPredicate(); - return removeScanFiles(icebergTable, domainPredicate); + return removeScanFiles(handle, icebergTable, domainPredicate); } @Override public void setTableProperties(ConnectorSession session, ConnectorTableHandle tableHandle, Map properties) { IcebergTableHandle handle = (IcebergTableHandle) tableHandle; + validateNoBranchSpecified(handle, "SET TABLE PROPERTIES"); Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); - transaction = icebergTable.newTransaction(); - UpdateProperties updateProperties = transaction.updateProperties(); + UpdateProperties updateProperties = icebergTable.updateProperties(); for (Map.Entry entry : properties.entrySet()) { if (!tableProperties.getUpdatableProperties() .contains(entry.getKey())) { @@ -1472,7 +1642,6 @@ public void setTableProperties(ConnectorSession session, ConnectorTableHandle ta } updateProperties.commit(); - transaction.commitTransaction(); } private static PrestoWarning getPrestoWarning(String newPropertyKey, String propertyName) @@ -1492,18 +1661,20 @@ private static PrestoWarning getPrestoWarning(String newPropertyKey, String prop * * @return the number of rows deleted from all files */ - private OptionalLong removeScanFiles(Table icebergTable, TupleDomain predicate) + private OptionalLong removeScanFiles(IcebergTableHandle tableHandle, Table icebergTable, TupleDomain predicate) { - transaction = icebergTable.newTransaction(); - DeleteFiles deleteFiles = transaction.newDelete() - .deleteFromRowFilter(toIcebergExpression(predicate)); + DeleteFiles deleteFiles = icebergTable.newDelete(); + Optional branchName = tableHandle.getIcebergTableName().getBranchName(); + if (branchName.isPresent()) { + deleteFiles = deleteFiles.toBranch(branchName.get()); + } + deleteFiles.deleteFromRowFilter(toIcebergExpression(predicate)); deleteFiles.commit(); - transaction.commitTransaction(); - Map summary = icebergTable.currentSnapshot().summary(); - long deletedRecords = parseLong(summary.getOrDefault(DELETED_RECORDS_PROP, "0")); - long removedPositionDeletes = parseLong(summary.getOrDefault(REMOVED_POS_DELETES_PROP, "0")); - long removedEqualityDeletes = parseLong(summary.getOrDefault(REMOVED_EQ_DELETES_PROP, "0")); + Map summary = deleteFiles.apply().summary(); + long deletedRecords = Long.parseLong(summary.getOrDefault(DELETED_RECORDS_PROP, "0")); + long removedPositionDeletes = Long.parseLong(summary.getOrDefault(REMOVED_POS_DELETES_PROP, "0")); + long removedEqualityDeletes = Long.parseLong(summary.getOrDefault(REMOVED_EQ_DELETES_PROP, "0")); // Removed rows count is inaccurate when existing equality delete files return OptionalLong.of(deletedRecords - removedPositionDeletes - removedEqualityDeletes); } @@ -1579,7 +1750,14 @@ public ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTable { IcebergTableHandle handle = (IcebergTableHandle) tableHandle; Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); - int formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion(); + validateBranchExists(handle, icebergTable); + int formatVersion = opsFromTable(icebergTable).current().formatVersion(); + + if (formatVersion > MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS) { + throw new PrestoException(NOT_SUPPORTED, + format("Iceberg table updates for format version %s are not supported yet", formatVersion)); + } + if (formatVersion < MIN_FORMAT_VERSION_FOR_DELETE || !Optional.ofNullable(icebergTable.properties().get(TableProperties.UPDATE_MODE)) .map(mode -> mode.equals(MERGE_ON_READ.modeName())) @@ -1587,7 +1765,6 @@ public ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTable throw new RuntimeException("Iceberg table updates require at least format version 2 and update mode must be merge-on-read"); } validateTableMode(session, icebergTable); - transaction = icebergTable.newTransaction(); return handle .withUpdatedColumns(updatedColumns.stream() .map(IcebergColumnHandle.class::cast) @@ -1610,7 +1787,7 @@ public void finishUpdate(ConnectorSession session, ConnectorTableHandle tableHan getCompressionCodec(session), icebergTable.properties(), handle.getSortOrder()); - finishWrite(session, outputTableHandle, fragments, UPDATE_AFTER); + finishWrite(session, handle.getSchemaTableName(), outputTableHandle, fragments, UPDATE_AFTER); } protected Optional getDataLocationBasedOnWarehouseDataDir(SchemaTableName schemaTableName) @@ -1638,6 +1815,8 @@ public void createMaterializedView( MaterializedViewDefinition viewDefinition, boolean ignoreExisting) { + shouldRunInAutoCommitTransaction("CREATE MATERIALIZED VIEW"); + validateNoBranchInBaseTables(viewDefinition.getBaseTables(), "CREATE MATERIALIZED VIEW"); try { SchemaTableName viewName = viewMetadata.getTable(); Map materializedViewProperties = viewMetadata.getProperties(); @@ -1656,34 +1835,48 @@ public void createMaterializedView( viewMetadata.getColumns(), materializedViewProperties, viewMetadata.getComment()); - createTable(session, storageTableMetadata, false); - Map properties = new HashMap<>(); - properties.put(PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION, CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION + ""); - properties.put(PRESTO_MATERIALIZED_VIEW_ORIGINAL_SQL, viewDefinition.getOriginalSql()); - properties.put(PRESTO_MATERIALIZED_VIEW_STORAGE_SCHEMA, storageTableName.getSchemaName()); - properties.put(PRESTO_MATERIALIZED_VIEW_STORAGE_TABLE_NAME, storageTableName.getTableName()); - - String baseTablesStr = serializeSchemaTableNames(viewDefinition.getBaseTables()); - properties.put(PRESTO_MATERIALIZED_VIEW_BASE_TABLES, baseTablesStr); - properties.put(PRESTO_MATERIALIZED_VIEW_COLUMN_MAPPINGS, serializeColumnMappings(viewDefinition.getColumnMappings())); - checkState(viewDefinition.getOwner().isPresent(), "Materialized view owner is required"); - properties.put(PRESTO_MATERIALIZED_VIEW_OWNER, viewDefinition.getOwner().get()); - checkState(viewDefinition.getSecurityMode().isPresent(), "Materialized view security mode is required"); - properties.put(PRESTO_MATERIALIZED_VIEW_SECURITY_MODE, viewDefinition.getSecurityMode().get().name()); - - getStaleReadBehavior(materializedViewProperties) - .ifPresent(behavior -> properties.put(PRESTO_MATERIALIZED_VIEW_STALE_READ_BEHAVIOR, behavior.name())); - getStalenessWindow(materializedViewProperties) - .ifPresent(window -> properties.put(PRESTO_MATERIALIZED_VIEW_STALENESS_WINDOW, window.toString())); - MaterializedViewRefreshType refreshType = getRefreshType(materializedViewProperties); - properties.put(PRESTO_MATERIALIZED_VIEW_REFRESH_TYPE, refreshType.name()); - - for (SchemaTableName baseTable : viewDefinition.getBaseTables()) { - properties.put(getBaseTableViewPropertyName(baseTable), "0"); - } - - createIcebergView(session, viewName, viewMetadata.getColumns(), viewDefinition.getOriginalSql(), properties); + // Create materialized view should run after the creation of the underlying storage table + transactionContext.registerCallback(() -> { + try { + Map properties = new HashMap<>(); + properties.put(PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION, CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION + ""); + properties.put(PRESTO_MATERIALIZED_VIEW_ORIGINAL_SQL, viewDefinition.getOriginalSql()); + properties.put(PRESTO_MATERIALIZED_VIEW_STORAGE_SCHEMA, storageTableName.getSchemaName()); + properties.put(PRESTO_MATERIALIZED_VIEW_STORAGE_TABLE_NAME, storageTableName.getTableName()); + + String baseTablesStr = serializeSchemaTableNames(viewDefinition.getBaseTables()); + properties.put(PRESTO_MATERIALIZED_VIEW_BASE_TABLES, baseTablesStr); + properties.put(PRESTO_MATERIALIZED_VIEW_COLUMN_MAPPINGS, serializeColumnMappings(viewDefinition.getColumnMappings())); + properties.put(PRESTO_MATERIALIZED_VIEW_OWNER, viewDefinition.getOwner() + .orElseThrow(() -> new PrestoException(INVALID_VIEW, "Materialized view owner is required"))); + properties.put(PRESTO_MATERIALIZED_VIEW_SECURITY_MODE, viewDefinition.getSecurityMode() + .orElseThrow(() -> new PrestoException(INVALID_VIEW, "Materialized view security mode is required (set legacy_materialized_views=false)")) + .name()); + + getStaleReadBehavior(materializedViewProperties) + .ifPresent(behavior -> properties.put(PRESTO_MATERIALIZED_VIEW_STALE_READ_BEHAVIOR, behavior.name())); + getStalenessWindow(materializedViewProperties) + .ifPresent(window -> properties.put(PRESTO_MATERIALIZED_VIEW_STALENESS_WINDOW, window.toString())); + MaterializedViewRefreshType refreshType = getRefreshType(materializedViewProperties); + properties.put(PRESTO_MATERIALIZED_VIEW_REFRESH_TYPE, refreshType.name()); + + for (SchemaTableName baseTable : viewDefinition.getBaseTables()) { + properties.put(getBaseTableViewPropertyName(baseTable), "0"); + } + createIcebergView(session, viewName, viewMetadata.getColumns(), viewDefinition.getOriginalSql(), properties); + } + catch (Exception e) { + try { + dropStorageTable(session, storageTableName); + } + catch (Exception cleanupException) { + e.addSuppressed(cleanupException); + } + throw e; + } + }); + createTable(session, storageTableMetadata, false); } catch (PrestoException e) { if (e.getErrorCode() == NOT_SUPPORTED.toErrorCode()) { @@ -1693,123 +1886,119 @@ public void createMaterializedView( } } + private void dropStorageTable(ConnectorSession session, SchemaTableName storageTableName) + { + ConnectorTableHandle storageTableHandle = getTableHandle(session, storageTableName); + if (storageTableHandle != null) { + dropTable(session, storageTableHandle); + } + } + @Override public List listMaterializedViews(ConnectorSession session, String schemaName) { - ImmutableList.Builder materializedViews = ImmutableList.builder(); - List views = listViews(session, Optional.of(schemaName)); - for (SchemaTableName viewName : views) { - View icebergView = getIcebergView(session, viewName); - Map properties = icebergView.properties(); - if (properties.containsKey(PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION)) { - materializedViews.add(viewName); - } - } - - return materializedViews.build(); + return views.stream() + .filter(viewName -> getViewMetadata(session, viewName) + .map(IcebergViewMetadata::isMaterializedView) + .orElse(false)) + .collect(toImmutableList()); } @Override public Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) { - try { - View icebergView = getIcebergView(session, viewName); + Optional viewMetadata = getViewMetadata(session, viewName); + if (!viewMetadata.isPresent() || !viewMetadata.get().isMaterializedView()) { + return Optional.empty(); + } - Map viewProperties = icebergView.properties(); - String originalSql = viewProperties.get(PRESTO_MATERIALIZED_VIEW_ORIGINAL_SQL); + Map viewProperties = viewMetadata.get().getProperties(); + String originalSql = viewProperties.get(PRESTO_MATERIALIZED_VIEW_ORIGINAL_SQL); - if (originalSql == null) { - return Optional.empty(); - } + if (originalSql == null) { + return Optional.empty(); + } - // Validate format version - String formatVersion = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION); - int version; - try { - version = Integer.parseInt(formatVersion); - } - catch (NumberFormatException e) { - throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, - format("Invalid materialized view format version: %s", formatVersion)); - } + // Validate format version + String formatVersion = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION); + int version; + try { + version = Integer.parseInt(formatVersion); + } + catch (NumberFormatException e) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, + format("Invalid materialized view format version: %s", formatVersion)); + } - if (version != CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION) { - throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, - format("Materialized view format version %d is not supported by this version of Presto (current version: %d). Please upgrade Presto.", - version, CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION)); - } + if (version != CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, + format("Materialized view format version %d is not supported by this version of Presto (current version: %d). Please upgrade Presto.", + version, CURRENT_MATERIALIZED_VIEW_FORMAT_VERSION)); + } - String baseTablesStr = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_BASE_TABLES); - List baseTables; - if (baseTablesStr.isEmpty()) { - baseTables = ImmutableList.of(); - } - else { - baseTables = deserializeSchemaTableNames(baseTablesStr); - } + String baseTablesStr = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_BASE_TABLES); + List baseTables; + if (baseTablesStr.isEmpty()) { + baseTables = ImmutableList.of(); + } + else { + baseTables = deserializeSchemaTableNames(baseTablesStr); + } - String columnMappingsJson = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_COLUMN_MAPPINGS); - List columnMappings = deserializeColumnMappings(columnMappingsJson); + String columnMappingsJson = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_COLUMN_MAPPINGS); + List columnMappings = deserializeColumnMappings(columnMappingsJson); - String storageSchema = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STORAGE_SCHEMA); - String storageTableName = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STORAGE_TABLE_NAME); + String storageSchema = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STORAGE_SCHEMA); + String storageTableName = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STORAGE_TABLE_NAME); - String owner = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_OWNER); - ViewSecurity securityMode; - try { - securityMode = ViewSecurity.valueOf(getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_SECURITY_MODE)); - } - catch (IllegalArgumentException | NullPointerException e) { - throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, "Invalid or missing materialized view security mode"); - } + String owner = getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_OWNER); + ViewSecurity securityMode; + try { + securityMode = ViewSecurity.valueOf(getRequiredMaterializedViewProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_SECURITY_MODE)); + } + catch (IllegalArgumentException | NullPointerException e) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, "Invalid or missing materialized view security mode"); + } - // Parse staleness config - staleness window defaults to 0s if behavior is set - Optional staleReadBehavior = getOptionalEnumProperty( - viewProperties, PRESTO_MATERIALIZED_VIEW_STALE_READ_BEHAVIOR, MaterializedViewStaleReadBehavior.class); - Optional stalenessWindow = getOptionalDurationProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STALENESS_WINDOW); + // Parse staleness config - staleness window defaults to 0s if behavior is set + Optional staleReadBehavior = getOptionalEnumProperty( + viewProperties, PRESTO_MATERIALIZED_VIEW_STALE_READ_BEHAVIOR, MaterializedViewStaleReadBehavior.class); + Optional stalenessWindow = getOptionalDurationProperty(viewProperties, PRESTO_MATERIALIZED_VIEW_STALENESS_WINDOW); - Optional stalenessConfig = Optional.empty(); - if (staleReadBehavior.isPresent()) { - stalenessConfig = Optional.of(new MaterializedViewStalenessConfig( - staleReadBehavior.get(), - stalenessWindow.orElse(new Duration(0, TimeUnit.SECONDS)))); - } + Optional stalenessConfig = Optional.empty(); + if (staleReadBehavior.isPresent()) { + stalenessConfig = Optional.of(new MaterializedViewStalenessConfig( + staleReadBehavior.get(), + stalenessWindow.orElse(new Duration(0, TimeUnit.SECONDS)))); + } - Optional refreshType = getOptionalEnumProperty( - viewProperties, PRESTO_MATERIALIZED_VIEW_REFRESH_TYPE, MaterializedViewRefreshType.class); + Optional refreshType = getOptionalEnumProperty( + viewProperties, PRESTO_MATERIALIZED_VIEW_REFRESH_TYPE, MaterializedViewRefreshType.class); - return Optional.of(new MaterializedViewDefinition( - originalSql, - storageSchema, - storageTableName, - baseTables, - Optional.of(owner), - Optional.of(securityMode), - columnMappings, - ImmutableList.of(), - Optional.empty(), - stalenessConfig, - refreshType)); - } - catch (NoSuchViewException e) { - return Optional.empty(); - } - catch (PrestoException e) { - if (e.getErrorCode() == NOT_SUPPORTED.toErrorCode()) { - return Optional.empty(); - } - throw e; - } + return Optional.of(new MaterializedViewDefinition( + originalSql, + storageSchema, + storageTableName, + baseTables, + Optional.of(owner), + Optional.of(securityMode), + columnMappings, + ImmutableList.of(), + Optional.empty(), + stalenessConfig, + refreshType)); } @Override public void dropMaterializedView(ConnectorSession session, SchemaTableName viewName) { + shouldRunInAutoCommitTransaction("DROP MATERIALIZED VIEW"); Optional definition = getMaterializedView(session, viewName); if (definition.isPresent()) { + // Drop materialized view should run before the dropping of the underlying storage table dropIcebergView(session, viewName); SchemaTableName storageTableName = new SchemaTableName( definition.get().getSchema(), @@ -1832,8 +2021,12 @@ public MaterializedViewStatus getMaterializedViewStatus( return new MaterializedViewStatus(NOT_MATERIALIZED, ImmutableMap.of()); } - View icebergView = getIcebergView(session, materializedViewName); - Map props = icebergView.properties(); + Optional viewMetadata = getViewMetadata(session, materializedViewName); + if (!viewMetadata.isPresent()) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, + format("Materialized view metadata not found for %s", materializedViewName)); + } + Map props = viewMetadata.get().getProperties(); String lastRefreshSnapshotStr = props.get(PRESTO_MATERIALIZED_VIEW_LAST_REFRESH_SNAPSHOT_ID); if (lastRefreshSnapshotStr == null) { return new MaterializedViewStatus(NOT_MATERIALIZED, ImmutableMap.of()); @@ -1842,16 +2035,23 @@ public MaterializedViewStatus getMaterializedViewStatus( SchemaTableName storageTableName = new SchemaTableName(definition.get().getSchema(), definition.get().getTable()); Table storageTable = getIcebergTable(session, storageTableName); long lastRefreshSnapshotId = parseLong(lastRefreshSnapshotStr); - Snapshot lastRefreshSnapshot = storageTable.snapshot(lastRefreshSnapshotId); - if (lastRefreshSnapshot == null) { - throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, - format("Storage table snapshot %d not found for materialized view %s. " + - "The snapshot may have been expired. Consider refreshing the view.", - lastRefreshSnapshotId, materializedViewName)); + Optional lastFreshTime; + if (lastRefreshSnapshotId == 0L) { + // Empty table refresh: no real snapshot was created + lastFreshTime = Optional.empty(); + } + else { + Snapshot lastRefreshSnapshot = storageTable.snapshot(lastRefreshSnapshotId); + if (lastRefreshSnapshot == null) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, + format("Storage table snapshot %d not found for materialized view %s. " + + "The snapshot may have been expired. Consider refreshing the view.", + lastRefreshSnapshotId, materializedViewName)); + } + lastFreshTime = Optional.of(lastRefreshSnapshot.timestampMillis()); } - Optional lastFreshTime = Optional.of(lastRefreshSnapshot.timestampMillis()); - boolean isStale = false; + Map>> dataDisjuncts = new HashMap<>(); for (SchemaTableName baseTable : definition.get().getBaseTables()) { Table baseIcebergTable = getIcebergTable(session, baseTable); long currentSnapshotId = baseIcebergTable.currentSnapshot() != null @@ -1867,22 +2067,186 @@ public MaterializedViewStatus getMaterializedViewStatus( long recordedSnapshotId = parseLong(recordedSnapshotStr); if (currentSnapshotId != recordedSnapshotId) { - isStale = true; - break; + Optional>> partitionConstraints = detectChangedPartitions( + session, + baseTable, + recordedSnapshotId, + currentSnapshotId); + + if (partitionConstraints.isEmpty()) { + // Couldn't determine changed partitions, treat as not materialized + return new MaterializedViewStatus(NOT_MATERIALIZED, ImmutableMap.of(), lastFreshTime); + } + dataDisjuncts.put(baseTable, partitionConstraints.get()); + } + } + + if (dataDisjuncts.isEmpty()) { + return new MaterializedViewStatus(FULLY_MATERIALIZED, ImmutableMap.of(), lastFreshTime); + } + + Map predicatesMap = dataDisjuncts.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + // We pass an empty list of column names for now as they are not used when legacy_materialized_views=false + entry -> new MaterializedViewStatus.MaterializedDataPredicates(entry.getValue(), ImmutableList.of()))); + + return new MaterializedViewStatus(PARTIALLY_MATERIALIZED, predicatesMap, lastFreshTime); + } + + private Optional>> detectChangedPartitions( + ConnectorSession session, + SchemaTableName baseTable, + long recordedSnapshotId, + long currentSnapshotId) + { + Table baseIcebergTable = getIcebergTable(session, baseTable); + PartitionSpec spec = baseIcebergTable.spec(); + + if (spec.isUnpartitioned()) { + // Optional.of(emptyList) = "the entire table is stale" (no per-partition predicates to stitch on). + // This is distinct from Optional.empty() = "we cannot determine staleness" (bail out to NOT_MATERIALIZED). + return Optional.of(ImmutableList.of()); + } + + for (PartitionField field : spec.fields()) { + if (!field.transform().isIdentity()) { + return Optional.of(ImmutableList.of()); + } + } + + if (hasPartitionEvolution(baseIcebergTable, recordedSnapshotId, currentSnapshotId)) { + return Optional.empty(); + } + + Optional> changedPartitions = collectChangedPartitions( + baseIcebergTable, + recordedSnapshotId, + currentSnapshotId); + + if (changedPartitions.isEmpty()) { + return Optional.empty(); + } + + if (changedPartitions.get().size() > getMaterializedViewMaxChangedPartitions(session)) { + return Optional.empty(); + } + + List> partitionConstraints = convertPartitionsToConstraints( + session, + baseIcebergTable, + spec, + changedPartitions.get()); + + return Optional.of(partitionConstraints); + } + + private Optional> collectChangedPartitions( + Table icebergTable, + long fromSnapshotId, + long toSnapshotId) + { + // Check for operations that would make incremental partition detection unreliable. + // IncrementalAppendScan only tracks files added by APPEND operations, so we must + // reject any snapshot with an operation whose changes it cannot capture: + // - APPEND: Safe - new files are tracked by IncrementalAppendScan + // - REPLACE: Safe - files rewritten without changing data (compaction), no staleness + // - OVERWRITE: Unsafe - adds and removes files, but IncrementalAppendScan only sees + // APPEND operations, so partitions affected by overwrites would be missed + // - DELETE: Unsafe - data removed without adding new files, IncrementalAppendScan misses this + long fromSequenceNumber = icebergTable.snapshot(fromSnapshotId).sequenceNumber(); + long toSequenceNumber = icebergTable.snapshot(toSnapshotId).sequenceNumber(); + for (Snapshot snapshot : icebergTable.snapshots()) { + if (snapshot.sequenceNumber() > fromSequenceNumber && + snapshot.sequenceNumber() <= toSequenceNumber) { + String operation = snapshot.operation(); + if (!APPEND.equals(operation) && !REPLACE.equals(operation)) { + return Optional.empty(); + } + } + } + + Set partitions = new HashSet<>(); + + IncrementalAppendScan scan = icebergTable.newIncrementalAppendScan() + .fromSnapshotExclusive(fromSnapshotId) + .toSnapshot(toSnapshotId); + + try (CloseableIterable tasks = scan.planFiles()) { + for (FileScanTask task : tasks) { + partitions.add(task.file().partition()); + } + } + catch (IOException e) { + throw new UncheckedIOException("Failed to scan changed partitions", e); + } + + return Optional.of(ImmutableSet.copyOf(partitions)); + } + + private List> convertPartitionsToConstraints( + ConnectorSession session, + Table icebergTable, + PartitionSpec spec, + Set partitions) + { + List> constraints = new ArrayList<>(); + + for (StructLike partition : partitions) { + Map domainMap = new HashMap<>(); + + for (int i = 0; i < spec.fields().size(); i++) { + PartitionField field = spec.fields().get(i); + String sourceColumnName = icebergTable.schema().findColumnName(field.sourceId()); + + if (sourceColumnName == null) { + throw new PrestoException(ICEBERG_INVALID_MATERIALIZED_VIEW, + format("Partition field %s references non-existent column ID %d", field.name(), field.sourceId())); + } + + String normalizedColumnName = normalizeIdentifier(session, sourceColumnName); + + Object value = partition.get(i, Object.class); + Type icebergType = icebergTable.schema().findType(field.sourceId()); + Domain domain = createDomainFromIcebergPartitionValue(value, icebergType, toPrestoType(icebergType, typeManager)); + domainMap.put(normalizedColumnName, domain); + } + + if (!domainMap.isEmpty()) { + constraints.add(TupleDomain.withColumnDomains(domainMap)); } } - if (isStale) { - return new MaterializedViewStatus( - PARTIALLY_MATERIALIZED, - ImmutableMap.of(), - lastFreshTime); + return constraints; + } + + private boolean hasPartitionEvolution( + Table icebergTable, + long fromSnapshotId, + long toSnapshotId) + { + Snapshot fromSnapshot = icebergTable.snapshot(fromSnapshotId); + if (fromSnapshot == null) { + throw new PrestoException(ICEBERG_INVALID_SNAPSHOT_ID, + format("Base table snapshot %d no longer exists (possibly expired). Materialized view requires full refresh.", fromSnapshotId)); + } + + Snapshot toSnapshot = icebergTable.snapshot(toSnapshotId); + if (toSnapshot == null) { + throw new PrestoException(ICEBERG_INVALID_SNAPSHOT_ID, + format("Base table snapshot %d does not exist", toSnapshotId)); } - return new MaterializedViewStatus( - FULLY_MATERIALIZED, - ImmutableMap.of(), - lastFreshTime); + // Get partition spec IDs from manifests in each snapshot + Set fromSpecIds = fromSnapshot.allManifests(icebergTable.io()).stream() + .map(ManifestFile::partitionSpecId) + .collect(toImmutableSet()); + + Set toSpecIds = toSnapshot.allManifests(icebergTable.io()).stream() + .map(ManifestFile::partitionSpecId) + .collect(toImmutableSet()); + + return !fromSpecIds.equals(toSpecIds); } @Override @@ -1890,6 +2254,7 @@ public ConnectorInsertTableHandle beginRefreshMaterializedView( ConnectorSession session, ConnectorTableHandle tableHandle) { + shouldRunInAutoCommitTransaction("REFRESH MATERIALIZED VIEW"); IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; if (icebergTableHandle.getMaterializedViewName().isEmpty()) { @@ -1902,9 +2267,7 @@ public ConnectorInsertTableHandle beginRefreshMaterializedView( IcebergTableHandle storageTableHandle = getTableHandle(session, storageTableName); Table storageTable = getIcebergTable(session, storageTableName); - transaction = storageTable.newTransaction(); - - transaction.newDelete().deleteFromRowFilter(alwaysTrue()).commit(); + storageTable.newDelete().deleteFromRowFilter(alwaysTrue()).commit(); SchemaTableName materializedViewName = icebergTableHandle.getMaterializedViewName().get(); @@ -1963,7 +2326,15 @@ public Optional finishRefreshMaterializedView( } } - updateIcebergViewProperties(session, materializedViewName, properties); + if (fragments.isEmpty()) { + // When no data was written, finishInsert already committed the transaction. + // Callbacks registered after that commit won't execute, so update properties directly. + updateIcebergViewProperties(session, materializedViewName, properties); + } + else { + // Update materialized view should run after the data refresh of the underlying storage table + this.transactionContext.registerCallback(() -> updateIcebergViewProperties(session, materializedViewName, properties)); + } }); return result; @@ -2050,12 +2421,21 @@ private static Optional getOptionalDurationProperty(Map getViewMetadata(ConnectorSession session, SchemaTableName viewName) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg Hive catalog does not support native Iceberg views."); + Optional
hiveTable = getHiveTable(session, viewName); + if (!hiveTable.isPresent()) { + return Optional.empty(); + } + + Table table = hiveTable.get(); + if (!isPrestoView(table)) { + return Optional.empty(); + } + + List columns = table.getDataColumns().stream() + .map(column -> ColumnMetadata.builder() + .setName(column.getName()) + .setType(column.getType().getType(typeManager)) + .setComment(column.getComment().orElse(null)) + .build()) + .collect(toImmutableList()); + + Map tableProperties = table.getParameters().entrySet().stream() + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + Optional comment = Optional.ofNullable(table.getParameters().get(TABLE_COMMENT)); + + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata( + viewName, + columns, + tableProperties, + comment); + + return Optional.of(new IcebergViewMetadata(table.getParameters(), tableMetadata)); } @Override @@ -240,6 +274,9 @@ protected boolean tableExists(ConnectorSession session, SchemaTableName schemaTa if (!hiveTable.isPresent()) { return false; } + if (isPrestoView(hiveTable.get())) { + return false; + } if (!isIcebergTable(hiveTable.get())) { throw new UnknownTableTypeException("Not an Iceberg table: " + schemaTableName); } @@ -305,6 +342,7 @@ public List listTables(ConnectorSession session, Optional properties) { + shouldRunInAutoCommitTransaction("CREATE SCHEMA"); Optional location = getLocation(properties).map(uri -> { try { hdfsEnvironment.getFileSystem(new HdfsContext(session, schemaName), new Path(uri)); @@ -329,6 +367,7 @@ public void createSchema(ConnectorSession session, String schemaName, Map existing = getHiveTable(session, viewName); if (existing.isPresent()) { - if (!replace || !isPrestoView(existing.get())) { + if (!replace || !isPrestoView(existing.get()) || isIcebergMaterializedView(existing.get())) { throw new ViewAlreadyExistsException(viewName); } @@ -471,7 +515,11 @@ public List listViews(ConnectorSession session, Optional table = getHiveTable(session, schemaTableName); + if (table.isPresent() && !isIcebergMaterializedView(table.get())) { + tableNames.add(schemaTableName); + } } } return tableNames.build(); @@ -480,7 +528,23 @@ public List listViews(ConnectorSession session, Optional listMaterializedViews(ConnectorSession session, String schemaName) { - return ImmutableList.of(); + MetastoreContext metastoreContext = getMetastoreContext(session); + ImmutableList.Builder materializedViews = ImmutableList.builder(); + + Optional> viewNames = metastore.getAllViews(metastoreContext, schemaName); + if (!viewNames.isPresent()) { + return ImmutableList.of(); + } + + for (String viewName : viewNames.get()) { + SchemaTableName schemaTableName = new SchemaTableName(schemaName, viewName); + Optional
table = getHiveTable(session, schemaTableName); + if (table.isPresent() && isIcebergMaterializedView(table.get())) { + materializedViews.add(schemaTableName); + } + } + + return materializedViews.build(); } @Override @@ -496,7 +560,7 @@ public Map getViews(ConnectorSession s } for (SchemaTableName schemaTableName : tableNames) { Optional
table = getHiveTable(session, schemaTableName); - if (table.isPresent() && isPrestoView(table.get())) { + if (table.isPresent() && isPrestoView(table.get()) && !isIcebergMaterializedView(table.get())) { verifyAndPopulateViews(table.get(), schemaTableName, decodeViewData(table.get().getViewOriginalText().get()), views); } } @@ -506,6 +570,7 @@ public Map getViews(ConnectorSession s @Override public void renameView(ConnectorSession session, SchemaTableName source, SchemaTableName target) { + shouldRunInAutoCommitTransaction("RENAME VIEW"); // Not checking if source view exists as this is already done in RenameViewTask metastore.renameTable(getMetastoreContext(session), source.getSchemaName(), source.getTableName(), target.getSchemaName(), target.getTableName()); } @@ -513,6 +578,7 @@ public void renameView(ConnectorSession session, SchemaTableName source, SchemaT @Override public void dropView(ConnectorSession session, SchemaTableName viewName) { + shouldRunInAutoCommitTransaction("DROP VIEW"); ConnectorViewDefinition view = getViews(session, viewName.toSchemaTablePrefix()).get(viewName); checkIfNullView(view, viewName); @@ -572,8 +638,7 @@ public TableStatisticsMetadata getStatisticsCollectionMetadata(ConnectorSession Set supportedStatistics = ImmutableSet.builder() .addAll(hiveColumnStatistics) // iceberg table-supported statistics - .addAll(!connectorSystemConfig.isNativeExecution() ? - super.getStatisticsCollectionMetadata(session, tableMetadata).getColumnStatistics() : ImmutableSet.of()) + .addAll(super.getStatisticsCollectionMetadata(session, tableMetadata).getColumnStatistics()) .build(); Set tableStatistics = ImmutableSet.of(ROW_COUNT); return new TableStatisticsMetadata(supportedStatistics, tableStatistics, emptyList()); @@ -599,12 +664,6 @@ private Set getHiveSupportedColumnStatistics(ConnectorS .collect(toImmutableSet()); } - @Override - public ConnectorTableHandle beginStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle) - { - return tableHandle; - } - @Override public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle tableHandle, Collection computedStatistics) { @@ -714,13 +773,51 @@ protected void createIcebergView( String viewSql, Map properties) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg Hive catalog does not support native Iceberg views for materialized views."); + MetastoreContext metastoreContext = getMetastoreContext(session); + + ImmutableMap.Builder tableProperties = ImmutableMap.builder(); + tableProperties.putAll(properties); + tableProperties.putAll(createIcebergViewProperties(session, nodeVersion.toString())); + + ConnectorTableMetadata viewMetadata = new ConnectorTableMetadata(viewName, columns); + + Table table = createTableObjectForViewCreation( + session, + viewMetadata, + tableProperties.build(), + new HiveTypeTranslator(), + metastoreContext, + encodeViewData(viewSql)); + + PrincipalPrivileges privileges = buildInitialPrivilegeSet(session.getUser()); + + try { + metastore.createTable(metastoreContext, table, privileges, emptyList()); + } + catch (TableAlreadyExistsException e) { + throw new PrestoException(ALREADY_EXISTS, "Materialized view already exists: " + viewName); + } + + tableCache.invalidate(viewName); } @Override protected void dropIcebergView(ConnectorSession session, SchemaTableName schemaTableName) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg Hive catalog does not support native Iceberg views for materialized views."); + MetastoreContext metastoreContext = getMetastoreContext(session); + + try { + metastore.dropTable( + metastoreContext, + schemaTableName.getSchemaName(), + schemaTableName.getTableName(), + true); + } + catch (TableNotFoundException e) { + throw new PrestoException(NOT_FOUND, "Materialized view not found: " + schemaTableName); + } + + tableCache.invalidate(schemaTableName); } @Override @@ -729,6 +826,36 @@ protected void updateIcebergViewProperties( SchemaTableName viewName, Map properties) { - throw new PrestoException(NOT_SUPPORTED, "Iceberg Hive catalog does not support native Iceberg views for materialized views."); + MetastoreContext metastoreContext = getMetastoreContext(session); + + Optional
existingTable = getHiveTable(session, viewName); + if (!existingTable.isPresent() || !isIcebergMaterializedView(existingTable.get())) { + throw new PrestoException(NOT_FOUND, "Materialized view not found: " + viewName); + } + + Table table = existingTable.get(); + + ImmutableMap.Builder mergedProperties = ImmutableMap.builder(); + mergedProperties.putAll(table.getParameters()); + mergedProperties.putAll(properties); + + Table updatedTable = Table.builder(table) + .setParameters(mergedProperties.buildKeepingLast()) + .build(); + + PrincipalPrivileges privileges = buildInitialPrivilegeSet(table.getOwner()); + metastore.replaceTable( + metastoreContext, + viewName.getSchemaName(), + viewName.getTableName(), + updatedTable, + privileges); + + tableCache.invalidate(viewName); + } + + private static boolean isIcebergMaterializedView(Table table) + { + return table.getParameters().containsKey(PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java index ca37b7910b009..a8f245340329f 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java @@ -19,18 +19,20 @@ import com.facebook.presto.hive.NodeVersion; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.iceberg.statistics.StatisticsFileCache; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSystemConfig; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.procedure.ProcedureRegistry; import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.transaction.IsolationLevel; import jakarta.inject.Inject; import java.util.List; import static com.facebook.presto.spi.MaterializedViewDefinition.ColumnMapping; +import static com.facebook.presto.spi.transaction.IsolationLevel.REPEATABLE_READ; import static java.util.Objects.requireNonNull; public class IcebergHiveMetadataFactory @@ -93,7 +95,12 @@ public IcebergHiveMetadataFactory( this.connectorSystemConfig = requireNonNull(connectorSystemConfig, "connectorSystemConfig is null"); } - public ConnectorMetadata create() + public IcebergTransactionMetadata create() + { + return create(REPEATABLE_READ, true); + } + + public IcebergTransactionMetadata create(IsolationLevel isolationLevel, boolean autoCommitContext) { return new IcebergHiveMetadata( catalogName, @@ -112,6 +119,8 @@ public ConnectorMetadata create() statisticsFileCache, manifestFileCache, tableProperties, - connectorSystemConfig); + connectorSystemConfig, + isolationLevel, + autoCommitContext); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java index 3823019e1dc73..38242d8ae782e 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveModule.java @@ -16,10 +16,12 @@ import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.presto.hive.MetastoreClientConfig; import com.facebook.presto.hive.PartitionMutator; +import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.HiveMetastoreCacheStats; import com.facebook.presto.hive.metastore.HivePartitionMutator; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.MetastoreCacheStats; import com.facebook.presto.hive.metastore.MetastoreConfig; import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreConfig; @@ -30,6 +32,8 @@ import java.util.Optional; import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.ALL; +import static com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheType.TABLE; import static com.google.common.base.Preconditions.checkArgument; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -50,14 +54,15 @@ public IcebergHiveModule(String connectorId, Optional met public void setup(Binder binder) { install(new IcebergHiveMetastoreModule(this.connectorId, this.metastore)); + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(IcebergHiveTableOperationsConfig.class); configBinder(binder).bindConfig(MetastoreClientConfig.class); configBinder(binder).bindConfig(ThriftHiveMetastoreConfig.class); - long metastoreCacheTtl = buildConfigObject(MetastoreClientConfig.class).getMetastoreCacheTtl().toMillis(); - checkArgument(metastoreCacheTtl == 0, "In-memory hive metastore caching must not be enabled for Iceberg"); + checkArgument(isCachingAllowed(buildConfigObject(MetastoreClientConfig.class)), + "In-memory hive metastore caching for tables must not be enabled for Iceberg"); binder.bind(PartitionMutator.class).to(HivePartitionMutator.class).in(Scopes.SINGLETON); @@ -68,4 +73,18 @@ public void setup(Binder binder) configBinder(binder).bindConfig(MetastoreConfig.class); } + + private boolean isCachingAllowed(MetastoreClientConfig config) + { + if (!config.getEnabledCaches().isEmpty()) { + return !config.getEnabledCaches().contains(ALL) && !config.getEnabledCaches().contains(TABLE); + } + + if (!config.getDisabledCaches().isEmpty()) { + return config.getDisabledCaches().contains(ALL) || config.getDisabledCaches().contains(TABLE); + } + + return config.getMetastoreCacheScope() != MetastoreCacheScope.ALL || + config.getDefaultMetastoreCacheTtl().toMillis() == 0; + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataFactory.java index 19e820832fefc..549e2055ff13c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergMetadataFactory.java @@ -13,9 +13,12 @@ */ package com.facebook.presto.iceberg; -import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; +import com.facebook.presto.spi.transaction.IsolationLevel; public interface IcebergMetadataFactory { - ConnectorMetadata create(); + IcebergTransactionMetadata create(); + + IcebergTransactionMetadata create(IsolationLevel isolationLevel, boolean autoCommitContext); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java index 9202be8e4d5c4..e47cb76abdf50 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadata.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.procedure.ProcedureRegistry; import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.transaction.IsolationLevel; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.fs.Path; @@ -75,7 +76,9 @@ import static com.facebook.presto.iceberg.IcebergUtil.getColumnsForWrite; import static com.facebook.presto.iceberg.IcebergUtil.getNativeIcebergTable; import static com.facebook.presto.iceberg.IcebergUtil.getNativeIcebergView; +import static com.facebook.presto.iceberg.IcebergUtil.getViewComment; import static com.facebook.presto.iceberg.IcebergUtil.populateTableProperties; +import static com.facebook.presto.iceberg.IcebergUtil.validateViewDefinitionForBranches; import static com.facebook.presto.iceberg.PartitionFields.parsePartitionFields; import static com.facebook.presto.iceberg.PartitionSpecConverter.toPrestoPartitionSpec; import static com.facebook.presto.iceberg.SchemaConverter.toPrestoSchema; @@ -120,10 +123,12 @@ public IcebergNativeMetadata( NodeVersion nodeVersion, FilterStatsCalculatorService filterStatsCalculatorService, StatisticsFileCache statisticsFileCache, - IcebergTableProperties tableProperties) + IcebergTableProperties tableProperties, + IsolationLevel isolationLevel, + boolean autoCommitContext) { super(typeManager, procedureRegistry, functionResolution, rowExpressionService, commitTaskCodec, columnMappingsCodec, schemaTableNamesCodec, - nodeVersion, filterStatsCalculatorService, statisticsFileCache, tableProperties); + nodeVersion, filterStatsCalculatorService, statisticsFileCache, tableProperties, isolationLevel, autoCommitContext); this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); this.catalogType = requireNonNull(catalogType, "catalogType is null"); this.warehouseDataDir = Optional.ofNullable(catalogFactory.getCatalogWarehouseDataDir()); @@ -135,8 +140,7 @@ protected Table getRawIcebergTable(ConnectorSession session, SchemaTableName sch return getNativeIcebergTable(catalogFactory, session, schemaTableName); } - @Override - protected View getIcebergView(ConnectorSession session, SchemaTableName schemaTableName) + private View getIcebergView(ConnectorSession session, SchemaTableName schemaTableName) { try { return icebergViews.computeIfAbsent( @@ -152,6 +156,27 @@ protected View getIcebergView(ConnectorSession session, SchemaTableName schemaTa } } + @Override + protected Optional getViewMetadata(ConnectorSession session, SchemaTableName viewName) + { + Catalog catalog = catalogFactory.getCatalog(session); + if (!(catalog instanceof ViewCatalog)) { + return Optional.empty(); + } + try { + View view = getIcebergView(session, viewName); + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata( + viewName, + getColumnMetadata(session, view), + createViewMetadataProperties(view), + getViewComment(view)); + return Optional.of(new IcebergViewMetadata(view.properties(), tableMetadata)); + } + catch (NoSuchViewException e) { + return Optional.empty(); + } + } + @Override protected boolean tableExists(ConnectorSession session, SchemaTableName schemaTableName) { @@ -213,6 +238,7 @@ public List listTables(ConnectorSession session, Optional properties) { + shouldRunInAutoCommitTransaction("CREATE SCHEMA"); catalogFactory.getNamespaces(session).createNamespace(toIcebergNamespace(Optional.of(schemaName), catalogFactory.isNestedNamespaceEnabled()), properties.entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> e.getValue().toString()))); @@ -221,6 +247,7 @@ public void createSchema(ConnectorSession session, String schemaName, Map listMaterializedViews(ConnectorSession session, Str @Override public void dropView(ConnectorSession session, SchemaTableName viewName) { + shouldRunInAutoCommitTransaction("DROP VIEW"); Catalog catalog = catalogFactory.getCatalog(session); if (!(catalog instanceof ViewCatalog)) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support dropping views"); @@ -370,6 +401,7 @@ public void dropView(ConnectorSession session, SchemaTableName viewName) @Override public void renameView(ConnectorSession session, SchemaTableName source, SchemaTableName target) { + shouldRunInAutoCommitTransaction("RENAME VIEW"); Catalog catalog = catalogFactory.getCatalog(session); if (!(catalog instanceof ViewCatalog)) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support renaming views"); @@ -403,27 +435,27 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con TableIdentifier tableIdentifier = toIcebergTableIdentifier(schemaTableName, catalogFactory.isNestedNamespaceEnabled()); String targetPath = getTableLocation(tableMetadata.getProperties()); if (!isNullOrEmpty(targetPath)) { - transaction = catalogFactory.getCatalog(session).newCreateTableTransaction( + openCreateTableTransaction(schemaTableName, catalogFactory.getCatalog(session).newCreateTableTransaction( tableIdentifier, schema, partitionSpec, targetPath, - populateTableProperties(this, tableMetadata, tableProperties, fileFormat, session)); + populateTableProperties(this, tableMetadata, tableProperties, fileFormat, session))); } else { - transaction = catalogFactory.getCatalog(session).newCreateTableTransaction( + openCreateTableTransaction(schemaTableName, catalogFactory.getCatalog(session).newCreateTableTransaction( tableIdentifier, schema, partitionSpec, - populateTableProperties(this, tableMetadata, tableProperties, fileFormat, session)); + populateTableProperties(this, tableMetadata, tableProperties, fileFormat, session))); } } catch (AlreadyExistsException e) { throw new TableAlreadyExistsException(schemaTableName); } - Table icebergTable = transaction.table(); - ReplaceSortOrder replaceSortOrder = transaction.replaceSortOrder(); + Table icebergTable = getIcebergTable(session, schemaTableName); + ReplaceSortOrder replaceSortOrder = icebergTable.replaceSortOrder(); SortOrder sortOrder = parseSortFields(schema, getSortOrder(tableMetadata.getProperties())); List sortFields = getSupportedSortFields(icebergTable.schema(), sortOrder); for (SortField sortField : sortFields) { @@ -444,7 +476,7 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con return new IcebergOutputTableHandle( schemaName, - new IcebergTableName(tableName, DATA, Optional.empty(), Optional.empty()), + new IcebergTableName(tableName, DATA, Optional.empty(), Optional.empty(), Optional.empty()), toPrestoSchema(icebergTable.schema(), typeManager), toPrestoPartitionSpec(icebergTable.spec(), typeManager), getColumnsForWrite(icebergTable.schema(), icebergTable.spec(), typeManager), @@ -458,6 +490,7 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con @Override public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { + shouldRunInAutoCommitTransaction("DROP TABLE"); IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can be dropped"); TableIdentifier tableIdentifier = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName(), catalogFactory.isNestedNamespaceEnabled()); @@ -467,6 +500,7 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle @Override public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTable) { + shouldRunInAutoCommitTransaction("RENAME TABLE"); IcebergTableHandle icebergTableHandle = (IcebergTableHandle) tableHandle; verify(icebergTableHandle.getIcebergTableName().getTableType() == DATA, "only the data table can be renamed"); TableIdentifier from = toIcebergTableIdentifier(icebergTableHandle.getSchemaTableName(), catalogFactory.isNestedNamespaceEnabled()); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadataFactory.java index 72f11ce078166..d43ff03385551 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergNativeMetadataFactory.java @@ -17,17 +17,19 @@ import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.hive.NodeVersion; import com.facebook.presto.iceberg.statistics.StatisticsFileCache; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.plan.FilterStatsCalculatorService; import com.facebook.presto.spi.procedure.ProcedureRegistry; import com.facebook.presto.spi.relation.RowExpressionService; +import com.facebook.presto.spi.transaction.IsolationLevel; import jakarta.inject.Inject; import java.util.List; +import static com.facebook.presto.spi.transaction.IsolationLevel.REPEATABLE_READ; import static java.util.Objects.requireNonNull; public class IcebergNativeMetadataFactory @@ -78,10 +80,15 @@ public IcebergNativeMetadataFactory( this.tableProperties = requireNonNull(tableProperties, "tableProperties is null"); } - public ConnectorMetadata create() + public IcebergTransactionMetadata create() { - return new IcebergNativeMetadata(catalogFactory, typeManager, procedureRegistry, functionResolution, rowExpressionService, - commitTaskCodec, columnMappingsCodec, schemaTableNamesCodec, catalogType, nodeVersion, filterStatsCalculatorService, - statisticsFileCache, tableProperties); + return create(REPEATABLE_READ, true); + } + + public IcebergTransactionMetadata create(IsolationLevel isolationLevel, boolean autoCommitContext) + { + return new IcebergNativeMetadata(catalogFactory, typeManager, procedureRegistry, functionResolution, + rowExpressionService, commitTaskCodec, columnMappingsCodec, schemaTableNamesCodec, catalogType, nodeVersion, + filterStatsCalculatorService, statisticsFileCache, tableProperties, isolationLevel, autoCommitContext); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java index 45e8a7164df8f..c5cc4e68d66a8 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSinkProvider.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Optional; +import static com.facebook.presto.iceberg.IcebergSessionProperties.getMaxPartitionsPerWriter; import static com.facebook.presto.iceberg.IcebergUtil.getLocationProvider; import static com.facebook.presto.iceberg.IcebergUtil.getShallowWrappedIcebergTable; import static com.facebook.presto.iceberg.PartitionSpecConverter.toIcebergPartitionSpec; @@ -51,7 +52,6 @@ public class IcebergPageSinkProvider private final JsonCodec jsonCodec; private final IcebergFileWriterFactory fileWriterFactory; private final PageIndexerFactory pageIndexerFactory; - private final int maxOpenPartitions; private final SortParameters sortParameters; @Inject @@ -60,15 +60,12 @@ public IcebergPageSinkProvider( JsonCodec jsonCodec, IcebergFileWriterFactory fileWriterFactory, PageIndexerFactory pageIndexerFactory, - IcebergConfig icebergConfig, SortParameters sortParameters) { this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.jsonCodec = requireNonNull(jsonCodec, "jsonCodec is null"); this.fileWriterFactory = requireNonNull(fileWriterFactory, "fileWriterFactory is null"); this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); - requireNonNull(icebergConfig, "icebergConfig is null"); - this.maxOpenPartitions = icebergConfig.getMaxPartitionsPerWriter(); this.sortParameters = sortParameters; } @@ -109,7 +106,7 @@ private ConnectorPageSink createPageSink(ConnectorSession session, IcebergWritab jsonCodec, session, tableHandle.getFileFormat(), - maxOpenPartitions, + getMaxPartitionsPerWriter(session), tableHandle.getSortOrder(), sortParameters); } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java index c789da98caa02..bbd26509292a0 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSessionProperties.java @@ -72,6 +72,8 @@ public final class IcebergSessionProperties public static final String STATISTICS_KLL_SKETCH_K_PARAMETER = "statistics_kll_sketch_k_parameter"; public static final String TARGET_SPLIT_SIZE_BYTES = "target_split_size_bytes"; public static final String MATERIALIZED_VIEW_STORAGE_PREFIX = "materialized_view_storage_prefix"; + public static final String MAX_PARTITIONS_PER_WRITER = "max_partitions_per_writer"; + public static final String MATERIALIZED_VIEW_MAX_CHANGED_PARTITIONS = "materialized_view_max_changed_partitions"; private final List> sessionProperties; @@ -214,6 +216,22 @@ public IcebergSessionProperties( "The K parameter for the Apache DataSketches KLL sketch when computing histogram statistics", icebergConfig.getStatisticsKllSketchKParameter(), false)) + .add(new PropertyMetadata<>( + MAX_PARTITIONS_PER_WRITER, + "Maximum number of partitions per writer", + INTEGER, + Integer.class, + icebergConfig.getMaxPartitionsPerWriter(), + false, + value -> { + int intValue = ((Number) value).intValue(); + if (intValue < 1) { + throw new PrestoException(INVALID_SESSION_PROPERTY, + format("Invalid value for %s: %s. It must be greater than or equal to 1.", MAX_PARTITIONS_PER_WRITER, intValue)); + } + return intValue; + }, + integer -> integer)) .add(longProperty( TARGET_SPLIT_SIZE_BYTES, "The target split size. Set to 0 to use the iceberg table's read.split.target-size property", @@ -225,6 +243,12 @@ public IcebergSessionProperties( "This is only used when the storage_table table property is not explicitly set. " + "When a custom table name is provided, it takes precedence over this prefix.", icebergConfig.getMaterializedViewStoragePrefix(), + false)) + .add(integerProperty( + MATERIALIZED_VIEW_MAX_CHANGED_PARTITIONS, + "Maximum number of changed partitions to track for materialized view staleness detection. " + + "If the number of changed partitions exceeds this threshold, the materialized view will fall back to full recompute.", + icebergConfig.getMaterializedViewMaxChangedPartitions(), false)); nessieConfig.ifPresent((config) -> propertiesBuilder @@ -365,6 +389,11 @@ public static int getStatisticsKllSketchKParameter(ConnectorSession session) return session.getProperty(STATISTICS_KLL_SKETCH_K_PARAMETER, Integer.class); } + public static int getMaxPartitionsPerWriter(ConnectorSession session) + { + return session.getProperty(MAX_PARTITIONS_PER_WRITER, Integer.class); + } + public static Long getTargetSplitSize(ConnectorSession session) { return session.getProperty(TARGET_SPLIT_SIZE_BYTES, Long.class); @@ -374,4 +403,9 @@ public static String getMaterializedViewStoragePrefix(ConnectorSession session) { return session.getProperty(MATERIALIZED_VIEW_STORAGE_PREFIX, String.class); } + + public static int getMaterializedViewMaxChangedPartitions(ConnectorSession session) + { + return session.getProperty(MATERIALIZED_VIEW_MAX_CHANGED_PARTITIONS, Integer.class); + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java index aeaf5dea8b202..c1b2f580afbb1 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitManager.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.iceberg.changelog.ChangelogSplitSource; import com.facebook.presto.iceberg.equalitydeletes.EqualityDeletesSplitSource; +import com.facebook.presto.iceberg.transaction.IcebergTransactionManager; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplitSource; import com.facebook.presto.spi.ConnectorTableLayoutHandle; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java index 98ee9f2693450..9d4cd1a615636 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergSplitSource.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.ConnectorSplit; import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SplitWeight; import com.facebook.presto.spi.connector.ConnectorPartitionHandle; import com.facebook.presto.spi.schedule.NodeSelectionStrategy; @@ -46,6 +47,7 @@ import static com.facebook.presto.iceberg.IcebergUtil.getTargetSplitSize; import static com.facebook.presto.iceberg.IcebergUtil.metadataColumnsMatchPredicates; import static com.facebook.presto.iceberg.IcebergUtil.partitionDataFromStructLike; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.limit; import static java.util.Objects.requireNonNull; @@ -124,6 +126,13 @@ private ConnectorSplit toIcebergSplit(FileScanTask task) PartitionSpec spec = task.spec(); Optional partitionData = partitionDataFromStructLike(spec, task.file().partition()); + // Validate no PUFFIN deletion vectors (Iceberg v3 feature not yet supported) + for (org.apache.iceberg.DeleteFile deleteFile : task.deletes()) { + if (deleteFile.format() == org.apache.iceberg.FileFormat.PUFFIN) { + throw new PrestoException(NOT_SUPPORTED, "Iceberg deletion vectors (PUFFIN format) are not supported"); + } + } + // TODO: We should leverage residual expression and convert that to TupleDomain. // The predicate here is used by readers for predicate push down at reader level, // so when we do not use residual expression, we are just wasting CPU cycles diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java index e619cdbcc2ef0..05ca054e18935 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableName.java @@ -42,13 +42,15 @@ public class IcebergTableName { private static final Pattern TABLE_PATTERN = Pattern.compile("" + - "(?
[^$@]+)" + + "(?
[^$@]+?)" + + "(?:\\.branch_(?[^$@.]+))?" + "(?:@(?[0-9]+))?" + "(?:\\$(?[^@]+)(?:@(?[0-9]+))?)?"); private final String tableName; private final IcebergTableType icebergTableType; private final Optional snapshotId; + private final Optional branchName; private final Optional changelogEndSnapshot; @@ -59,11 +61,13 @@ public IcebergTableName( @JsonProperty("tableName") String tableName, @JsonProperty("tableType") IcebergTableType icebergTableType, @JsonProperty("snapshotId") Optional snapshotId, + @JsonProperty("branchName") Optional branchName, @JsonProperty("changelogEndSnapshot") Optional changelogEndSnapshot) { this.tableName = requireNonNull(tableName, "tableName is null"); this.icebergTableType = requireNonNull(icebergTableType, "tableType is null"); this.snapshotId = requireNonNull(snapshotId, "snapshotId is null"); + this.branchName = requireNonNull(branchName, "branchName is null"); this.changelogEndSnapshot = requireNonNull(changelogEndSnapshot, "changelogEndSnapshot is null"); } @@ -79,6 +83,12 @@ public IcebergTableType getTableType() return icebergTableType; } + @JsonProperty + public Optional getBranchName() + { + return branchName; + } + @JsonProperty public Optional getChangelogEndSnapshot() { @@ -115,10 +125,16 @@ public static IcebergTableName from(String name) } String table = match.group("table"); + String branch = match.group("branch"); String typeString = match.group("type"); String version1 = match.group("ver1"); String version2 = match.group("ver2"); + // Branches cannot be combined with snapshot versions + if (branch != null && (version1 != null || version2 != null)) { + throw new PrestoException(NOT_SUPPORTED, format("Invalid Iceberg table name (cannot use @ version with branch): %s", name)); + } + IcebergTableType type = DATA; if (typeString != null) { try { @@ -154,6 +170,6 @@ else if (version1 != null || version2 != null) { throw new PrestoException(NOT_SUPPORTED, format("Invalid Iceberg table name (cannot use @ version with table type '%s'): %s", type, name)); } - return new IcebergTableName(table, type, version, changelogEndVersion); + return new IcebergTableName(table, type, version, Optional.ofNullable(branch), changelogEndVersion); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index 6ec40b607a022..e06553e66bc65 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -53,6 +53,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.BaseTransaction; import org.apache.iceberg.ContentFile; import org.apache.iceberg.ContentScanTask; import org.apache.iceberg.DataFile; @@ -67,6 +68,7 @@ import org.apache.iceberg.Scan; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; @@ -107,6 +109,8 @@ import java.util.stream.Stream; import static com.facebook.airlift.units.DataSize.succinctBytes; +import static com.facebook.presto.common.predicate.Domain.onlyNull; +import static com.facebook.presto.common.predicate.Domain.singleValue; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.Chars.isCharType; @@ -148,6 +152,7 @@ import static com.facebook.presto.iceberg.TypeConverter.toIcebergType; import static com.facebook.presto.iceberg.util.IcebergPrestoModelConverters.toIcebergTableIdentifier; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; @@ -160,6 +165,7 @@ import static com.google.common.collect.Streams.stream; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; +import static java.lang.Double.doubleToLongBits; import static java.lang.Double.doubleToRawLongBits; import static java.lang.Double.longBitsToDouble; import static java.lang.Double.parseDouble; @@ -215,6 +221,8 @@ public final class IcebergUtil { private static final Logger log = Logger.get(IcebergUtil.class); public static final int MIN_FORMAT_VERSION_FOR_DELETE = 2; + public static final int MAX_FORMAT_VERSION_FOR_ROW_LEVEL_OPERATIONS = 2; + public static final int MAX_SUPPORTED_FORMAT_VERSION = 3; public static final long DOUBLE_POSITIVE_ZERO = 0x0000000000000000L; public static final long DOUBLE_POSITIVE_INFINITE = 0x7ff0000000000000L; @@ -285,6 +293,19 @@ private static SchemaTableName getBaseSchemaTableName(SchemaTableName table) return new SchemaTableName(table.getSchemaName(), icebergTableName.getTableName()); } + public static TableOperations opsFromTable(Table table) + { + if (table instanceof BaseTransaction.TransactionTable) { + return ((BaseTransaction.TransactionTable) table).operations(); + } + else if (table instanceof BaseTable) { + return ((BaseTable) table).operations(); + } + else { + throw new PrestoException(NOT_SUPPORTED, "Unsupported Table type: " + table.getClass().getName()); + } + } + public static List getPartitionKeyColumnHandles(IcebergTableHandle tableHandle, Table table, TypeManager typeManager) { Set partitionSpecs = tableHandle.getIcebergTableName().getSnapshotId() @@ -312,6 +333,15 @@ public static Optional resolveSnapshotIdByName(Table table, IcebergTableNa return name.getSnapshotId(); } + if (name.getBranchName().isPresent()) { + String branchName = name.getBranchName().get(); + SnapshotRef branchRef = table.refs().get(branchName); + if (branchRef != null && branchRef.isBranch()) { + return Optional.of(branchRef.snapshotId()); + } + throw new PrestoException(NOT_FOUND, format("Branch '%s' does not exist in table %S", branchName, table)); + } + if (name.getTableType() == IcebergTableType.CHANGELOG) { return Optional.ofNullable(SnapshotUtil.oldestAncestor(table)).map(Snapshot::snapshotId); } @@ -730,6 +760,36 @@ public static Object deserializePartitionValue(Type type, String valueString, St throw new PrestoException(GENERIC_INTERNAL_ERROR, "Invalid partition type " + type.toString()); } + public static Domain createDomainFromIcebergPartitionValue( + Object value, + org.apache.iceberg.types.Type icebergType, + Type prestoType) + { + if (value == null) { + return onlyNull(prestoType); + } + + switch (icebergType.typeId()) { + case INTEGER: + case DATE: + return singleValue(prestoType, ((Integer) value).longValue()); + case LONG: + case BOOLEAN: + return singleValue(prestoType, value); + case TIME: + case TIMESTAMP: + return singleValue(prestoType, MICROSECONDS.toMillis((Long) value)); + case STRING: + return singleValue(prestoType, utf8Slice(value.toString())); + case FLOAT: + return singleValue(prestoType, (long) floatToRawIntBits((Float) value)); + case DOUBLE: + return singleValue(prestoType, doubleToLongBits((Double) value)); + default: + throw new UnsupportedOperationException("Unsupported partition column type: " + icebergType); + } + } + /** * Returns the adjacent value that compares bigger than or less than {@code value} based on parameter {@code isPrevious}. *

@@ -1174,7 +1234,11 @@ public static Map populateTableProperties(IcebergAbstractMetadat public static int parseFormatVersion(String formatVersion) { try { - return parseInt(formatVersion); + int version = parseInt(formatVersion); + if (version > MAX_SUPPORTED_FORMAT_VERSION) { + throw new PrestoException(NOT_SUPPORTED, format("Iceberg table format version %d is not supported", version)); + } + return version; } catch (NumberFormatException | IndexOutOfBoundsException e) { throw new PrestoException(ICEBERG_INVALID_FORMAT_VERSION, "Unable to parse user provided format version"); @@ -1279,6 +1343,47 @@ public static String dataLocation(Table icebergTable) return dataLocation; } + public static void validateNoBranchSpecified(IcebergTableHandle tableHandle, String operation) + { + if (tableHandle.getIcebergTableName().getBranchName().isPresent()) { + throw new PrestoException(NOT_SUPPORTED, format("%s is not supported on branch-specific tables. Branch '%s' was specified in table name '%s'", + operation, + tableHandle.getIcebergTableName().getBranchName().get(), + tableHandle.getIcebergTableName().getTableNameWithType())); + } + } + + public static void validateViewDefinitionForBranches(String viewData, String operation) + { + if (viewData != null && viewData.contains(".branch_")) { + throw new PrestoException(NOT_SUPPORTED, format("%s is not supported with branch-specific table references in the view definition. " + + "The view SQL appears to reference a branch using '.branch_' syntax. " + + "Please use the main table or FOR SYSTEM_VERSION AS OF syntax instead.", operation)); + } + } + + public static void validateNoBranchInBaseTables(List baseTables, String operation) + { + for (SchemaTableName baseTable : baseTables) { + if (baseTable.getTableName().contains(".branch_")) { + throw new PrestoException(NOT_SUPPORTED, format("%s is not supported with branch-specific table references. Table '%s' appears to reference a branch. " + + "Please use the main table or FOR SYSTEM_VERSION AS OF syntax instead.", operation, baseTable)); + } + } + } + + public static void validateBranchExists(IcebergTableHandle tableHandle, Table icebergTable) + { + Optional branchName = tableHandle.getIcebergTableName().getBranchName(); + if (branchName.isPresent()) { + String branch = branchName.get(); + SnapshotRef branchRef = icebergTable.refs().get(branch); + if (branchRef == null || !branchRef.isBranch()) { + throw new PrestoException(NOT_FOUND, format("Branch '%s' does not exist in table %s.%s", branch, tableHandle.getSchemaName(), tableHandle.getIcebergTableName().getTableName())); + } + } + } + public static Long getSplitSize(Table table) { return Long.parseLong(table.properties() diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergViewMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergViewMetadata.java new file mode 100644 index 0000000000000..9990f1ce11513 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergViewMetadata.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +import static com.facebook.presto.iceberg.IcebergAbstractMetadata.PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION; +import static java.util.Objects.requireNonNull; + +public class IcebergViewMetadata +{ + private final Map properties; + private final ConnectorTableMetadata tableMetadata; + + public IcebergViewMetadata(Map properties, ConnectorTableMetadata tableMetadata) + { + this.properties = ImmutableMap.copyOf(requireNonNull(properties, "properties is null")); + this.tableMetadata = requireNonNull(tableMetadata, "tableMetadata is null"); + } + + public Map getProperties() + { + return properties; + } + + public ConnectorTableMetadata getTableMetadata() + { + return tableMetadata; + } + + public boolean isMaterializedView() + { + return properties.containsKey(PRESTO_MATERIALIZED_VIEW_FORMAT_VERSION); + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/InternalIcebergConnectorFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/InternalIcebergConnectorFactory.java index 325e529c1b01b..abc4e6d32a4e9 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/InternalIcebergConnectorFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/InternalIcebergConnectorFactory.java @@ -19,10 +19,10 @@ import com.facebook.airlift.json.JsonModule; import com.facebook.presto.cache.CachingModule; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.hive.HiveCommonModule; import com.facebook.presto.hive.HiveCommonSessionProperties; import com.facebook.presto.hive.NodeVersion; -import com.facebook.presto.hive.RebindSafeMBeanServer; import com.facebook.presto.hive.SchemaProperties; import com.facebook.presto.hive.authentication.HiveAuthenticationModule; import com.facebook.presto.hive.gcs.HiveGcsModule; @@ -30,6 +30,7 @@ import com.facebook.presto.hive.s3.HiveS3Module; import com.facebook.presto.hive.security.SystemTableAwareAccessControl; import com.facebook.presto.iceberg.security.IcebergSecurityModule; +import com.facebook.presto.iceberg.transaction.IcebergTransactionManager; import com.facebook.presto.spi.ConnectorSystemConfig; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java index a379a0492675b..92fd0ce08475b 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/TableStatisticsMaker.java @@ -524,6 +524,9 @@ private void updatePartitionedStats( Map newStats, Predicate predicate) { + if (!summary.hasValidColumnMetrics()) { + return; + } for (PartitionField field : partitionFields) { int id = field.sourceId(); if (summary.getCorruptedStats().contains(id)) { diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java index f06a2fba1542e..162675bfa1e75 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/hive/IcebergHiveFileMetastoreModule.java @@ -16,6 +16,7 @@ import com.facebook.presto.hive.ForCachingHiveMetastore; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.hive.metastore.InMemoryCachingHiveMetastore; +import com.facebook.presto.hive.metastore.MetastoreCacheSpecProvider; import com.facebook.presto.hive.metastore.file.FileHiveMetastoreConfig; import com.google.inject.Binder; import com.google.inject.Module; @@ -40,6 +41,7 @@ public IcebergHiveFileMetastoreModule(String connectorId) public void configure(Binder binder) { configBinder(binder).bindConfig(FileHiveMetastoreConfig.class); + binder.bind(MetastoreCacheSpecProvider.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(IcebergFileHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).to(InMemoryCachingHiveMetastore.class).in(Scopes.SINGLETON); newExporter(binder).export(ExtendedHiveMetastore.class) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java index ebef2d465de58..b4fa9a0bf59bb 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergEqualityDeleteAsJoin.java @@ -27,8 +27,8 @@ import com.facebook.presto.iceberg.IcebergTableLayoutHandle; import com.facebook.presto.iceberg.IcebergTableName; import com.facebook.presto.iceberg.IcebergTableType; -import com.facebook.presto.iceberg.IcebergTransactionManager; import com.facebook.presto.iceberg.IcebergUtil; +import com.facebook.presto.iceberg.transaction.IcebergTransactionManager; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPlanOptimizer; import com.facebook.presto.spi.ConnectorPlanRewriter; @@ -351,6 +351,7 @@ private TableScanNode createDeletesTableScan(ImmutableMap partitionSpecIds.contains(partitionSpec.specId())) .allMatch(spec -> canEnforceConstraintWithinPartitioningSpec(spec, columnHandle, domain, session)); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java index 8c80cbdd97163..796ea4f3af867 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergPlanOptimizerProvider.java @@ -15,7 +15,7 @@ import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.iceberg.IcebergTableProperties; -import com.facebook.presto.iceberg.IcebergTransactionManager; +import com.facebook.presto.iceberg.transaction.IcebergTransactionManager; import com.facebook.presto.spi.ConnectorPlanOptimizer; import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; import com.facebook.presto.spi.function.FunctionMetadataManager; diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/ExpireSnapshotsProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/ExpireSnapshotsProcedure.java index aea6060a571f6..58bf392d01c0e 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/ExpireSnapshotsProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/ExpireSnapshotsProcedure.java @@ -106,7 +106,7 @@ private void doExpireSnapshots(ConnectorSession clientSession, String schema, St expireSnapshots = expireSnapshots.retainLast(retainLast); } - expireSnapshots.cleanExpiredFiles(true) - .commit(); + expireSnapshots.cleanExpiredFiles(true).commit(); + metadata.commit(); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/FastForwardBranchProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/FastForwardBranchProcedure.java index 08b43a84640a5..d1eba998e5c0f 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/FastForwardBranchProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/FastForwardBranchProcedure.java @@ -14,9 +14,9 @@ package com.facebook.presto.iceberg.procedure; import com.facebook.presto.iceberg.IcebergMetadataFactory; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -69,8 +69,9 @@ public Procedure get() public void fastForwardToBranch(ConnectorSession clientSession, String schemaName, String tableName, String fromBranch, String targetBranch) { SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); - ConnectorMetadata metadata = metadataFactory.create(); + IcebergTransactionMetadata metadata = metadataFactory.create(); Table icebergTable = getIcebergTable(metadata, clientSession, schemaTableName); icebergTable.manageSnapshots().fastForwardBranch(fromBranch, targetBranch).commit(); + metadata.commit(); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteManifestsProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteManifestsProcedure.java index b82084cc70926..158e3cb6d8dee 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteManifestsProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RewriteManifestsProcedure.java @@ -13,12 +13,12 @@ */ package com.facebook.presto.iceberg.procedure; +import com.facebook.presto.iceberg.IcebergAbstractMetadata; import com.facebook.presto.iceberg.IcebergMetadataFactory; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -73,7 +73,7 @@ public void rewriteManifests(ConnectorSession clientSession, String schemaName, { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(getClass().getClassLoader())) { SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); - ConnectorMetadata metadata = metadataFactory.create(); + IcebergAbstractMetadata metadata = (IcebergAbstractMetadata) metadataFactory.create(); Table icebergTable = getIcebergTable(metadata, clientSession, schemaTableName); RewriteManifests rewriteManifests = icebergTable.rewriteManifests().clusterBy(file -> "file"); int targetSpecId; @@ -87,6 +87,7 @@ public void rewriteManifests(ConnectorSession clientSession, String schemaName, targetSpecId = icebergTable.spec().specId(); } rewriteManifests.rewriteIf(manifest -> manifest.partitionSpecId() == targetSpecId).commit(); + metadata.commit(); } } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToSnapshotProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToSnapshotProcedure.java index c50c2c8b1e6c9..2462e49b7c336 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToSnapshotProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToSnapshotProcedure.java @@ -14,9 +14,9 @@ package com.facebook.presto.iceberg.procedure; import com.facebook.presto.iceberg.IcebergMetadataFactory; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -67,8 +67,9 @@ public Procedure get() public void rollbackToSnapshot(ConnectorSession clientSession, String schema, String table, Long snapshotId) { SchemaTableName schemaTableName = new SchemaTableName(schema, table); - ConnectorMetadata metadata = metadataFactory.create(); + IcebergTransactionMetadata metadata = metadataFactory.create(); getIcebergTable(metadata, clientSession, schemaTableName) .manageSnapshots().rollbackTo(snapshotId).commit(); + metadata.commit(); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToTimestampProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToTimestampProcedure.java index 513a4f4ae57d7..53c5f1681afd7 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToTimestampProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/RollbackToTimestampProcedure.java @@ -15,10 +15,10 @@ import com.facebook.presto.common.type.SqlTimestamp; import com.facebook.presto.iceberg.IcebergMetadataFactory; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -76,10 +76,11 @@ public void rollbackToTimestamp(ConnectorSession clientSession, String schema, S private void doRollbackToTimestamp(ConnectorSession clientSession, String schema, String tableName, SqlTimestamp timestamp) { SchemaTableName schemaTableName = new SchemaTableName(schema, tableName); - ConnectorMetadata metadata = metadataFactory.create(); + IcebergTransactionMetadata metadata = metadataFactory.create(); getIcebergTable(metadata, clientSession, schemaTableName) .manageSnapshots() .rollbackToTime(timestamp.isLegacyTimestamp() ? timestamp.getMillisUtc() : timestamp.getMillis()) .commit(); + metadata.commit(); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetCurrentSnapshotProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetCurrentSnapshotProcedure.java index 8fdfa2310fac9..6413273f0599c 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetCurrentSnapshotProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetCurrentSnapshotProcedure.java @@ -14,9 +14,9 @@ package com.facebook.presto.iceberg.procedure; import com.facebook.presto.iceberg.IcebergMetadataFactory; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -74,10 +74,11 @@ public void setCurrentSnapshot(ConnectorSession clientSession, String schema, St checkState((snapshotId != null && reference == null) || (snapshotId == null && reference != null), "Either snapshot_id or reference must be provided, not both"); SchemaTableName schemaTableName = new SchemaTableName(schema, table); - ConnectorMetadata metadata = metadataFactory.create(); + IcebergTransactionMetadata metadata = metadataFactory.create(); Table icebergTable = getIcebergTable(metadata, clientSession, schemaTableName); long targetSnapshotId = snapshotId != null ? snapshotId : getSnapshotIdFromReference(icebergTable, reference); icebergTable.manageSnapshots().setCurrentSnapshot(targetSnapshotId).commit(); + metadata.commit(); } private long getSnapshotIdFromReference(Table table, String refName) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetTablePropertyProcedure.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetTablePropertyProcedure.java index d4f5457bc3a80..9491899d221ae 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetTablePropertyProcedure.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/procedure/SetTablePropertyProcedure.java @@ -17,11 +17,11 @@ import com.facebook.presto.iceberg.IcebergTableName; import com.facebook.presto.iceberg.IcebergTableProperties; import com.facebook.presto.iceberg.IcebergUtil; +import com.facebook.presto.iceberg.transaction.IcebergTransactionMetadata; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.google.common.collect.ImmutableList; @@ -93,7 +93,7 @@ public void setTableProperty(ConnectorSession session, String schema, String tab session.getWarningCollector().add(warning); } - ConnectorMetadata metadata = metadataFactory.create(); + IcebergTransactionMetadata metadata = metadataFactory.create(); IcebergTableName tableName = IcebergTableName.from(table); SchemaTableName schemaTableName = new SchemaTableName(schema, tableName.getTableName()); Table icebergTable = IcebergUtil.getIcebergTable(metadata, session, schemaTableName); @@ -101,6 +101,7 @@ public void setTableProperty(ConnectorSession session, String schema, String tab icebergTable.updateProperties() .set(key, value) .commit(); + metadata.commit(); } } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionContext.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionContext.java new file mode 100644 index 0000000000000..d0d3e42d18b33 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionContext.java @@ -0,0 +1,306 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.transaction; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.transaction.IsolationLevel; +import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataTableScan; +import org.apache.iceberg.DeleteFiles; +import org.apache.iceberg.ExpireSnapshots; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManageSnapshots; +import org.apache.iceberg.OverwriteFiles; +import org.apache.iceberg.ReplacePartitions; +import org.apache.iceberg.ReplaceSortOrder; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.Transactions; +import org.apache.iceberg.UpdateLocation; +import org.apache.iceberg.UpdatePartitionSpec; +import org.apache.iceberg.UpdateProperties; +import org.apache.iceberg.UpdateSchema; +import org.apache.iceberg.UpdateStatistics; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_TRANSACTION_CONFLICT_ERROR; +import static com.facebook.presto.iceberg.IcebergUtil.opsFromTable; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Iterators.getOnlyElement; +import static java.util.Objects.requireNonNull; +import static org.apache.iceberg.IcebergLibUtils.getScanContext; + +public class IcebergTransactionContext +{ + private final IsolationLevel isolationLevel; + private final boolean autoCommitContext; + private final Map txByTable; + private final Map initiallyReadTables; + private final AtomicReference callbacksOnCommit = new AtomicReference<>(); + + public IcebergTransactionContext(IsolationLevel isolationLevel, boolean autoCommitContext) + { + this.isolationLevel = requireNonNull(isolationLevel, "isolationLevel is null"); + this.autoCommitContext = autoCommitContext; + txByTable = new ConcurrentHashMap<>(); + initiallyReadTables = new ConcurrentHashMap<>(); + } + + public IsolationLevel getIsolationLevel() + { + return this.isolationLevel; + } + + public boolean isAutoCommitContext() + { + return this.autoCommitContext; + } + + public Optional

getTransactionTable(SchemaTableName tableName) + { + if (txByTable.containsKey(tableName)) { + return Optional.ofNullable(txByTable.get(tableName).table()); + } + + return Optional.empty(); + } + + public Optional getTransaction(SchemaTableName tableName) + { + if (txByTable.containsKey(tableName)) { + return Optional.ofNullable(txByTable.get(tableName)); + } + + return Optional.empty(); + } + + public Optional
initiallyReadTable(SchemaTableName tableName) + { + if (initiallyReadTables.containsKey(tableName)) { + return Optional.ofNullable(initiallyReadTables.get(tableName)); + } + + return Optional.empty(); + } + + public void registerTransaction(SchemaTableName tableName, Transaction transaction) + { + if (txByTable.isEmpty()) { + txByTable.put(tableName, transaction); + } + else if (!txByTable.containsKey(tableName)) { + throw new PrestoException(ICEBERG_TRANSACTION_CONFLICT_ERROR, "Not allowed to open write transactions on different tables"); + } + } + + public Table getIcebergTable(SchemaTableName schemaTableName, Function rawIcebergTableLoader) + { + Table table = getTransactionTable(schemaTableName) + .orElseGet(() -> initiallyReadTable(schemaTableName) + .orElseGet(() -> { + Table loadTable = rawIcebergTableLoader.apply(schemaTableName); + initiallyReadTables.computeIfAbsent(schemaTableName, ignored -> loadTable); + return loadTable; + })); + return new TransactionalTable(schemaTableName, table, opsFromTable(table)); + } + + public void registerCallback(Runnable callback) + { + checkArgument(this.callbacksOnCommit.get() == null, "Cannot set callbacksOnCommit multiple times"); + this.callbacksOnCommit.set(callback); + } + + public void commit() + { + if (!txByTable.isEmpty()) { + getOnlyElement(txByTable.values().iterator()).commitTransaction(); + if (callbacksOnCommit.get() != null) { + callbacksOnCommit.get().run(); + } + txByTable.clear(); + } + initiallyReadTables.clear(); + callbacksOnCommit.set(null); + } + + public void rollback() + { + txByTable.clear(); + initiallyReadTables.clear(); + } + + /** + * We're using a {@link Transaction} per table so that we can keep track of pending changes for a + * particular table. + */ + private Transaction txForTable(SchemaTableName tableName, Table table) + { + if (!txByTable.isEmpty() && !txByTable.containsKey(tableName)) { + throw new PrestoException(ICEBERG_TRANSACTION_CONFLICT_ERROR, "Not allowed to open write transactions on multiple tables"); + } + + return txByTable.computeIfAbsent( + tableName, + k -> Transactions.newTransaction(table.name(), ((HasTableOperations) table).operations())); + } + + private class TransactionalTable + extends BaseTable + { + private final SchemaTableName tableName; + private final Table table; + + private TransactionalTable(SchemaTableName tableName, Table table, TableOperations ops) + { + super(ops, table.name()); + this.tableName = tableName; + this.table = table; + } + + @Override + public TableScan newScan() + { + TableScan tableScan = super.newScan(); + if (tableScan instanceof DataTableScan) { + return new TransactionalTableScan((DataTableScan) tableScan); + } + + return tableScan; + } + + @Override + public UpdateSchema updateSchema() + { + return txForTable(tableName, table).updateSchema(); + } + + @Override + public UpdatePartitionSpec updateSpec() + { + return txForTable(tableName, table).updateSpec(); + } + + @Override + public UpdateProperties updateProperties() + { + return txForTable(tableName, table).updateProperties(); + } + + @Override + public ReplaceSortOrder replaceSortOrder() + { + return txForTable(tableName, table).replaceSortOrder(); + } + + @Override + public UpdateLocation updateLocation() + { + return txForTable(tableName, table).updateLocation(); + } + + @Override + public AppendFiles newAppend() + { + return txForTable(tableName, table).newAppend(); + } + + @Override + public AppendFiles newFastAppend() + { + return txForTable(tableName, table).newFastAppend(); + } + + @Override + public RewriteFiles newRewrite() + { + return txForTable(tableName, table).newRewrite(); + } + + @Override + public RewriteManifests rewriteManifests() + { + return txForTable(tableName, table).rewriteManifests(); + } + + @Override + public OverwriteFiles newOverwrite() + { + return txForTable(tableName, table).newOverwrite(); + } + + @Override + public RowDelta newRowDelta() + { + return txForTable(tableName, table).newRowDelta(); + } + + @Override + public ReplacePartitions newReplacePartitions() + { + return txForTable(tableName, table).newReplacePartitions(); + } + + @Override + public DeleteFiles newDelete() + { + return txForTable(tableName, table).newDelete(); + } + + @Override + public UpdateStatistics updateStatistics() + { + return txForTable(tableName, table).updateStatistics(); + } + + @Override + public ExpireSnapshots expireSnapshots() + { + return txForTable(tableName, table).expireSnapshots(); + } + + @Override + public ManageSnapshots manageSnapshots() + { + return txForTable(tableName, table).manageSnapshots(); + } + + @Override + public Transaction newTransaction() + { + return txForTable(tableName, table); + } + } + + private static class TransactionalTableScan + extends DataTableScan + { + protected TransactionalTableScan(DataTableScan delegate) + { + super(delegate.table(), delegate.schema(), getScanContext(delegate)); + } + } +} diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTransactionManager.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionManager.java similarity index 67% rename from presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTransactionManager.java rename to presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionManager.java index ec0fb91930a22..617e1f56ebe01 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTransactionManager.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionManager.java @@ -11,9 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.iceberg; +package com.facebook.presto.iceberg.transaction; -import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import java.util.Map; @@ -24,24 +23,24 @@ public class IcebergTransactionManager { - private final Map transactions = new ConcurrentHashMap<>(); + private final Map transactions = new ConcurrentHashMap<>(); - public ConnectorMetadata get(ConnectorTransactionHandle transaction) + public IcebergTransactionMetadata get(ConnectorTransactionHandle transaction) { - ConnectorMetadata metadata = transactions.get(transaction); + IcebergTransactionMetadata metadata = transactions.get(transaction); checkArgument(metadata != null, "no such transaction: %s", transaction); return metadata; } public void remove(ConnectorTransactionHandle transaction) { - ConnectorMetadata metadata = transactions.remove(transaction); + IcebergTransactionMetadata metadata = transactions.remove(transaction); checkArgument(metadata != null, "no such transaction: %s", transaction); } - public void put(ConnectorTransactionHandle transaction, ConnectorMetadata metadata) + public void put(ConnectorTransactionHandle transaction, IcebergTransactionMetadata metadata) { - ConnectorMetadata existing = transactions.putIfAbsent(transaction, metadata); + IcebergTransactionMetadata existing = transactions.putIfAbsent(transaction, metadata); checkState(existing == null, "transaction already exists: %s", existing); } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionMetadata.java new file mode 100644 index 0000000000000..642ec71317709 --- /dev/null +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/transaction/IcebergTransactionMetadata.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.transaction; + +import com.facebook.presto.spi.connector.ConnectorCommitHandle; +import com.facebook.presto.spi.connector.ConnectorMetadata; + +public interface IcebergTransactionMetadata + extends ConnectorMetadata +{ + ConnectorCommitHandle commit(); + + void rollback(); +} diff --git a/presto-iceberg/src/main/java/org/apache/iceberg/IcebergLibUtils.java b/presto-iceberg/src/main/java/org/apache/iceberg/IcebergLibUtils.java index ba3543661dc3e..56b5da1677a77 100644 --- a/presto-iceberg/src/main/java/org/apache/iceberg/IcebergLibUtils.java +++ b/presto-iceberg/src/main/java/org/apache/iceberg/IcebergLibUtils.java @@ -31,4 +31,9 @@ public static ExpireSnapshots withIncrementalCleanup(ExpireSnapshots expireSnaps checkArgument(expireSnapshots instanceof RemoveSnapshots, "expireSnapshots is not an instance of RemoveSnapshots"); return ((RemoveSnapshots) expireSnapshots).withIncrementalCleanup(incrementalCleanup); } + + public static TableScanContext getScanContext(DataTableScan tableScan) + { + return tableScan.context(); + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java index a9723243deeec..c497520e70d2e 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedSmokeTestBase.java @@ -1111,6 +1111,54 @@ protected void unregisterTable(String schemaName, String newTableName) assertUpdate("CALL system.unregister_table('" + schemaName + "', '" + newTableName + "')"); } + @Test + public void testCTASWithLargeAmountOfPartitions() + { + String tableName = "test_ctas_with_many_partitions"; + try { + assertQueryFails(format("CREATE TABLE %s WITH(PARTITIONING = ARRAY['bucket(orderkey, 1000)']) AS SELECT * FROM tpch.tiny.lineitem", tableName), + "Exceeded limit of 100 open writers for partitions"); + + Session sessionWithSpecifiedPartitionsPerWriter = Session.builder(getSession()) + .setCatalogSessionProperty("iceberg", "parquet_writer_block_size", "100kB") + .setCatalogSessionProperty("iceberg", "parquet_writer_page_size", "10kB") + .setCatalogSessionProperty("iceberg", "max_partitions_per_writer", "1000") + .build(); + assertUpdate(sessionWithSpecifiedPartitionsPerWriter, + format("CREATE TABLE %s WITH(PARTITIONING = ARRAY['bucket(orderkey, 1000)']) AS SELECT * FROM tpch.tiny.lineitem", tableName), 60175); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 60175"); + assertQuery(format("SELECT count(*) FROM \"%s$partitions\"", tableName), "SELECT 1000"); + } + finally { + dropTable(getSession(), tableName); + } + } + + @Test + public void testInsertIntoTableWithLargeAmountOfPartitions() + { + String tableName = "test_insert_with_many_partitions"; + try { + assertUpdate(format("CREATE TABLE %s WITH(PARTITIONING = ARRAY['partkey']) AS SELECT * FROM tpch.tiny.lineitem WITH NO DATA", tableName), 0); + + assertQueryFails(format("INSERT INTO %s SELECT * FROM tpch.tiny.lineitem", tableName), + "Exceeded limit of 100 open writers for partitions"); + + Session sessionWithSpecifiedPartitionsPerWriter = Session.builder(getSession()) + .setCatalogSessionProperty("iceberg", "parquet_writer_block_size", "100kB") + .setCatalogSessionProperty("iceberg", "parquet_writer_page_size", "10kB") + .setCatalogSessionProperty("iceberg", "max_partitions_per_writer", "2000") + .build(); + assertUpdate(sessionWithSpecifiedPartitionsPerWriter, + format("INSERT INTO %s SELECT * FROM tpch.tiny.lineitem", tableName), 60175); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 60175"); + assertQuery(format("SELECT count(*) from \"%s$partitions\"", tableName), "SELECT 2000"); + } + finally { + dropTable(getSession(), tableName); + } + } + @DataProvider public Object[][] compressionCodecTestData() { @@ -2496,4 +2544,27 @@ public void testIOExplainWithTimestampWithTimeZone() assertUpdate("DROP TABLE test_tstz_io"); } } + + @Test + public void testPushdownSubfieldsWithDml() + { + QueryRunner queryRunner = getQueryRunner(); + Session session = Session.builder(getSession()) + .setSystemProperty("pushdown_subfields_enabled", "true") + .build(); + try { + queryRunner.execute("CREATE TABLE test_pushdown_subfields_dml(a INTEGER, b VARCHAR, c VARCHAR)"); + queryRunner.execute("INSERT INTO test_pushdown_subfields_dml VALUES (1, 'x', 'p'), (2, 'y', 'q'), (3, 'z', 'r')"); + + assertUpdate(session, "UPDATE test_pushdown_subfields_dml SET a = 10 WHERE c = 'q'", 1); + assertQuery("SELECT a, b, c FROM test_pushdown_subfields_dml WHERE c = 'q'", "VALUES (10, 'y', 'q')"); + + assertUpdate(session, "UPDATE test_pushdown_subfields_dml SET a = 20 WHERE c = 'nonexistent'", 0); + assertUpdate("DELETE FROM test_pushdown_subfields_dml WHERE c = 'r'", 1); + assertQuery("SELECT a, b, c FROM test_pushdown_subfields_dml ORDER BY a", "VALUES (1, 'x', 'p'), (10, 'y', 'q')"); + } + finally { + queryRunner.execute("DROP TABLE IF EXISTS test_pushdown_subfields_dml"); + } + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java index 88944686b961f..20703ba2a7b55 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java @@ -829,13 +829,21 @@ public void testPartitionedByTimeType() assertQuerySucceeds("drop table test_partition_columns_time"); } - @Test - public void testPartitionedByVarbinaryType() + @DataProvider(name = "insertValues") + public Object[][] getInsertValues() + { + return new Object[][] { + {"(1, X'bcd1'), (2, X'e3bcd1')"}, + {"(2, X'e3bcd1'), (1, X'bcd1')"}}; + } + + @Test(dataProvider = "insertValues") + public void testPartitionedByVarbinaryType(String insertValues) { // create iceberg table partitioned by column of VarbinaryType, and insert some data assertQuerySucceeds("drop table if exists test_partition_columns_varbinary"); assertQuerySucceeds("create table test_partition_columns_varbinary(a bigint, b varbinary) with (partitioning = ARRAY['b'])"); - assertQuerySucceeds("insert into test_partition_columns_varbinary values(1, X'bcd1'), (2, X'e3bcd1')"); + assertQuerySucceeds("insert into test_partition_columns_varbinary values " + insertValues); // validate return data of VarbinaryType List varbinaryColumnDatas = getQueryRunner().execute("select b from test_partition_columns_varbinary order by a asc").getOnlyColumn().collect(Collectors.toList()); @@ -861,7 +869,7 @@ public void testPartitionedByVarbinaryType() assertEquals(varbinaryColumnDatas.get(0), new byte[] {(byte) 0xe3, (byte) 0xbc, (byte) 0xd1}); assertEquals(getQueryRunner().execute("select b FROM test_partition_columns_varbinary where b = X'e3bcd1'").getOnlyValue(), new byte[] {(byte) 0xe3, (byte) 0xbc, (byte) 0xd1}); - assertEquals(getQueryRunner().execute("select count(*) from \"test_partition_columns_varbinary$partitions\"").getOnlyValue(), 1L); + assertEquals(getQueryRunner().execute("select count(*) from \"test_partition_columns_varbinary$partitions\"").getOnlyValue(), 2L); assertEquals(getQueryRunner().execute("select row_count from \"test_partition_columns_varbinary$partitions\" where b = X'e3bcd1'").getOnlyValue(), 1L); assertQuerySucceeds("drop table test_partition_columns_varbinary"); @@ -2424,6 +2432,30 @@ public void testRefsTable() assertQuery("SELECT * FROM test_table_references FOR SYSTEM_VERSION AS OF 'testTag' where id1=1", "VALUES(1, NULL)"); } + @Test + public void testQueryBranch() + { + assertUpdate("CREATE TABLE test_branch_dot_notation (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_branch_dot_notation VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + Table icebergTable = loadTable("test_branch_dot_notation"); + icebergTable.manageSnapshots().createBranch("audit_branch").commit(); + assertUpdate("INSERT INTO test_branch_dot_notation VALUES (3, 'Charlie', 300), (4, 'David', 400)", 2); + // Test querying branch using FOR SYSTEM_VERSION AS OF syntax + assertQuery("SELECT count(*) FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'audit_branch'", "VALUES 2"); + assertQuery("SELECT count(*) FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'main'", "VALUES 4"); + // Test querying branch using dot notation syntax + assertQuery("SELECT count(*) FROM \"test_branch_dot_notation.branch_audit_branch\"", "VALUES 2"); + assertQuery("SELECT id, name, value FROM \"test_branch_dot_notation.branch_audit_branch\" ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + // Verify both syntaxes return the same results by comparing actual results + MaterializedResult resultWithForSyntax = computeActual("SELECT id FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'audit_branch' ORDER BY id"); + MaterializedResult resultWithDotNotation = computeActual("SELECT id FROM \"test_branch_dot_notation.branch_audit_branch\" ORDER BY id"); + assertEquals(resultWithForSyntax, resultWithDotNotation); + // Test that main table has all records + assertQuery("SELECT count(*) FROM test_branch_dot_notation", "VALUES 4"); + assertQuerySucceeds("DROP TABLE test_branch_dot_notation"); + } + @Test public void testMetadataLogTable() { @@ -4175,7 +4207,7 @@ private void writePositionDeleteToNationTable(Table icebergTable, String dataFil FileSystem fs = getHdfsEnvironment().getFileSystem(new HdfsContext(SESSION), metadataDir); Path path = new Path(metadataDir, deleteFileName); PositionDeleteWriter writer = Parquet.writeDeletes(HadoopOutputFile.fromPath(path, fs)) - .createWriterFunc(GenericParquetWriter::buildWriter) + .createWriterFunc(GenericParquetWriter::create) .forTable(icebergTable) .overwrite() .rowSchema(icebergTable.schema()) @@ -4209,7 +4241,7 @@ private void writeEqualityDeleteToNationTable(Table icebergTable, Map 100", 1); + assertQuery(session, "SELECT value FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'complex_branch' WHERE id = 2", "VALUES 400"); + assertQuery(session, "SELECT value FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'complex_branch' WHERE id = 1", "VALUES 100"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'complex_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testDeleteFromBranchWithComplexWhere() + { + String tableName = "test_delete_branch_complex"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'delete_complex_branch'"); + // Delete with complex WHERE clause + assertUpdate(session, "DELETE FROM \"" + tableName + ".branch_delete_complex_branch\" WHERE value >= 200", 1); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'delete_complex_branch'", "VALUES 1"); + assertQuery(session, "SELECT id FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'delete_complex_branch'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'delete_complex_branch'"); + } + finally { + dropTable(tableName); + } + } + @Test + public void testMetadataDeleteFromBranch() + { + String tableName = "test_metadata_delete_branch"; + assertUpdate(session, "CREATE TABLE IF NOT EXISTS " + tableName + " (id BIGINT, name VARCHAR, value INTEGER, partition_key VARCHAR) " + + "WITH (format = 'PARQUET', partitioning = ARRAY['partition_key'])"); + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (1, 'Alice', 100, 'p1'), (2, 'Bob', 200, 'p1'), (3, 'Charlie', 300, 'p2')", 3); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'metadata_delete_branch'"); + assertUpdate(session, "INSERT INTO \"" + tableName + ".branch_metadata_delete_branch\" VALUES (4, 'David', 400, 'p2')", 1); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'metadata_delete_branch'", "VALUES 4"); + // Delete entire partition from branch (should trigger metadata delete) + assertUpdate(session, "DELETE FROM \"" + tableName + ".branch_metadata_delete_branch\" WHERE partition_key = 'p2'", 2); + // Verify branch has only p1 partition data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'metadata_delete_branch'", "VALUES 2"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'metadata_delete_branch' WHERE partition_key = 'p2'", "VALUES 0"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 3"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " WHERE partition_key = 'p2'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'metadata_delete_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testBranchIsolation() + { + String tableName = "test_branch_isolation"; + createTable(tableName); + try { + // Create two branches + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'branch_a'"); + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'branch_b'"); + // Insert different data into each branch + assertUpdate(session, "INSERT INTO \"" + tableName + ".branch_branch_a\" VALUES (10, 'Branch A', 1000)", 1); + assertUpdate(session, "INSERT INTO \"" + tableName + ".branch_branch_b\" VALUES (20, 'Branch B', 2000)", 1); + // Verify isolation - branch_a should not see branch_b's data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'branch_a' WHERE id = 20", "VALUES 0"); + // Verify isolation - branch_b should not see branch_a's data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'branch_b' WHERE id = 10", "VALUES 0"); + // Verify each branch has its own data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'branch_a' WHERE id = 10", "VALUES 1"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'branch_b' WHERE id = 20", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'branch_a'"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'branch_b'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testInsertIntoBranchFromSelect() + { + String tableName = "test_insert_branch_select"; + String sourceTable = "test_insert_branch_source"; + createTable(tableName); + createTable(sourceTable); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'select_branch'"); + assertUpdate(session, "INSERT INTO \"" + tableName + ".branch_select_branch\" SELECT * FROM " + sourceTable, 2); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'select_branch'", "VALUES 4"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'select_branch'"); + } + finally { + dropTable(tableName); + dropTable(sourceTable); + } + } + + @Test + public void testMergeIntoBranch() + { + String targetTable = "test_merge_target_branch"; + String sourceTable = "test_merge_source_branch"; + + assertUpdate(session, "CREATE TABLE IF NOT EXISTS " + targetTable + " (id BIGINT, name VARCHAR, value INTEGER) " + + "WITH (format = 'PARQUET', \"format-version\" = '2', \"write.update.mode\" = 'merge-on-read')"); + assertUpdate(session, "INSERT INTO " + targetTable + " VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + + assertUpdate(session, "CREATE TABLE IF NOT EXISTS " + sourceTable + " (id BIGINT, name VARCHAR, value INTEGER) WITH (format = 'PARQUET')"); + assertUpdate(session, "INSERT INTO " + sourceTable + " VALUES (2, 'Bob_Updated', 250), (3, 'Charlie', 300)", 2); + try { + assertUpdate(session, "ALTER TABLE " + targetTable + " CREATE BRANCH 'merge_branch'"); + assertUpdate(session, "MERGE INTO \"" + targetTable + ".branch_merge_branch\" t USING " + sourceTable + " s ON t.id = s.id " + + "WHEN MATCHED THEN UPDATE SET name = s.name, value = s.value WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)", 2); + assertQuery(session, "SELECT count(*) FROM " + targetTable + " FOR SYSTEM_VERSION AS OF 'merge_branch'", "VALUES 3"); + assertQuery(session, "SELECT id, name, value FROM " + targetTable + " FOR SYSTEM_VERSION AS OF 'merge_branch' WHERE id = 2", "VALUES (2, 'Bob_Updated', 250)"); + assertQuery(session, "SELECT id, name, value FROM " + targetTable + " FOR SYSTEM_VERSION AS OF 'merge_branch' WHERE id = 3", "VALUES (3, 'Charlie', 300)"); + // Verify main table is unchanged + assertQuery(session, "SELECT count(*) FROM " + targetTable, "VALUES 2"); + assertQuery(session, "SELECT value FROM " + targetTable + " WHERE id = 2", "VALUES 200"); + assertQuery(session, "SELECT count(*) FROM " + targetTable + " WHERE id = 3", "VALUES 0"); + assertUpdate(session, "ALTER TABLE " + targetTable + " DROP BRANCH 'merge_branch'"); + } + finally { + dropTable(targetTable); + dropTable(sourceTable); + } + } + + @Test + public void testTruncateTableWithBranch() + { + String tableName = "test_truncate_branch"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertUpdate(session, "INSERT INTO \"" + tableName + ".branch_test_branch\" VALUES (3, 'Charlie', 300)", 1); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'test_branch'", "VALUES 3"); + assertUpdate(session, "TRUNCATE TABLE \"" + tableName + ".branch_test_branch\""); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'test_branch'", "VALUES 0"); + assertQuery(session, "SELECT count(*) FROM " + tableName, "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testAddColumnWithBranch() + { + String tableName = "test_add_column_branch_fail"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertQueryFails(session, "ALTER TABLE \"" + tableName + ".branch_test_branch\" ADD COLUMN new_col VARCHAR", ".*ADD COLUMN is not supported on branch-specific tables.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testDropColumnWithBranch() + { + String tableName = "test_drop_column_branch_fail"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertQueryFails(session, "ALTER TABLE \"" + tableName + ".branch_test_branch\" DROP COLUMN value", ".*DROP COLUMN is not supported on branch-specific tables.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testRenameColumnWithBranch() + { + String tableName = "test_rename_column_branch_fail"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertQueryFails(session, "ALTER TABLE \"" + tableName + ".branch_test_branch\" RENAME COLUMN value TO new_value", ".*RENAME COLUMN is not supported on branch-specific tables.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testSetTablePropertiesWithBranch() + { + String tableName = "test_set_properties_branch_fail"; + createTable(tableName); + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertQueryFails(session, "ALTER TABLE \"" + tableName + ".branch_test_branch\" SET PROPERTIES (\"commit.retry.num-retries\" = 6)", ".*SET TABLE PROPERTIES is not supported on branch-specific tables.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java index bf25891c38d57..9712dddc9bf30 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConfig.java @@ -74,7 +74,8 @@ public void testDefaults() .setManifestCacheMaxChunkSize(succinctDataSize(2, MEGABYTE)) .setMaxStatisticsFileCacheSize(succinctDataSize(256, MEGABYTE)) .setStatisticsKllSketchKParameter(1024) - .setMaterializedViewStoragePrefix("__mv_storage__")); + .setMaterializedViewStoragePrefix("__mv_storage__") + .setMaterializedViewMaxChangedPartitions(100)); } @Test @@ -111,6 +112,7 @@ public void testExplicitPropertyMappings() .put("iceberg.max-statistics-file-cache-size", "512MB") .put("iceberg.statistics-kll-sketch-k-parameter", "4096") .put("iceberg.materialized-view-storage-prefix", "custom_mv_prefix") + .put("iceberg.materialized-view-max-changed-partitions", "2000") .build(); IcebergConfig expected = new IcebergConfig() @@ -143,7 +145,8 @@ public void testExplicitPropertyMappings() .setMetricsMaxInferredColumn(16) .setMaxStatisticsFileCacheSize(succinctDataSize(512, MEGABYTE)) .setStatisticsKllSketchKParameter(4096) - .setMaterializedViewStoragePrefix("custom_mv_prefix"); + .setMaterializedViewStoragePrefix("custom_mv_prefix") + .setMaterializedViewMaxChangedPartitions(2000); assertFullMapping(properties, expected); } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConnectorFactory.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConnectorFactory.java index c9572b3499e34..e043b8279cd4a 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConnectorFactory.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergConnectorFactory.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.iceberg; +import com.facebook.presto.hive.metastore.AbstractCachingHiveMetastore.MetastoreCacheScope; import com.facebook.presto.spi.connector.ConnectorFactory; import com.facebook.presto.testing.TestingConnectorContext; import com.google.common.collect.ImmutableMap; @@ -30,11 +31,53 @@ public void testCachingHiveMetastore() { Map config = ImmutableMap.builder() .put("hive.metastore.uri", "thrift://localhost:9083") - .put("hive.metastore-cache-ttl", "10m") + .put("hive.metastore.cache.ttl.default", "10m") .buildOrThrow(); assertThatThrownBy(() -> createConnector(config)) - .hasMessageContaining("In-memory hive metastore caching must not be enabled for Iceberg"); + .hasMessageContaining("In-memory hive metastore caching for tables must not be enabled for Iceberg"); + } + + @Test + public void testMetastoreCachingDisallowedWhenTableCacheEnabledViaEnabledCachesAll() + { + Map config = ImmutableMap.builder() + .put("hive.metastore.uri", "thrift://localhost:9083") + .put("hive.metastore.cache.ttl.default", "10m") + // Enabling all caches implicitly enables table cache + .put("hive.metastore.cache.enabled-caches", "ALL") + .buildOrThrow(); + + assertThatThrownBy(() -> createConnector(config)) + .hasMessageContaining("In-memory hive metastore caching for tables must not be enabled for Iceberg"); + } + + @Test + public void testMetastoreCachingDisallowedWhenTableCacheExplicitlyEnabledViaEnabledCachesTable() + { + Map config = ImmutableMap.builder() + .put("hive.metastore.uri", "thrift://localhost:9083") + .put("hive.metastore.cache.ttl.default", "10m") + // Explicitly enable table cache + .put("hive.metastore.cache.enabled-caches", "TABLE") + .buildOrThrow(); + + assertThatThrownBy(() -> createConnector(config)) + .hasMessageContaining("In-memory hive metastore caching for tables must not be enabled for Iceberg"); + } + + @Test + public void testLegacyMetastoreCacheScopeAllWithNonZeroTtlDisallowed() + { + Map config = ImmutableMap.builder() + .put("hive.metastore.uri", "thrift://localhost:9083") + // Non-zero default TTL combined with ALL scope should be disallowed + .put("hive.metastore.cache.ttl.default", "10m") + .put("hive.metastore.cache.scope", MetastoreCacheScope.ALL.name()) + .buildOrThrow(); + + assertThatThrownBy(() -> createConnector(config)) + .hasMessageContaining("In-memory hive metastore caching for tables must not be enabled for Iceberg"); } private static void createConnector(Map config) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateBranch.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateBranch.java new file mode 100644 index 0000000000000..7f1650b6f322c --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateBranch.java @@ -0,0 +1,337 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import static com.facebook.presto.iceberg.CatalogType.HIVE; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +@Test(singleThreaded = true) +public class TestIcebergCreateBranch + extends AbstractTestQueryFramework +{ + public static final String ICEBERG_CATALOG = "iceberg"; + public static final String TEST_SCHEMA = "test_schema_branch"; + private Session session; + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + session = testSessionBuilder() + .setCatalog(ICEBERG_CATALOG) + .setSchema(TEST_SCHEMA) + .build(); + + return IcebergQueryRunner.builder() + .setCatalogType(HIVE) + .setSchemaName(TEST_SCHEMA) + .setCreateTpchTables(false) + .build().getQueryRunner(); + } + + @BeforeClass + public void setUp() + { + assertUpdate(session, format("CREATE SCHEMA IF NOT EXISTS %s", TEST_SCHEMA)); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + assertUpdate(session, format("DROP SCHEMA IF EXISTS %s", TEST_SCHEMA)); + } + + private void createTable(String tableName) + { + assertUpdate(session, "CREATE TABLE IF NOT EXISTS " + tableName + " (id BIGINT, name VARCHAR) WITH (format = 'PARQUET')"); + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (1, 'Alice'), (2, 'Bob')", 2); + } + + private void dropTable(String tableName) + { + assertQuerySucceeds(session, "DROP TABLE IF EXISTS " + TEST_SCHEMA + "." + tableName); + } + + @Test + public void testCreateBranchBasic() + { + String tableName = "create_branch_basic_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'test_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'test_branch' and type = 'BRANCH'", "VALUES 1"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'test_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'test_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchFromVersion() + { + String tableName = "create_branch_version_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (3, 'Charlie')", 1); + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH 'version_branch' FOR SYSTEM_VERSION AS OF %d", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'version_branch'", "VALUES 3"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'version_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchFromTimestamp() + { + String tableName = "create_branch_ts_table_test"; + createTable(tableName); + + try { + ZonedDateTime committedAt = (ZonedDateTime) computeScalar(session, "SELECT committed_at FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + DateTimeFormatter prestoTimestamp = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS XXX"); + String timestampLiteral = committedAt.format(prestoTimestamp); + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH 'time_branch' FOR SYSTEM_TIME AS OF TIMESTAMP '%s'", tableName, timestampLiteral)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'time_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'time_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchWithRetention() + { + String tableName = "create_branch_retention_table_test"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH 'retention_branch' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'retention_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'retention_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchWithSnapshotRetention() + { + String tableName = "create_branch_snapshot_retention"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH 'full_retention_branch' " + + "FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 2 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'full_retention_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'full_retention_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchDuplicate() + { + String tableName = "create_branch_duplicate_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'duplicate_branch'"); + assertQueryFails(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'duplicate_branch'", ".*Branch.*already exists.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'duplicate_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchWithBothVersionAndTime() + { + String tableName = "create_branch_both_table_test"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + ZonedDateTime committedAt = (ZonedDateTime) computeScalar(session, "SELECT committed_at FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + DateTimeFormatter prestoTimestamp = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS XXX"); + String timestampLiteral = committedAt.format(prestoTimestamp); + assertQueryFails(session, format("ALTER TABLE " + tableName + " CREATE BRANCH 'both_branch' FOR SYSTEM_VERSION AS OF %d FOR SYSTEM_TIME AS OF TIMESTAMP '%s'", + snapshotId, timestampLiteral), ".*mismatched input.*"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchIfNotExists() + { + String tableName = "create_branch_ne_table_test"; + createTable(tableName); + + try { + // Create branch first time - should succeed + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH IF NOT EXISTS 'if_not_exists_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_not_exists_branch' and type = 'BRANCH'", "VALUES 1"); + + // Create same branch again with IF NOT EXISTS - should succeed (no-op) + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH IF NOT EXISTS 'if_not_exists_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_not_exists_branch' and type = 'BRANCH'", "VALUES 1"); + + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'if_not_exists_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'if_not_exists_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceBranch() + { + String tableName = "create_branch_replace_table_test"; + createTable(tableName); + + try { + // Create branch first time + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE BRANCH 'or_replace_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'or_replace_branch' and type = 'BRANCH'", "VALUES 1"); + long firstSnapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$refs\" where name = 'or_replace_branch'"); + // Insert more data + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (4, 'David')", 1); + // Replace branch - should point to new snapshot + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE OR REPLACE BRANCH 'or_replace_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'or_replace_branch' and type = 'BRANCH'", "VALUES 1"); + long secondSnapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$refs\" where name = 'or_replace_branch'"); + // Verify snapshot IDs are different + if (firstSnapshotId == secondSnapshotId) { + throw new AssertionError("Expected different snapshot IDs after OR REPLACE"); + } + // Verify branch now has updated data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'or_replace_branch'", "VALUES 3"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'or_replace_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceBranchNonExistent() + { + String tableName = "create_branch_cr_ne_table_test"; + createTable(tableName); + + try { + // OR REPLACE should work even if branch doesn't exist + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE OR REPLACE BRANCH 'new_or_replace_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'new_or_replace_branch' and type = 'BRANCH'", "VALUES 1"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'new_or_replace_branch'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'new_or_replace_branch'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchWithBothReplaceAndIfNotExists() + { + // Cannot specify both OR REPLACE and IF NOT EXISTS + assertQueryFails(session, "ALTER TABLE test_table_for_branch CREATE OR REPLACE BRANCH IF NOT EXISTS 'invalid_branch'", ".*Cannot specify both OR REPLACE and IF NOT EXISTS.*"); + } + + @Test + public void testCreateBranchIfNotExistsWithRetention() + { + String tableName = "create_branch_ne_retention"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + // Create with retention + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH IF NOT EXISTS 'retention_if_not_exists' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_if_not_exists' and type = 'BRANCH'", "VALUES 1"); + // Try to create again - should be no-op + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH IF NOT EXISTS 'retention_if_not_exists' FOR SYSTEM_VERSION AS OF %d RETAIN 14 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_if_not_exists' and type = 'BRANCH'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'retention_if_not_exists'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceBranchWithRetention() + { + String tableName = "create_branch_cr_with_retention"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + // Create with retention + assertUpdate(session, format("ALTER TABLE %s CREATE BRANCH 'retention_or_replace' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + // Replace with different retention + assertUpdate(session, format("ALTER TABLE %s CREATE OR REPLACE BRANCH 'retention_or_replace' FOR SYSTEM_VERSION AS OF %d RETAIN 14 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_or_replace' and type = 'BRANCH'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'retention_or_replace'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateBranchIfTableExists() + { + String tableName = "create_branch_table_not_exist"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE IF EXISTS " + tableName + " CREATE BRANCH 'if_exists_branch'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_exists_branch' and type = 'BRANCH'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP BRANCH 'if_exists_branch'"); + + assertUpdate(session, "ALTER TABLE IF EXISTS " + tableName + " CREATE BRANCH 'should_not_fail'"); + assertQueryFails(session, "ALTER TABLE non_existent_table CREATE BRANCH 'should_fail'", "No value present"); + } + finally { + dropTable(tableName); + } + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateTag.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateTag.java new file mode 100644 index 0000000000000..79ad7d45d8678 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergCreateTag.java @@ -0,0 +1,319 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import static com.facebook.presto.iceberg.CatalogType.HIVE; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +@Test(singleThreaded = true) +public class TestIcebergCreateTag + extends AbstractTestQueryFramework +{ + public static final String ICEBERG_CATALOG = "iceberg"; + public static final String TEST_SCHEMA = "test_schema_tag"; + private Session session; + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + session = testSessionBuilder() + .setCatalog(ICEBERG_CATALOG) + .setSchema(TEST_SCHEMA) + .build(); + + return IcebergQueryRunner.builder() + .setCatalogType(HIVE) + .setSchemaName(TEST_SCHEMA) + .setCreateTpchTables(false) + .build().getQueryRunner(); + } + + @BeforeClass + public void setUp() + { + assertUpdate(session, format("CREATE SCHEMA IF NOT EXISTS %s", TEST_SCHEMA)); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + assertUpdate(session, format("DROP SCHEMA IF EXISTS %s", TEST_SCHEMA)); + } + + private void createTable(String tableName) + { + assertUpdate(session, "CREATE TABLE IF NOT EXISTS " + tableName + " (id BIGINT, name VARCHAR) WITH (format = 'PARQUET')"); + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (1, 'Alice'), (2, 'Bob')", 2); + } + + private void dropTable(String tableName) + { + assertQuerySucceeds(session, "DROP TABLE IF EXISTS " + TEST_SCHEMA + "." + tableName); + } + + @Test + public void testCreateTagBasic() + { + String tableName = "create_tag_basic_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE TAG 'test_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'test_tag' and type = 'TAG'", "VALUES 1"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'test_tag'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'test_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagFromVersion() + { + String tableName = "create_tag_version_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (3, 'Charlie')", 1); + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + assertUpdate(session, format("ALTER TABLE %s CREATE TAG 'version_tag' FOR SYSTEM_VERSION AS OF %d", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'version_tag'", "VALUES 3"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'version_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagFromTimestamp() + { + String tableName = "create_tag_ts_table_test"; + createTable(tableName); + + try { + ZonedDateTime committedAt = (ZonedDateTime) computeScalar(session, "SELECT committed_at FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + DateTimeFormatter prestoTimestamp = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS XXX"); + String timestampLiteral = committedAt.format(prestoTimestamp); + assertUpdate(session, format("ALTER TABLE %s CREATE TAG 'time_tag' FOR SYSTEM_TIME AS OF TIMESTAMP '%s'", tableName, timestampLiteral)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'time_tag'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'time_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagWithRetention() + { + String tableName = "create_tag_retention_table_test"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + assertUpdate(session, format("ALTER TABLE %s CREATE TAG 'retention_tag' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'retention_tag'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'retention_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagDuplicate() + { + String tableName = "create_tag_duplicate_table_test"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE TAG 'duplicate_tag'"); + assertQueryFails(session, "ALTER TABLE " + tableName + " CREATE TAG 'duplicate_tag'", ".*Tag.*already exists.*"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'duplicate_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagWithBothVersionAndTime() + { + String tableName = "create_tag_both_table_test"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + ZonedDateTime committedAt = (ZonedDateTime) computeScalar(session, "SELECT committed_at FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + DateTimeFormatter prestoTimestamp = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS XXX"); + String timestampLiteral = committedAt.format(prestoTimestamp); + assertQueryFails(session, format("ALTER TABLE " + tableName + " CREATE TAG 'both_tag' FOR SYSTEM_VERSION AS OF %d FOR SYSTEM_TIME AS OF TIMESTAMP '%s'", + snapshotId, timestampLiteral), ".*mismatched input.*"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagIfNotExists() + { + String tableName = "create_tag_ne_table_test"; + createTable(tableName); + + try { + // Create tag first time - should succeed + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE TAG IF NOT EXISTS 'if_not_exists_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_not_exists_tag' and type = 'TAG'", "VALUES 1"); + + // Create same tag again with IF NOT EXISTS - should succeed (no-op) + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE TAG IF NOT EXISTS 'if_not_exists_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_not_exists_tag' and type = 'TAG'", "VALUES 1"); + + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'if_not_exists_tag'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'if_not_exists_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceTag() + { + String tableName = "create_tag_replace_table_test"; + createTable(tableName); + + try { + // Create tag first time + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE TAG 'or_replace_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'or_replace_tag' and type = 'TAG'", "VALUES 1"); + long firstSnapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$refs\" where name = 'or_replace_tag'"); + // Insert more data + assertUpdate(session, "INSERT INTO " + tableName + " VALUES (4, 'David')", 1); + // Replace tag - should point to new snapshot + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE OR REPLACE TAG 'or_replace_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'or_replace_tag' and type = 'TAG'", "VALUES 1"); + long secondSnapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$refs\" where name = 'or_replace_tag'"); + // Verify snapshot IDs are different + if (firstSnapshotId == secondSnapshotId) { + throw new AssertionError("Expected different snapshot IDs after OR REPLACE"); + } + // Verify tag now has updated data + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'or_replace_tag'", "VALUES 3"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'or_replace_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceTagNonExistent() + { + String tableName = "create_tag_cr_ne_table_test"; + createTable(tableName); + + try { + // OR REPLACE should work even if tag doesn't exist + assertUpdate(session, "ALTER TABLE " + tableName + " CREATE OR REPLACE TAG 'new_or_replace_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'new_or_replace_tag' and type = 'TAG'", "VALUES 1"); + assertQuery(session, "SELECT count(*) FROM " + tableName + " FOR SYSTEM_VERSION AS OF 'new_or_replace_tag'", "VALUES 2"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'new_or_replace_tag'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagWithBothReplaceAndIfNotExists() + { + // Cannot specify both OR REPLACE and IF NOT EXISTS + assertQueryFails(session, "ALTER TABLE test_table_for_tag CREATE OR REPLACE TAG IF NOT EXISTS 'invalid_tag'", ".*Cannot specify both OR REPLACE and IF NOT EXISTS.*"); + } + + @Test + public void testCreateTagIfNotExistsWithRetention() + { + String tableName = "create_tag_ne_retention"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + // Create with retention + assertUpdate(session, format("ALTER TABLE %s CREATE TAG IF NOT EXISTS 'retention_if_not_exists' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_if_not_exists' and type = 'TAG'", "VALUES 1"); + // Try to create again - should be no-op + assertUpdate(session, format("ALTER TABLE %s CREATE TAG IF NOT EXISTS 'retention_if_not_exists' FOR SYSTEM_VERSION AS OF %d RETAIN 14 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_if_not_exists' and type = 'TAG'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'retention_if_not_exists'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateOrReplaceTagWithRetention() + { + String tableName = "create_tag_cr_with_retention"; + createTable(tableName); + + try { + long snapshotId = (Long) computeScalar(session, "SELECT snapshot_id FROM \"" + tableName + "$snapshots\" ORDER BY committed_at DESC LIMIT 1"); + // Create with retention + assertUpdate(session, format("ALTER TABLE %s CREATE TAG 'retention_or_replace' FOR SYSTEM_VERSION AS OF %d RETAIN 7 DAYS", tableName, snapshotId)); + // Replace with different retention + assertUpdate(session, format("ALTER TABLE %s CREATE OR REPLACE TAG 'retention_or_replace' FOR SYSTEM_VERSION AS OF %d RETAIN 14 DAYS", tableName, snapshotId)); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'retention_or_replace' and type = 'TAG'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'retention_or_replace'"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateTagIfTableExists() + { + String tableName = "create_tag_table_not_exist"; + createTable(tableName); + + try { + assertUpdate(session, "ALTER TABLE IF EXISTS " + tableName + " CREATE TAG 'if_exists_tag'"); + assertQuery(session, "SELECT count(*) FROM \"" + tableName + "$refs\" where name = 'if_exists_tag' and type = 'TAG'", "VALUES 1"); + assertUpdate(session, "ALTER TABLE " + tableName + " DROP TAG 'if_exists_tag'"); + + assertUpdate(session, "ALTER TABLE IF EXISTS " + tableName + " CREATE TAG 'should_not_fail'"); + assertQueryFails(session, "ALTER TABLE non_existent_table CREATE TAG 'should_fail'", "No value present"); + } + finally { + dropTable(tableName); + } + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergDistributedQueries.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergDistributedQueries.java index 57dd62d142e3a..8368d920da521 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergDistributedQueries.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergDistributedQueries.java @@ -223,4 +223,534 @@ public void testRenameViewIfNotExists() format("line 1:1: View '%s.%s.test_rename_view_not_exist' does not exist", catalog, schema)); assertQuerySucceeds("ALTER VIEW IF EXISTS test_rename_view_not_exist RENAME TO test_renamed_view_not_exist"); } + + @Test + public void testSupportedIsolationLevelForTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableNameForIsolationLevel = "test_supported_isolation_level"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableNameForIsolationLevel)); + + // Does not support serializable isolation in Iceberg connector + Session txnSession = assertStartTransaction(session, "START TRANSACTION ISOLATION LEVEL SERIALIZABLE"); + assertQueryFails(txnSession, format("insert into %s values(1, '1001')", tableNameForIsolationLevel), + "Connector supported isolation level REPEATABLE READ does not meet requested isolation level SERIALIZABLE"); + session = assertEndTransaction(txnSession, "rollback"); + + // Support repeatable_read(snapshot) isolation in Iceberg connector + txnSession = assertStartTransaction(session, "START TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + assertQuery(txnSession, format("select count(*) from %s", tableNameForIsolationLevel), "values(0)"); + assertUpdate(txnSession, format("insert into %s values(1, '1001')", tableNameForIsolationLevel), 1); + assertQuery(txnSession, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001')"); + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001')"); + assertQuery(getSession(), format("select * from %s", tableNameForIsolationLevel), "values(1, '1001')"); + + // Support read committed isolation in Iceberg connector + txnSession = assertStartTransaction(session, "START TRANSACTION ISOLATION LEVEL READ COMMITTED"); + assertQuery(txnSession, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001')"); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableNameForIsolationLevel), 1); + assertUpdate(txnSession, format("insert into %s values(3, '1003')", tableNameForIsolationLevel), 1); + assertQuery(txnSession, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001'), (2, '1002'), (3, '1003')"); + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertQuery(getSession(), format("select * from %s", tableNameForIsolationLevel), "values(1, '1001'), (2, '1002'), (3, '1003')"); + + // Support read uncommitted isolation in Iceberg connector + txnSession = assertStartTransaction(session, "START TRANSACTION ISOLATION LEVEL READ UNCOMMITTED"); + assertQuery(txnSession, format("select * from %s", tableNameForIsolationLevel), "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("delete from %s where a < 2", tableNameForIsolationLevel), 1); + assertUpdate(txnSession, format("update %s set a = a + 10 where b > '1002'", tableNameForIsolationLevel), 1); + assertQuery(txnSession, format("select * from %s", tableNameForIsolationLevel), "values(2, '1002'), (13, '1003')"); + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, format("select * from %s", tableNameForIsolationLevel), "values(2, '1002'), (13, '1003')"); + assertQuery(getSession(), format("select * from %s", tableNameForIsolationLevel), "values(2, '1002'), (13, '1003')"); + + assertUpdate("drop table if exists " + tableNameForIsolationLevel); + } + + @Test + public void testNotAllowCertainDDLStatementInNonAutoCommitTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String catalog = session.getCatalog().get(); + String schema = session.getSchema().get(); + String tableNameForCreate = "test_non_autocommit_table_for_create"; + + // CREATE TABLE + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, format("create table %s(a int, b varchar)", tableNameForCreate), + "CREATE TABLE cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + assertQueryFails(session, "select * from " + tableNameForCreate, + format("Table %s.%s.%s does not exist", catalog, schema, tableNameForCreate)); + + String tableNameForDDL = "test_non_autocommit_table_for_ddl"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableNameForDDL)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002')", tableNameForDDL), 2); + + // DROP TABLE + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "drop table " + tableNameForDDL, + "DROP TABLE cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableNameForDDL, "values(1, '1001'), (2, '1002')"); + + // RENAME TABLE + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "alter table " + tableNameForDDL + " rename to test_rename_to_new_table", + "RENAME TABLE cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableNameForDDL, "values(1, '1001'), (2, '1002')"); + + // CREATE SCHEMA + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "create schema " + catalog + ".test_new_create_schema", + "CREATE SCHEMA cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + + // DROP SCHEMA + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "drop schema " + catalog + "." + schema, + "DROP SCHEMA cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + + // RENAME SCHEMA + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "alter schema " + catalog + "." + schema + " rename to new_schema_name", + "RENAME SCHEMA cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + + assertQuery(session, "select * from " + tableNameForDDL, "values(1, '1001'), (2, '1002')"); + assertUpdate(session, "drop table " + tableNameForDDL); + } + + @Test + public void testNotAllowViewDDLInNonAutoCommitTransaction() + { + skipTestUnless(supportsViews()); + Session session = getQueryRunner().getDefaultSession(); + String catalog = session.getCatalog().get(); + String schema = session.getSchema().get(); + String tableNameForDDL = "test_table_for_create_view"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableNameForDDL)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002')", tableNameForDDL), 2); + String viewNameForDDL = "test_non_autocommit_view_for_ddl"; + + // CREATE VIEW + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, format("create view %s as select * from %s", viewNameForDDL, tableNameForDDL), + "CREATE VIEW cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + assertQueryFails(session, "select * from " + viewNameForDDL, + format("Table %s.%s.%s does not exist", catalog, schema, viewNameForDDL)); + + assertUpdate(session, format("create view %s as select * from %s", viewNameForDDL, tableNameForDDL)); + + // DROP VIEW + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "drop view " + viewNameForDDL, + "DROP VIEW cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + + // RENAME VIEW + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, "alter view " + viewNameForDDL + " rename to test_rename_to_new_view", + "RENAME VIEW cannot be called within a transaction \\(use autocommit mode\\) in Iceberg connector\\."); + session = assertEndTransaction(txnSession, "rollback"); + + assertQuery(session, "select * from " + viewNameForDDL, "values(1, '1001'), (2, '1002')"); + assertUpdate(session, "drop view " + viewNameForDDL); + assertUpdate(session, "drop table " + tableNameForDDL); + } + + @Test + public void testNotAllowCallProceduresInNonAutoCommitTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String catalog = session.getCatalog().get(); + String schema = session.getSchema().get(); + String tableNameForProcedure = "test_non_autocommit_table_for_procedure"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableNameForProcedure)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002')", tableNameForProcedure), 2); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, format("CALL %s.system.remove_orphan_files('%s', '%s', TIMESTAMP '2025-05-31 00:00:00.000')", catalog, schema, tableNameForProcedure), + "Procedures cannot be called within a transaction \\(use autocommit mode\\)"); + session = assertEndTransaction(txnSession, "rollback"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQueryFails(txnSession, format("CALL %s.system.expire_snapshots('%s', '%s', TIMESTAMP '1984-12-08 00:10:00.000')", catalog, schema, tableNameForProcedure), + "Procedures cannot be called within a transaction \\(use autocommit mode\\)"); + session = assertEndTransaction(txnSession, "rollback"); + + assertUpdate(session, "drop table " + tableNameForProcedure); + } + + @Test + public void testNotAllowMultiTableWritesInNonAutocommitTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName1 = "test_non_autocommit_table1"; + String tableName2 = "test_non_autocommit_table2"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName1)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002'), (3, '1003')", tableName1), 3); + assertUpdate(session, format("create table %s(a int, b varchar)", tableName2)); + assertUpdate(session, format("insert into %s values(1, '1001'), (3, '1003'), (5, '1005')", tableName2), 3); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName1, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("insert into %s values(4, '1004')", tableName1), 1); + + assertQuery(txnSession, "select * from " + tableName2, "values(1, '1001'), (3, '1003'), (5, '1005')"); + assertQueryFails(txnSession, format("insert into %s values(2, '1002')", tableName2), + "Not allowed to open write transactions on multiple tables"); + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableName1, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertQuery(session, "select * from " + tableName2, "values(1, '1001'), (3, '1003'), (5, '1005')"); + + assertUpdate("drop table " + tableName1); + assertUpdate("drop table " + tableName2); + } + + @Test + public void testSingleTableMultipleInsertsTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_non_autocommit_table_for_inserts"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertUpdate(txnSession, format("insert into %s values(1, '1001')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(3, '1003'), (4, '1004')", tableName), 2); + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select count(*) from " + tableName, "values(0)"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertUpdate(txnSession, format("insert into %s values(1, '1001')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(3, '1003'), (4, '1004')", tableName), 2); + + // Can read its own writes + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + assertQuery(getSession(), "select count(*) from " + tableName, "values(0)"); + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testSingleTableMixedOperationsTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_non_autocommit_table_for_mix_operations"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002'), (3, '1003')", tableName), 3); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("insert into %s values(4, '1004')", tableName), 1); + + // Can read its own writes + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + assertUpdate(txnSession, format("delete from %s where a > 2", tableName), 2); + assertUpdate(txnSession, format("update %s set a = 1000 + a where b < '1004'", tableName), 2); + assertQuery(txnSession, "select * from " + tableName, "values(1001, '1001'), (1002, '1002')"); + + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("insert into %s values(4, '1004')", tableName), 1); + + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + assertUpdate(txnSession, format("delete from %s where a > 1", tableName), 3); + assertUpdate(txnSession, format("insert into %s values(5, '1005')", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertUpdate(txnSession, format("update %s set a = a + 1000 where b < '1005'", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(3, '1003')", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testInsertBySubqueryInMixedOperationsTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_non_autocommit_table_for_insert_by_subquery_operations"; + assertUpdate(session, format("create table %s as select * from lineitem with no data", tableName), 0); + long totalCount = (long) getQueryRunner().execute(session, "select count(*) from lineitem").getOnlyValue(); + long rowCountWithReturnFlag = (long) getQueryRunner().execute(session, "select count(*) from lineitem where returnflag = 'N'").getOnlyValue(); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select count(*) from " + tableName, "values(0)"); + assertUpdate(txnSession, format("insert into %s select * from lineitem", tableName), totalCount); + + // Can read its own writes + assertQuery(txnSession, "select count(*) from " + tableName, format("values(%s)", totalCount)); + assertUpdate(txnSession, format("delete from %s where returnflag = 'N'", tableName), rowCountWithReturnFlag); + assertQuery(txnSession, "select count(*) from " + tableName, format("values(%s)", totalCount - rowCountWithReturnFlag)); + assertQuery(getSession(), "select count(*) from " + tableName, "values(0)"); + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select count(*) from " + tableName, "values(0)"); + assertQuery(getSession(), "select count(*) from " + tableName, "values(0)"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertUpdate(txnSession, format("insert into %s select * from lineitem", tableName), totalCount); + // Can read its own writes + assertQuery(txnSession, "select count(*) from " + tableName, format("values(%s)", totalCount)); + assertUpdate(txnSession, format("delete from %s where returnflag = 'N'", tableName), rowCountWithReturnFlag); + assertQuery(txnSession, "select count(*) from " + tableName, format("values(%s)", totalCount - rowCountWithReturnFlag)); + assertQuery(getSession(), "select count(*) from " + tableName, "values(0)"); + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select count(*) from " + tableName, format("values(%s)", totalCount - rowCountWithReturnFlag)); + assertQuery(getSession(), "select count(*) from " + tableName, format("values(%s)", totalCount - rowCountWithReturnFlag)); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testMixedOperationsOnSinglePartitionTable() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_non_autocommit_partition_table_for_mix_operations"; + assertUpdate(session, format("create table %s(a int, b varchar) with(partitioning = ARRAY['a'])", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002'), (3, '1003')", tableName), 3); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("insert into %s values(4, '1004')", tableName), 1); + + // Can read its own writes + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + + // This should do metadata delete + assertUpdate(txnSession, format("delete from %s where a > 2", tableName), 2); + assertUpdate(txnSession, format("update %s set a = 1000 + a where b < '1004'", tableName), 2); + assertQuery(txnSession, "select * from " + tableName, "values(1001, '1001'), (1002, '1002')"); + + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession, format("insert into %s values(4, '1004')", tableName), 1); + + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + + // This should do metadata delete + assertUpdate(txnSession, format("delete from %s where a > 1", tableName), 3); + assertUpdate(txnSession, format("insert into %s values(5, '1005')", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + assertUpdate(txnSession, format("update %s set a = a + 1000 where b < '1005'", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(3, '1003')", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (5, '1005')"); + + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1001, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testSingleTableSchemaConflictTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_schema_conflict_for_single_table"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001')", tableName), 1); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertUpdate(txnSession, format("delete from %s", tableName), 1); + assertUpdate(txnSession, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession, format("alter table %s rename column b to b_new", tableName)); + assertUpdate(txnSession, format("insert into %s(b_new) values('1003')", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(null, '1003'), (2, '1002')"); + assertQueryFails(txnSession, format("alter table %s drop column b", tableName), ".* Column 'b' does not exist"); + + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableName, "values(1, '1001')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001')"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testSingleTableSchemaConflictBetweenTransactions() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_schema_conflict_between_transactions"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001')", tableName), 1); + + Session txnSession1 = assertStartTransaction(session, "START TRANSACTION"); + Session txnSession2 = assertStartTransaction(session, "START TRANSACTION"); + assertUpdate(txnSession1, format("delete from %s", tableName), 1); + assertUpdate(txnSession1, format("insert into %s values(2, '1002')", tableName), 1); + assertUpdate(txnSession2, format("alter table %s drop column b", tableName)); + assertUpdate(txnSession2, format("insert into %s values 5", tableName), 1); + assertUpdate(txnSession1, format("alter table %s rename column b to b_new", tableName)); + assertUpdate(txnSession1, format("insert into %s(b_new) values('1003')", tableName), 1); + assertQuery(txnSession1, "select * from " + tableName, "values(null, '1003'), (2, '1002')"); + assertQuery(txnSession2, "select * from " + tableName, "values(1), (5)"); + + // Commit transaction2 which updates the table for dropping column "b" (which causes in-progress transaction1 commit fail) + session = assertEndTransaction(txnSession2, "commit"); + + // Fail to commit transaction1 which includes the action for renaming column "b" + assertQueryFails(txnSession1, "commit", "Table metadata refresh is required"); + + assertQuery(session, "select * from " + tableName, "values(1), (5)"); + assertQuery(getSession(), "select * from " + tableName, "values(1), (5)"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testSingleTableSchemaUpdateVisibilityTransaction() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_schema_update_visibility_for_single_table"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001')", tableName), 1); + + Session txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001')"); + assertUpdate(txnSession, format("alter table %s add column c double", tableName)); + assertUpdate(txnSession, format("insert into %s values(2, '1002', 1.2)", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001', null), (2, '1002', 1.2)"); + assertUpdate(txnSession, format("alter table %s drop column a", tableName)); + assertUpdate(txnSession, format("insert into %s values('1003', 1.3)", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values('1001', null), ('1002', 1.2), ('1003', 1.3)"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001')"); + + session = assertEndTransaction(txnSession, "rollback"); + assertQuery(session, "select * from " + tableName, "values(1, '1001')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001')"); + + txnSession = assertStartTransaction(session, "START TRANSACTION"); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001')"); + assertUpdate(txnSession, format("alter table %s add column c double", tableName)); + assertUpdate(txnSession, format("insert into %s values(2, '1002', 1.2)", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values(1, '1001', null), (2, '1002', 1.2)"); + assertUpdate(txnSession, format("alter table %s drop column a", tableName)); + assertUpdate(txnSession, format("insert into %s values('1003', 1.3)", tableName), 1); + assertQuery(txnSession, "select * from " + tableName, "values('1001', null), ('1002', 1.2), ('1003', 1.3)"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001')"); + + session = assertEndTransaction(txnSession, "commit"); + assertQuery(session, "select * from " + tableName, "values('1001', null), ('1002', 1.2), ('1003', 1.3)"); + assertQuery(getSession(), "select * from " + tableName, "values('1001', null), ('1002', 1.2), ('1003', 1.3)"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testMultipleConcurrentTransactionsIsolation() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_multiple_concurrent_transactions_isolation"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002'), (3, '1003')", tableName), 3); + + Session txnSession1 = assertStartTransaction(session, "START TRANSACTION"); + Session txnSession2 = assertStartTransaction(session, "START TRANSACTION"); + + assertQuery(txnSession1, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertQuery(txnSession2, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + assertUpdate(txnSession1, format("insert into %s values(4, '1004')", tableName), 1); + assertUpdate(txnSession2, format("insert into %s values(5, '1005')", tableName), 1); + + // transaction1 can just read its own writes + assertQuery(txnSession1, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + // transaction2 can just read its own writes + assertQuery(txnSession2, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + // Cannot read any change outside transaction1 and transaction2 + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + Session session1 = assertEndTransaction(txnSession1, "commit"); + // Can read the writes of transaction1 from outside + assertQuery(session1, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004')"); + // transaction2 can still just read its own writes, unaware of outside change + assertQuery(txnSession2, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (5, '1005')"); + + Session session2 = assertEndTransaction(txnSession2, "commit"); + // Can read the writes of transaction1 and transaction2 from outside + assertQuery(session1, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004'), (5, '1005')"); + assertQuery(session2, "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003'), (4, '1004'), (5, '1005')"); + + assertUpdate("drop table " + tableName); + } + + @Test + public void testMultipleConcurrentTransactionsWriteSkew() + { + Session session = getQueryRunner().getDefaultSession(); + String tableName = "test_multiple_concurrent_transactions_skew"; + assertUpdate(session, format("create table %s(a int, b varchar)", tableName)); + assertUpdate(session, format("insert into %s values(1, '1001'), (2, '1002'), (3, '1003')", tableName), 3); + + Session txnSession1 = assertStartTransaction(session, "START TRANSACTION"); + Session txnSession2 = assertStartTransaction(session, "START TRANSACTION"); + + assertUpdate(txnSession1, format("insert into %s values(4, '1004')", tableName), 1); + assertUpdate(txnSession2, format("insert into %s values(5, '1005')", tableName), 1); + assertUpdate(txnSession1, format("delete from %s where a > 2", tableName), 2); + assertUpdate(txnSession2, format("update %s set a = a + 10 where b < '1003'", tableName), 2); + + // transaction1 can just read its own writes + assertQuery(txnSession1, "select * from " + tableName, "values(1, '1001'), (2, '1002')"); + // transaction2 can just read its own writes + assertQuery(txnSession2, "select * from " + tableName, "values(11, '1001'), (12, '1002'), (3, '1003'), (5, '1005')"); + // Cannot read any change outside transaction1 and transaction2 + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002'), (3, '1003')"); + + // transaction1 commit successfully + Session session1 = assertEndTransaction(txnSession1, "commit"); + // Can read the writes of transaction1 from outside + assertQuery(session1, "select * from " + tableName, "values(1, '1001'), (2, '1002')"); + assertQuery(getSession(), "select * from " + tableName, "values(1, '1001'), (2, '1002')"); + // transaction2 can just read its own writes, unaware of outside change + assertQuery(txnSession2, "select * from " + tableName, "values(11, '1001'), (12, '1002'), (3, '1003'), (5, '1005')"); + + // The commission of transaction2 lead in some kind of write skew + Session session2 = assertEndTransaction(txnSession2, "commit"); + assertQuery(session1, "select * from " + tableName, "values(11, '1001'), (12, '1002'), (5, '1005')"); + assertQuery(session2, "select * from " + tableName, "values(11, '1001'), (12, '1002'), (5, '1005')"); + assertQuery(getSession(), "select * from " + tableName, "values(11, '1001'), (12, '1002'), (5, '1005')"); + + assertUpdate("drop table " + tableName); + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewMetadata.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewMetadata.java index 45c1fad7c8740..c441132c7654a 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewMetadata.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewMetadata.java @@ -14,6 +14,7 @@ package com.facebook.presto.iceberg; import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.presto.Session; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableMap; @@ -98,7 +99,9 @@ protected QueryRunner createQueryRunner() .setDataDirectory(Optional.of(warehouseLocation.toPath())) .setSchemaName("test_schema") .setCreateTpchTables(false) - .setExtraProperties(ImmutableMap.of("experimental.legacy-materialized-views", "false")) + .setExtraProperties(ImmutableMap.of( + "experimental.legacy-materialized-views", "false", + "experimental.allow-legacy-materialized-views-toggle", "true")) .build().getQueryRunner(); } @@ -756,4 +759,44 @@ public void testStalenessPropertiesStoredInView() assertUpdate("DROP MATERIALIZED VIEW test_staleness_props_mv"); assertUpdate("DROP TABLE test_staleness_props_base"); } + + @Test + public void testNoOrphanStorageTableOnValidationFailure() + throws Exception + { + try (RESTCatalog catalog = new RESTCatalog()) { + assertUpdate("CREATE TABLE test_orphan_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_orphan_base VALUES (1, 100)", 1); + + Session legacySession = Session.builder(getSession()) + .setSystemProperty("legacy_materialized_views", "true") + .build(); + + String mvName = "test_orphan_mv"; + String storageTableName = "__mv_storage__" + mvName; + + assertQueryFails( + legacySession, + "CREATE MATERIALIZED VIEW " + mvName + " AS SELECT id, value FROM test_orphan_base", + ".*Materialized view security mode is required.*"); + + assertQueryFails( + "SELECT COUNT(*) FROM \"" + storageTableName + "\"", + ".*(does not exist|not found).*"); + + Map catalogProps = new HashMap<>(); + catalogProps.put("uri", serverUri); + catalogProps.put("warehouse", warehouseLocation.getAbsolutePath()); + catalog.initialize("test_catalog", catalogProps); + + TableIdentifier storageTableId = TableIdentifier.of(Namespace.of("test_schema"), storageTableName); + boolean tableExists = catalog.tableExists(storageTableId); + assertFalse(tableExists, + "Storage table should not exist after failed MV creation. " + + "This would indicate validation happened after storage table creation."); + } + finally { + assertUpdate("DROP TABLE test_orphan_base"); + } + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewOptimizer.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewOptimizer.java index 8480b987817a1..4a90e82f1f246 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewOptimizer.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewOptimizer.java @@ -14,8 +14,13 @@ package com.facebook.presto.iceberg; import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.presto.Session; +import com.facebook.presto.common.predicate.Range; +import com.facebook.presto.common.predicate.SortedRangeSet; +import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.assertj.core.util.Files; import org.testng.annotations.AfterClass; @@ -25,23 +30,35 @@ import java.io.File; import java.util.Optional; +import static com.facebook.presto.common.predicate.Domain.create; +import static com.facebook.presto.common.predicate.Domain.singleValue; +import static com.facebook.presto.common.predicate.Range.greaterThan; +import static com.facebook.presto.common.predicate.Range.lessThan; +import static com.facebook.presto.common.type.DateType.DATE; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.iceberg.CatalogType.REST; import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.getRestServer; import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.restConnectorProperties; +import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.constrainedTableScan; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.exchange; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static io.airlift.slice.Slices.utf8Slice; -/** - * Plan-level tests for MaterializedView optimizer rule. - * Verifies that the optimizer correctly decides when to use UNION stitching vs full recompute. - */ @Test(singleThreaded = true) public class TestIcebergMaterializedViewOptimizer extends AbstractTestQueryFramework { + private static final String MV_STORAGE = "__mv_storage__"; private File warehouseLocation; private TestingHttpServer restServer; @@ -74,79 +91,1617 @@ protected QueryRunner createQueryRunner() { return IcebergQueryRunner.builder() .setCatalogType(REST) - .setExtraConnectorProperties(restConnectorProperties(restServer.getBaseUrl().toString())) + .setExtraConnectorProperties(ImmutableMap.builder() + .putAll(restConnectorProperties(restServer.getBaseUrl().toString())) + .put("iceberg.materialized-view-storage-prefix", MV_STORAGE) + .build()) .setDataDirectory(Optional.of(warehouseLocation.toPath())) .setSchemaName("test_schema") .setCreateTpchTables(false) - .setExtraProperties(ImmutableMap.of("experimental.legacy-materialized-views", "false")) + .setExtraProperties(ImmutableMap.of( + "experimental.legacy-materialized-views", "false", + "optimizer.optimize-hash-generation", "false", + "materialized-view-stale-read-behavior", "USE_STITCHING")) .build().getQueryRunner(); } @Test public void testBasicOptimization() { - assertUpdate("CREATE TABLE base_no_parts (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO base_no_parts VALUES (1, 100), (2, 200)", 2); + try { + assertUpdate("CREATE TABLE base_no_parts (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO base_no_parts VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_no_parts AS SELECT id, value FROM base_no_parts"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_no_parts"); + + assertUpdate("INSERT INTO base_no_parts VALUES (3, 300)", 1); - assertUpdate("CREATE MATERIALIZED VIEW mv_no_parts AS SELECT id, value FROM base_no_parts"); - getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_no_parts"); + assertPlan("SELECT * FROM mv_no_parts", + anyTree(tableScan("base_no_parts"))); - assertUpdate("INSERT INTO base_no_parts VALUES (3, 300)", 1); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_no_parts"); - assertPlan("SELECT * FROM mv_no_parts", - anyTree(tableScan("base_no_parts"))); + assertPlan("SELECT * FROM mv_no_parts", + anyTree(tableScan("__mv_storage__mv_no_parts"))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_no_parts"); + assertUpdate("DROP TABLE IF EXISTS base_no_parts"); + } + } + + @Test + public void testUnionStitchingWithStalePartition() + { + try { + assertUpdate("CREATE TABLE base_table (id BIGINT, ds VARCHAR) WITH (partitioning = ARRAY['ds'])"); + assertUpdate("INSERT INTO base_table VALUES (1, '2024-01-01'), (2, '2024-01-02')", 2); - getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_no_parts"); + assertUpdate("CREATE MATERIALIZED VIEW test_mv " + + "WITH (partitioning = ARRAY['ds']) AS SELECT id, ds FROM base_table"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW test_mv"); - assertPlan("SELECT * FROM mv_no_parts", - anyTree(tableScan("__mv_storage__mv_no_parts"))); + assertUpdate("INSERT INTO base_table VALUES (3, '2024-01-03')", 1); - assertUpdate("DROP MATERIALIZED VIEW mv_no_parts"); - assertUpdate("DROP TABLE base_no_parts"); + assertPlan("SELECT * FROM test_mv", + output( + exchange( + constrainedTableScan("__mv_storage__test_mv", + ImmutableMap.of("ds", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-0T3")), + greaterThan(VARCHAR, utf8Slice("2024-01-03")))), false)), + ImmutableMap.of("ds", "ds", "id", "id")), + project(constrainedTableScan("base_table", + ImmutableMap.of("ds", singleValue(VARCHAR, utf8Slice("2024-01-03"))), + ImmutableMap.of("id_1", "id")))))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS test_mv"); + assertUpdate("DROP TABLE IF EXISTS base_table"); + } + } + + @Test + public void testFallbackForNonPartitionedTable() + { + try { + assertUpdate("CREATE TABLE base_no_parts (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO base_no_parts VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_no_parts AS SELECT id, value FROM base_no_parts"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_no_parts"); + + assertUpdate("INSERT INTO base_no_parts VALUES (3, 300)", 1); + + assertPlan("SELECT * FROM mv_no_parts", + anyTree(tableScan("base_no_parts"))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_no_parts"); + assertUpdate("DROP TABLE IF EXISTS base_no_parts"); + } } @Test public void testMultiTableStaleness() { - // Create two partitioned base tables - assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, ds VARCHAR) " + - "WITH (partitioning = ARRAY['ds'])"); - assertUpdate("CREATE TABLE customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + - "WITH (partitioning = ARRAY['reg_date'])"); - - assertUpdate("INSERT INTO orders VALUES (1, 100, '2024-01-01')", 1); - assertUpdate("INSERT INTO customers VALUES (100, 'Alice', '2024-01-01')", 1); - - // Create JOIN MV with partition columns in output - assertUpdate("CREATE MATERIALIZED VIEW mv_join AS " + - "SELECT o.order_id, c.name, o.ds, c.reg_date " + - "FROM orders o JOIN customers c ON o.customer_id = c.customer_id"); - getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_join"); - - // Make one table stale - assertUpdate("INSERT INTO orders VALUES (2, 200, '2024-01-02')", 1); - - assertPlan("SELECT * FROM mv_join", - anyTree( - anyTree( - join( - anyTree(tableScan("orders")), - anyTree(tableScan("customers")))))); - - getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_join"); - - // Make both tables stale - assertUpdate("INSERT INTO orders VALUES (2, 200, '2024-01-02')", 1); - assertUpdate("INSERT INTO customers VALUES (200, 'Bob', '2024-01-02')", 1); - - assertPlan("SELECT * FROM mv_join", - anyTree( - anyTree( - join( - anyTree(tableScan("orders")), - anyTree(tableScan("customers")))))); - assertUpdate("DROP MATERIALIZED VIEW mv_join"); - assertUpdate("DROP TABLE customers"); - assertUpdate("DROP TABLE orders"); + try { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, ds VARCHAR) " + + "WITH (partitioning = ARRAY['ds'])"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO orders VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_join " + + "WITH (partitioning = ARRAY['ds', 'reg_date']) AS " + + "SELECT o.order_id, c.name, o.ds, c.reg_date " + + "FROM orders o JOIN customers c ON o.customer_id = c.customer_id"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_join"); + + assertUpdate("INSERT INTO orders VALUES (2, 200, '2024-01-02')", 1); + assertUpdate("INSERT INTO customers VALUES (200, 'Bob', '2024-01-02')", 1); + + PlanMatchPattern staleBranchPattern = join( + anyTree(tableScan("orders")), + anyTree(tableScan("customers"))); + + assertPlan("SELECT * FROM mv_join", + output( + exchange( + constrainedTableScan("__mv_storage__mv_join", + ImmutableMap.of( + "ds", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false), + "reg_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("ds", "ds", "reg_date", "reg_date", "order_id", "order_id", "name", "name")), + exchange( + project(staleBranchPattern), + project(staleBranchPattern))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_join", + output(exchange(staleBranchPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_join"); + assertUpdate("DROP TABLE IF EXISTS customers"); + assertUpdate("DROP TABLE IF EXISTS orders"); + } + } + + @Test + public void testAggregationMV() + { + try { + assertUpdate("CREATE TABLE sales (product_id BIGINT, amount DOUBLE, sale_date VARCHAR) " + + "WITH (partitioning = ARRAY['sale_date'])"); + + assertUpdate("INSERT INTO sales VALUES (1, 100.0, '2024-01-01'), (2, 200.0, '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_sales_agg " + + "WITH (partitioning = ARRAY['sale_date']) AS " + + "SELECT product_id, sale_date, SUM(amount) as total_amount " + + "FROM sales GROUP BY product_id, sale_date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_sales_agg"); + + assertUpdate("INSERT INTO sales VALUES (3, 150.0, '2024-01-02')", 1); + + PlanMatchPattern staleBranchPattern = aggregation( + ImmutableMap.of(), + anyTree(tableScan("sales"))); + PlanMatchPattern stalePlan = project(staleBranchPattern); + + assertPlan("SELECT * FROM mv_sales_agg", + output( + exchange( + constrainedTableScan("__mv_storage__mv_sales_agg", + ImmutableMap.of("sale_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("sale_date", "sale_date", "product_id", "product_id", "total_amount", "total_amount")), + stalePlan))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_sales_agg", + output(exchange(staleBranchPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_sales_agg"); + assertUpdate("DROP TABLE IF EXISTS sales"); + } + } + + @Test + public void testThreeTableJoinMV() + { + try { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, product_id BIGINT, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, customer_name VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + assertUpdate("CREATE TABLE products (product_id BIGINT, product_name VARCHAR, category VARCHAR) " + + "WITH (partitioning = ARRAY['category'])"); + + assertUpdate("INSERT INTO orders VALUES (1, 100, 1000, '2024-01-01')", 1); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice', 'US')", 1); + assertUpdate("INSERT INTO products VALUES (1000, 'Widget', 'tools')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_order_details " + + "WITH (partitioning = ARRAY['order_date', 'region', 'category']) AS " + + "SELECT o.order_id, c.customer_name, p.product_name, o.order_date, c.region, p.category " + + "FROM orders o " + + "JOIN customers c ON o.customer_id = c.customer_id " + + "JOIN products p ON o.product_id = p.product_id"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_order_details"); + + assertUpdate("INSERT INTO orders VALUES (2, 200, 2000, '2024-01-02')", 1); + assertUpdate("INSERT INTO customers VALUES (200, 'Bob', 'EU')", 1); + assertUpdate("INSERT INTO products VALUES (2000, 'Gadget', 'electronics')", 1); + + // For a 3-table join (orders JOIN customers JOIN products), with 3 stale partitions + // from 3 different tables, the delta algebra creates complex nested unions. + // Actual structure: LocalExchange[ROUND_ROBIN] with 2 children: + // 1. InnerJoin(LocalExchange[ROUND_ROBIN] -> [2 orders/customers delta branches], products) + // 2. Project -> InnerJoin((orders JOIN customers) JOIN products with category constraint) + + // First stale branch: orders/customers delta joined with products + // LocalExchange[ROUND_ROBIN] has two project branches: + // - Project -> InnerJoin(orders with order_date='2024-01-02', customers full) + // - Project -> InnerJoin(orders with order_date != '2024-01-02', customers with region='EU') + PlanMatchPattern orderCustomerDelta1 = project( + join( + exchange(constrainedTableScan("orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(tableScan("customers"))))); + + PlanMatchPattern orderCustomerDelta2 = project( + join( + exchange(constrainedTableScan("orders", + ImmutableMap.of("order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("customers", + ImmutableMap.of("region", singleValue(VARCHAR, utf8Slice("EU"))), + ImmutableMap.of()))))); + + // First branch: InnerJoin with nested LocalExchange for orders/customers deltas, then products + PlanMatchPattern firstBranch = join( + exchange( + exchange(orderCustomerDelta1), + exchange(orderCustomerDelta2)), + exchange(exchange(tableScan("products")))); + + // Second branch: orders JOIN customers JOIN products with category='electronics' + PlanMatchPattern ordersCustomersForCategoryDelta = join( + exchange(constrainedTableScan("orders", + ImmutableMap.of("order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("customers", + ImmutableMap.of("region", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("EU")), + greaterThan(VARCHAR, utf8Slice("EU")))), false)), + ImmutableMap.of())))); + + PlanMatchPattern secondBranch = project( + join( + exchange(ordersCustomersForCategoryDelta), + exchange(exchange(constrainedTableScan("products", + ImmutableMap.of("category", singleValue(VARCHAR, utf8Slice("electronics"))), + ImmutableMap.of()))))); + + assertPlan("SELECT * FROM mv_order_details", + output( + exchange( + constrainedTableScan("__mv_storage__mv_order_details", + ImmutableMap.of( + "order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false), + "region", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("EU")), + greaterThan(VARCHAR, utf8Slice("EU")))), false), + "category", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("electronics")), + greaterThan(VARCHAR, utf8Slice("electronics")))), false)), + ImmutableMap.of("order_date", "order_date", "region", "region", "category", "category", + "order_id", "order_id", "customer_name", "customer_name", "product_name", "product_name")), + exchange( + firstBranch, + secondBranch)))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + PlanMatchPattern nestedJoinPattern = join( + exchange(join( + exchange(tableScan("orders")), + exchange(exchange(tableScan("customers"))))), + exchange(exchange(tableScan("products")))); + assertPlan(skipStorageSession, "SELECT * FROM mv_order_details", + output(exchange(nestedJoinPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_order_details"); + assertUpdate("DROP TABLE IF EXISTS products"); + assertUpdate("DROP TABLE IF EXISTS customers"); + assertUpdate("DROP TABLE IF EXISTS orders"); + } + } + + @Test + public void testJoinWithFilterMV() + { + try { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, amount DOUBLE, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, name VARCHAR, status VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO orders VALUES (1, 100, 50.0, '2024-01-01')", 1); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice', 'active', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_active_orders " + + "WITH (partitioning = ARRAY['order_date', 'reg_date']) AS " + + "SELECT o.order_id, c.name, o.amount, o.order_date, c.reg_date " + + "FROM orders o JOIN customers c ON o.customer_id = c.customer_id " + + "WHERE c.status = 'active' AND o.amount > 10.0"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_active_orders"); + + assertUpdate("INSERT INTO orders VALUES (2, 200, 100.0, '2024-01-02')", 1); + assertUpdate("INSERT INTO customers VALUES (200, 'Bob', 'active', '2024-01-02')", 1); + + PlanMatchPattern staleBranchPattern = join( + anyTree(tableScan("orders")), + anyTree(tableScan("customers"))); + + assertPlan("SELECT * FROM mv_active_orders", + output( + exchange( + constrainedTableScan("__mv_storage__mv_active_orders", + ImmutableMap.of( + "order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false), + "reg_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("order_date", "order_date", "reg_date", "reg_date", + "order_id", "order_id", "name", "name", "amount", "amount")), + exchange( + project(staleBranchPattern), + project(staleBranchPattern))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_active_orders", + output(exchange(staleBranchPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_active_orders"); + assertUpdate("DROP TABLE IF EXISTS customers"); + assertUpdate("DROP TABLE IF EXISTS orders"); + } + } + + @Test + public void testSingleTableStaleness() + { + try { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, ds VARCHAR) " + + "WITH (partitioning = ARRAY['ds'])"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO orders VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_partial_stale " + + "WITH (partitioning = ARRAY['ds', 'reg_date']) AS " + + "SELECT o.order_id, c.name, o.ds, c.reg_date " + + "FROM orders o JOIN customers c ON o.customer_id = c.customer_id"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_partial_stale"); + + assertUpdate("INSERT INTO orders VALUES (2, 100, '2024-01-02')", 1); + + PlanMatchPattern staleBranchPattern = join( + anyTree(tableScan("orders")), + anyTree(tableScan("customers"))); + PlanMatchPattern stalePlan = project(staleBranchPattern); + + assertPlan("SELECT * FROM mv_partial_stale", + output( + exchange( + constrainedTableScan("__mv_storage__mv_partial_stale", + ImmutableMap.of( + "ds", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("ds", "ds", "reg_date", "reg_date", "order_id", "order_id", "name", "name")), + stalePlan))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_partial_stale", + output(exchange(staleBranchPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_partial_stale"); + assertUpdate("DROP TABLE IF EXISTS customers"); + assertUpdate("DROP TABLE IF EXISTS orders"); + } + } + + @Test + public void testUnionPredicatePushdown() + { + try { + assertUpdate("CREATE TABLE union_table1 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE union_table2 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO union_table1 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-02')", 2); + assertUpdate("INSERT INTO union_table2 VALUES (3, 'c', '2024-01-01'), (4, 'd', '2024-01-02')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_union " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM union_table1 " + + "UNION " + + "SELECT id, value, dt FROM union_table2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_union"); + + assertUpdate("INSERT INTO union_table1 VALUES (5, 'e', '2024-01-03')", 1); + + assertPlan("SELECT * FROM mv_union", + output( + exchange( + constrainedTableScan("__mv_storage__mv_union", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-03")), + greaterThan(VARCHAR, utf8Slice("2024-01-03")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + aggregation( + ImmutableMap.of(), + FINAL, + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(constrainedTableScan("union_table1", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-03"))), + ImmutableMap.of("id_1", "id", "value_1", "value"))))))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_union", + output(exchange( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + tableScan("union_table1", ImmutableMap.of("dt", "dt"))))), + project( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + tableScan("union_table2"))))))))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_union"); + assertUpdate("DROP TABLE IF EXISTS union_table2"); + assertUpdate("DROP TABLE IF EXISTS union_table1"); + } + } + + @Test + public void testJoinPredicatePushdown() + { + try { + assertUpdate("CREATE TABLE join_orders (order_id BIGINT, customer_id BIGINT, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE join_customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO join_orders VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO join_customers VALUES (100, 'Alice', '2024-01-01')", 1); + assertUpdate("INSERT INTO join_customers VALUES (300, 'Candace', '2024-01-02')", 1); + assertUpdate("INSERT INTO join_orders VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO join_customers VALUES (400, 'Billy', '2024-01-03')", 1); + assertUpdate("INSERT INTO join_orders VALUES (4, 400, '2024-01-03')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_join " + + "WITH (partitioning = ARRAY['order_date']) AS " + + "SELECT o.order_id, c.name, o.order_date, c.reg_date " + + "FROM join_orders o " + + "JOIN join_customers c ON o.customer_id = c.customer_id"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_join"); + + assertUpdate("INSERT INTO join_orders VALUES (2, 200, '2024-01-02')", 1); + assertUpdate("INSERT INTO join_customers VALUES (200, 'Bob', '2024-01-01')", 1); + + // MV is partitioned by order_date only. The filter for reg_date becomes a residual + // filter predicate (ScanFilter node) rather than just a domain constraint, because + // reg_date is not a partition column in the MV storage table. + assertPlan("SELECT * FROM mv_join", + output( + exchange( + filter("(reg_date) <> (VARCHAR '2024-01-01')", + constrainedTableScan("__mv_storage__mv_join", + ImmutableMap.of( + "order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(lessThan(VARCHAR, utf8Slice("2024-01-02")), greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false), + "reg_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(lessThan(VARCHAR, utf8Slice("2024-01-01")), greaterThan(VARCHAR, utf8Slice("2024-01-01")))), false)), + ImmutableMap.of("order_date", "order_date", "reg_date", "reg_date", "order_id", "order_id", "name", "name"))), + exchange( + project(join( + exchange(constrainedTableScan("join_orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(tableScan("join_customers"))))), + project(join( + exchange(constrainedTableScan("join_orders", + // R (unchanged) = all non-stale partitions: order_date != '2024-01-02' + ImmutableMap.of("order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of(lessThan(VARCHAR, utf8Slice("2024-01-02")), greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("join_customers", + ImmutableMap.of("reg_date", singleValue(VARCHAR, utf8Slice("2024-01-01"))), + ImmutableMap.of()))))))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_join", + output(exchange(join( + anyTree(tableScan("join_orders")), + anyTree(tableScan("join_customers")))))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_join"); + assertUpdate("DROP TABLE IF EXISTS join_customers"); + assertUpdate("DROP TABLE IF EXISTS join_orders"); + } + } + + @Test + public void testJoinPassthroughPartition() + { + try { + assertUpdate("CREATE TABLE passthrough_orders (order_id BIGINT, customer_id BIGINT, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE passthrough_customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO passthrough_orders VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO passthrough_customers VALUES (100, 'Alice', '2024-01-01')", 1); + assertUpdate("INSERT INTO passthrough_customers VALUES (300, 'Candace', '2024-01-02')", 1); + assertUpdate("INSERT INTO passthrough_orders VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO passthrough_customers VALUES (400, 'Billy', '2024-01-03')", 1); + assertUpdate("INSERT INTO passthrough_orders VALUES (4, 400, '2024-01-03')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_passthrough " + + "WITH (partitioning = ARRAY['order_date']) AS " + + "SELECT o.order_id, c.name, o.order_date " + + "FROM passthrough_orders o " + + "JOIN passthrough_customers c ON o.customer_id = c.customer_id AND o.order_date = c.reg_date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_passthrough"); + + assertUpdate("INSERT INTO passthrough_orders VALUES (2, 200, '2024-01-02')", 1); + assertUpdate("INSERT INTO passthrough_customers VALUES (200, 'Bob', '2024-01-01')", 1); + + assertPlan("SELECT * FROM mv_passthrough", + output( + exchange( + constrainedTableScan("__mv_storage__mv_passthrough", + ImmutableMap.of( + "order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-01")), + Range.range(VARCHAR, utf8Slice("2024-01-01"), false, utf8Slice("2024-01-02"), false), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("order_date", "order_date", "order_id", "order_id", "name", "name")), + exchange( + project(join( + anyTree(constrainedTableScan("passthrough_orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of("order_id_1", "order_id", "customer_id_1", "customer_id"))), + anyTree(constrainedTableScan("passthrough_customers", + ImmutableMap.of("reg_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of("customer_id_2", "customer_id", "name_2", "name"))))), + project(join( + anyTree(constrainedTableScan("passthrough_orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-01"))), + ImmutableMap.of("order_id_3", "order_id", "customer_id_3", "customer_id"))), + anyTree(constrainedTableScan("passthrough_customers", + ImmutableMap.of("reg_date", singleValue(VARCHAR, utf8Slice("2024-01-01"))), + ImmutableMap.of("customer_id_4", "customer_id", "name_4", "name"))))))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_passthrough", + output(exchange(join( + anyTree(tableScan("passthrough_orders", ImmutableMap.of("order_date", "order_date"))), + anyTree(tableScan("passthrough_customers", ImmutableMap.of("reg_date", "reg_date"))))))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_passthrough"); + assertUpdate("DROP TABLE IF EXISTS passthrough_customers"); + assertUpdate("DROP TABLE IF EXISTS passthrough_orders"); + } + } + + @Test + public void testIntersectPredicatePushdown() + { + try { + assertUpdate("CREATE TABLE intersect_table1 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE intersect_table2 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO intersect_table1 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01')", 2); + assertUpdate("INSERT INTO intersect_table2 VALUES (2, 'b', '2024-01-01'), (3, 'c', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_intersect " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM intersect_table1 " + + "INTERSECT " + + "SELECT id, value, dt FROM intersect_table2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_intersect"); + + assertUpdate("INSERT INTO intersect_table1 VALUES (4, 'd', '2024-01-02'), (5, 'e', '2024-01-02')", 2); + assertUpdate("INSERT INTO intersect_table2 VALUES (5, 'e', '2024-01-02')", 1); + + // For INTERSECT with stale data on both tables, the plan creates TWO union branches. + // Branch 1: FilterProject -> Aggregation(FINAL) -> LocalExchange -> + // [Project -> ... -> Aggregation(PARTIAL) -> table1, + // Project -> ... -> Aggregation(PARTIAL) -> table2] + // Branch 2: FilterProject -> Aggregation(FINAL) -> LocalExchange -> + // [Project -> ... -> Aggregation(PARTIAL) -> table2] (only table2!) + // This is because table1 has 2 new rows while table2 has 1 new row, creating + // different delta algebra terms. + PlanMatchPattern intersectBranchBoth = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(constrainedTableScan("intersect_table1", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))), + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(constrainedTableScan("intersect_table2", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))))))); + + PlanMatchPattern intersectBranchTable2Only = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(constrainedTableScan("intersect_table2", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))))))); + + assertPlan("SELECT * FROM mv_intersect", + output( + exchange( + constrainedTableScan("__mv_storage__mv_intersect", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + intersectBranchBoth, + intersectBranchTable2Only))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + // When forcing stale read, the full query scans both tables WITHOUT partition constraints + PlanMatchPattern fullIntersectPattern = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(tableScan("intersect_table1")))), + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + anyTree(tableScan("intersect_table2")))))))); + assertPlan(skipStorageSession, "SELECT * FROM mv_intersect", + output(exchange(fullIntersectPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_intersect"); + assertUpdate("DROP TABLE IF EXISTS intersect_table2"); + assertUpdate("DROP TABLE IF EXISTS intersect_table1"); + } + } + + /** + * Test INTERSECT where only the LEFT side becomes stale. + * Verifies that the RIGHT side gets filtered to the left's stale partitions + * via predicate propagation using column equivalences. + */ + @Test + public void testIntersectWithOnlyLeftSideStale() + { + try { + assertUpdate("CREATE TABLE intersect_left_only1 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE intersect_left_only2 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data - same rows in both tables for intersection + assertUpdate("INSERT INTO intersect_left_only1 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01')", 2); + assertUpdate("INSERT INTO intersect_left_only2 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01'), (3, 'c', '2024-01-01')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_intersect_left " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM intersect_left_only1 " + + "INTERSECT " + + "SELECT id, value, dt FROM intersect_left_only2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_intersect_left"); + + // Make ONLY left side stale + assertUpdate("INSERT INTO intersect_left_only1 VALUES (4, 'd', '2024-01-02')", 1); + // Add matching data to right side so intersection has results + assertUpdate("INSERT INTO intersect_left_only2 VALUES (4, 'd', '2024-01-02')", 1); + // Refresh to make right side fresh again + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_intersect_left"); + + // Now make left stale again with a new partition + assertUpdate("INSERT INTO intersect_left_only1 VALUES (5, 'e', '2024-01-03')", 1); + assertUpdate("INSERT INTO intersect_left_only2 VALUES (5, 'e', '2024-01-03')", 1); + + // The optimization should filter intersect_left_only2 to dt='2024-01-03' + // even though it's not directly stale. + // NOTE: There are TWO union branches because only one row is inserted into each table, + // creating different delta algebra terms: one branch scans both tables, another scans only table2 + // Structure for each branch: + // - FilterProject (combined filter+project node = project(filter(...))) + // - Aggregate(FINAL) + // - LocalExchange[HASH] (partitioning on grouping keys) + // - Project (first child) + // - RemoteStreamingExchange[REPARTITION] + // - Aggregate(PARTIAL) + // - ScanProject[table1] (= project(tableScan(...))) + // - Project (second child) + // - RemoteStreamingExchange[REPARTITION] + // - Aggregate(PARTIAL) + // - ScanProject[table2] (= project(tableScan(...))) + PlanMatchPattern intersectBranchBoth = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("intersect_left_only1"))))), + project( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("intersect_left_only2"))))))))); + + PlanMatchPattern intersectBranchTable2Only = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("intersect_left_only2")))))))); + + assertPlan("SELECT * FROM mv_intersect_left", + output( + exchange( + constrainedTableScan("__mv_storage__mv_intersect_left", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-03")), + greaterThan(VARCHAR, utf8Slice("2024-01-03")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + intersectBranchBoth, + intersectBranchTable2Only))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_intersect_left"); + assertUpdate("DROP TABLE IF EXISTS intersect_left_only2"); + assertUpdate("DROP TABLE IF EXISTS intersect_left_only1"); + } + } + + /** + * Test INTERSECT where only the RIGHT side becomes stale. + * Verifies that the LEFT side gets filtered to the right's stale partitions + * via predicate propagation using column equivalences. + */ + @Test + public void testIntersectWithOnlyRightSideStale() + { + try { + assertUpdate("CREATE TABLE intersect_right_only1 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE intersect_right_only2 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO intersect_right_only1 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01')", 2); + assertUpdate("INSERT INTO intersect_right_only2 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_intersect_right " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM intersect_right_only1 " + + "INTERSECT " + + "SELECT id, value, dt FROM intersect_right_only2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_intersect_right"); + + // Make ONLY right side stale + assertUpdate("INSERT INTO intersect_right_only2 VALUES (3, 'c', '2024-01-02')", 1); + // Add matching data to left side + assertUpdate("INSERT INTO intersect_right_only1 VALUES (3, 'c', '2024-01-02')", 1); + // Refresh to make left side fresh + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_intersect_right"); + + // Now make right stale again + assertUpdate("INSERT INTO intersect_right_only2 VALUES (4, 'd', '2024-01-03')", 1); + assertUpdate("INSERT INTO intersect_right_only1 VALUES (4, 'd', '2024-01-03')", 1); + + // The optimization should filter intersect_right_only1 (left side) to dt='2024-01-03' + // even though it's not directly stale, because we're computing R ∩ ∆S + PlanMatchPattern intersectBranch1 = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(constrainedTableScan("intersect_right_only1", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-03"))), + ImmutableMap.of()))))), + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(constrainedTableScan("intersect_right_only2", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-03"))), + ImmutableMap.of()))))))))); + + // Second intersect branch: only table2 (simpler structure since table1 is pruned) + PlanMatchPattern intersectBranch2 = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(constrainedTableScan("intersect_right_only2", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-03"))), + ImmutableMap.of())))))))); + + assertPlan("SELECT * FROM mv_intersect_right", + output( + exchange( + constrainedTableScan("__mv_storage__mv_intersect_right", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-03")), + greaterThan(VARCHAR, utf8Slice("2024-01-03")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + intersectBranch1, + intersectBranch2))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_intersect_right"); + assertUpdate("DROP TABLE IF EXISTS intersect_right_only2"); + assertUpdate("DROP TABLE IF EXISTS intersect_right_only1"); + } + } + + @Test + public void testDeeplyNestedJoins() + { + try { + assertUpdate("CREATE TABLE dnj_orders (order_id BIGINT, customer_id BIGINT, product_id BIGINT, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE dnj_customers (customer_id BIGINT, customer_name VARCHAR, region VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + assertUpdate("CREATE TABLE dnj_products (product_id BIGINT, product_name VARCHAR, category_id BIGINT, product_date VARCHAR) " + + "WITH (partitioning = ARRAY['product_date'])"); + assertUpdate("CREATE TABLE dnj_categories (category_id BIGINT, category_name VARCHAR, cat_date VARCHAR) " + + "WITH (partitioning = ARRAY['cat_date'])"); + + assertUpdate("INSERT INTO dnj_orders VALUES " + + "(1, 100, 1000, '2024-01-01'), " + + "(2, 200, 2000, '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_customers VALUES " + + "(100, 'Alice', 'US', '2024-01-01'), " + + "(200, 'Bob', 'EU', '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_products VALUES " + + "(1000, 'Laptop', 10, '2024-01-01'), " + + "(2000, 'Phone', 20, '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_categories VALUES " + + "(10, 'Electronics', '2024-01-01'), " + + "(20, 'Mobile', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_dnj " + + "WITH (partitioning = ARRAY['order_date']) AS " + + "SELECT oc.order_id, oc.customer_name, pc.product_name, pc.category_name, oc.order_date " + + "FROM " + + " (SELECT o.order_id, c.customer_name, o.product_id, o.order_date FROM dnj_orders o " + + " JOIN dnj_customers c ON o.customer_id = c.customer_id AND o.order_date = c.reg_date) oc " + + " JOIN " + + " (SELECT p.product_id, p.product_name, cat.category_name, p.product_date FROM dnj_products p " + + " JOIN dnj_categories cat ON p.category_id = cat.category_id AND p.product_date = cat.cat_date) pc " + + " ON oc.product_id = pc.product_id AND oc.order_date = pc.product_date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + assertUpdate("INSERT INTO dnj_customers VALUES (300, 'Charlie', 'US', '2024-01-02')", 1); + assertUpdate("INSERT INTO dnj_products VALUES (3000, 'Tablet', 10, '2024-01-02')", 1); + assertUpdate("INSERT INTO dnj_categories VALUES (10, 'Electronics', '2024-01-02')", 1); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + assertUpdate("INSERT INTO dnj_orders VALUES (3, 300, 3000, '2024-01-02')", 1); + + // The stale branch for a 4-table deeply nested join has this structure: + // InnerJoin[(orders-customers) x (products-categories)] + // Actual structure: + // - Left branch: exchange -> project -> innerJoin(orders, customers) + // - Right branch: exchange -> exchange -> innerJoin(products, categories) - NO project wrapper! + PlanMatchPattern innerJoin1 = join( + exchange(constrainedTableScan("dnj_orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("dnj_customers", + ImmutableMap.of("reg_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))); + PlanMatchPattern innerJoin2 = join( + exchange(constrainedTableScan("dnj_products", + ImmutableMap.of("product_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("dnj_categories", + ImmutableMap.of("cat_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))); + // Outer join structure: left has project wrapper, right does NOT + PlanMatchPattern nestedJoinPattern = join( + exchange(project(innerJoin1)), + exchange(exchange(innerJoin2))); + + assertPlan("SELECT * FROM mv_dnj", + output( + exchange( + tableScan("__mv_storage__mv_dnj"), + nestedJoinPattern))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + // For skipStorageSession, tables are scanned without partition constraints (full scan) + PlanMatchPattern innerJoin1Full = join( + exchange(tableScan("dnj_orders")), + exchange(exchange(tableScan("dnj_customers")))); + PlanMatchPattern innerJoin2Full = join( + exchange(tableScan("dnj_products")), + exchange(exchange(tableScan("dnj_categories")))); + PlanMatchPattern nestedJoinPatternFull = join( + exchange(innerJoin1Full), + exchange(exchange(innerJoin2Full))); + assertPlan(skipStorageSession, "SELECT * FROM mv_dnj", + output(exchange(nestedJoinPatternFull))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_dnj"); + assertUpdate("DROP TABLE IF EXISTS dnj_categories"); + assertUpdate("DROP TABLE IF EXISTS dnj_products"); + assertUpdate("DROP TABLE IF EXISTS dnj_customers"); + assertUpdate("DROP TABLE IF EXISTS dnj_orders"); + } + } + + @Test + public void testAggregationMVSkipsMarkDistinct() + { + try { + assertUpdate("CREATE TABLE agg_orders (order_id BIGINT, customer_id BIGINT, amount DOUBLE, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE agg_customers (customer_id BIGINT, name VARCHAR, reg_date VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO agg_orders VALUES (1, 100, 50.0, '2024-01-01')", 1); + assertUpdate("INSERT INTO agg_customers VALUES (100, 'Alice', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_agg_join " + + "WITH (partitioning = ARRAY['order_date', 'reg_date']) AS " + + "SELECT c.name, o.order_date, c.reg_date, SUM(o.amount) as total_amount, COUNT(*) as order_count " + + "FROM agg_orders o JOIN agg_customers c ON o.customer_id = c.customer_id " + + "GROUP BY c.name, o.order_date, c.reg_date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_agg_join"); + + assertUpdate("INSERT INTO agg_orders VALUES (2, 200, 100.0, '2024-01-02')", 1); + assertUpdate("INSERT INTO agg_customers VALUES (200, 'Bob', '2024-01-02')", 1); + + PlanMatchPattern aggregationBranch = + aggregation( + ImmutableMap.of(), + anyTree( + join( + anyTree(tableScan("agg_orders")), + anyTree(tableScan("agg_customers"))))); + + assertPlan("SELECT * FROM mv_agg_join", + output( + exchange( + constrainedTableScan("__mv_storage__mv_agg_join", + ImmutableMap.of( + "order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false), + "reg_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("order_date", "order_date", "reg_date", "reg_date", + "name", "name", "total_amount", "total_amount", "order_count", "order_count")), + aggregation(ImmutableMap.of(), anyTree(join(anyTree(tableScan("agg_orders")), anyTree(tableScan("agg_customers")))))))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + assertPlan(skipStorageSession, "SELECT * FROM mv_agg_join", + output(exchange(aggregationBranch))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_agg_join"); + assertUpdate("DROP TABLE IF EXISTS agg_customers"); + assertUpdate("DROP TABLE IF EXISTS agg_orders"); + } + } + + @Test + public void testExceptPredicatePushdown() + { + try { + assertUpdate("CREATE TABLE except_table1 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE except_table2 (id BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO except_table1 VALUES (1, 'a', '2024-01-01'), (2, 'b', '2024-01-01')", 2); + assertUpdate("INSERT INTO except_table2 VALUES (2, 'b', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_except " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM except_table1 " + + "EXCEPT " + + "SELECT id, value, dt FROM except_table2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_except"); + + assertUpdate("INSERT INTO except_table1 VALUES (3, 'c', '2024-01-02'), (4, 'd', '2024-01-02')", 2); + + // First stale branch: FilterProject -> Aggregate(FINAL) -> LocalExchange[HASH] + // with two children: Project -> RemoteExchange -> Aggregate(PARTIAL) -> ScanProject for each table + PlanMatchPattern firstStaleBranch = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(constrainedTableScan("except_table1", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of()))))), + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("except_table2"))))))))); + + // Second stale branch: FilterProject -> Aggregate(FINAL) -> LocalExchange[HASH] + // with single child: RemoteExchange -> Aggregate(PARTIAL) -> ScanProject[except_table2] + PlanMatchPattern secondStaleBranch = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("except_table2")))))))); + + assertPlan("SELECT * FROM mv_except", + output( + exchange( + constrainedTableScan("__mv_storage__mv_except", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + firstStaleBranch, + secondStaleBranch))); + + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + PlanMatchPattern fullExceptPattern = project( + filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("except_table1")))), + anyTree( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(tableScan("except_table2")))))))); + assertPlan(skipStorageSession, "SELECT * FROM mv_except", + output(exchange(fullExceptPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_except"); + assertUpdate("DROP TABLE IF EXISTS except_table2"); + assertUpdate("DROP TABLE IF EXISTS except_table1"); + } + } + + @Test + public void testPartialPassthroughColumns() + { + try { + assertUpdate("CREATE TABLE partial_orders (order_id BIGINT, amount DOUBLE, order_date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['order_date', 'region'])"); + + assertUpdate("INSERT INTO partial_orders VALUES (1, 100.0, '2024-01-01', 'US'), (2, 200.0, '2024-01-01', 'EU')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_partial_passthrough " + + "WITH (partitioning = ARRAY['order_date']) AS " + + "SELECT order_id, amount, order_date " + + "FROM partial_orders"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_partial_passthrough"); + + assertUpdate("INSERT INTO partial_orders VALUES (3, 150.0, '2024-01-02', 'US')", 1); + + assertPlan("SELECT * FROM mv_partial_passthrough", + output( + exchange( + constrainedTableScan("__mv_storage__mv_partial_passthrough", + ImmutableMap.of("order_date", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("order_date", "order_date", "order_id", "order_id", "amount", "amount")), + project(constrainedTableScan("partial_orders", + ImmutableMap.of("order_date", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of("order_id_1", "order_id", "amount_1", "amount")))))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_partial_passthrough"); + assertUpdate("DROP TABLE IF EXISTS partial_orders"); + } + } + + @Test + public void testSelectDistinctMVWithMultipleStaleTables() + { + try { + assertUpdate("CREATE TABLE test_distinct_t1 (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("CREATE TABLE test_distinct_t2 (" + + "id BIGINT, " + + "region VARCHAR, " + + "code VARCHAR, " + + "reg_date DATE) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO test_distinct_t1 VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'B', 200, DATE '2024-01-01')", 2); + + assertUpdate("INSERT INTO test_distinct_t2 VALUES " + + "(1, 'US', 'X', DATE '2024-01-01'), " + + "(2, 'EU', 'Y', DATE '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_distinct_mv " + + "WITH (partitioning = ARRAY['event_date', 'reg_date']) AS " + + "SELECT DISTINCT t1.category, t2.region, t1.event_date, t2.reg_date " + + "FROM test_distinct_t1 t1 " + + "JOIN test_distinct_t2 t2 ON t1.id = t2.id"); + + getQueryRunner().execute("REFRESH MATERIALIZED VIEW test_distinct_mv"); + + assertUpdate("INSERT INTO test_distinct_t1 VALUES " + + "(1, 'A', 150, DATE '2024-01-02'), " + + "(3, 'C', 300, DATE '2024-01-02')", 2); + + assertUpdate("INSERT INTO test_distinct_t2 VALUES " + + "(3, 'APAC', 'Z', DATE '2024-01-02')", 1); + + long jan2InDays = java.time.LocalDate.of(2024, 1, 2).toEpochDay(); + // The actual plan structure without ORDER BY: + // Output -> exchange -> [MV scan, Aggregate(FINAL) -> LocalExchange -> two stale branches] + // Branch 1: t1 constrained to event_date='2024-01-02', t2 with all rows + // Branch 2: t1 with event_date != '2024-01-02', t2 constrained to reg_date='2024-01-02' + + // Branch 1: JOIN(t1 constrained to event_date=jan2, t2 full scan) + PlanMatchPattern joinBranch1 = join( + exchange(constrainedTableScan("test_distinct_t1", + ImmutableMap.of("event_date", singleValue(DATE, jan2InDays)), + ImmutableMap.of())), + exchange(exchange(tableScan("test_distinct_t2")))); + + // Branch 2: JOIN(t1 constrained to event_date != jan2, t2 constrained to reg_date=jan2) + PlanMatchPattern joinBranch2 = join( + exchange(constrainedTableScan("test_distinct_t1", + ImmutableMap.of("event_date", create(SortedRangeSet.copyOf(DATE, ImmutableList.of( + lessThan(DATE, jan2InDays), + greaterThan(DATE, jan2InDays))), false)), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("test_distinct_t2", + ImmutableMap.of("reg_date", singleValue(DATE, jan2InDays)), + ImmutableMap.of())))); + + // The Aggregate(FINAL) with both branches under LocalExchange + PlanMatchPattern aggregationPattern = aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project(exchange(aggregation(ImmutableMap.of(), PARTIAL, project(joinBranch1)))), + project(exchange(aggregation(ImmutableMap.of(), PARTIAL, project(joinBranch2)))))); + + // Query without ORDER BY to simplify plan (testing MV stitching, not sorting) + assertPlan("SELECT * FROM test_distinct_mv", + output( + exchange( + constrainedTableScan("__mv_storage__test_distinct_mv", + ImmutableMap.of( + "event_date", create(SortedRangeSet.copyOf(DATE, ImmutableList.of( + lessThan(DATE, jan2InDays), + greaterThan(DATE, jan2InDays))), false), + "reg_date", create(SortedRangeSet.copyOf(DATE, ImmutableList.of( + lessThan(DATE, jan2InDays), + greaterThan(DATE, jan2InDays))), false)), + ImmutableMap.of()), + aggregationPattern))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS test_distinct_mv"); + assertUpdate("DROP TABLE IF EXISTS test_distinct_t2"); + assertUpdate("DROP TABLE IF EXISTS test_distinct_t1"); + } + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where A (left side of EXCEPT) becomes stale. + * Verifies that the MV storage reads fresh partitions while the stale branch + * reads from base tables. The stale predicate on A may propagate to B via the + * join condition (a.dt = b.dt), and tables C/D may be pruned to Values if they + * have no data matching the stale partition. + */ + @Test + public void testJoinExceptJoinWithLeftSideStale() + { + try { + // Create 4 tables for (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE TABLE jexj_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO jexj_a VALUES (1, 10, '2024-01-01'), (2, 20, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexj_b VALUES (10, 'x', '2024-01-01'), (20, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jexj_c VALUES (1, 10, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexj_d VALUES (10, 'x', '2024-01-01')", 1); + + // Create MV: (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE MATERIALIZED VIEW mv_jexj " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexj_a a JOIN jexj_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexj_c c JOIN jexj_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexj"); + + // Make A stale by inserting new partition + assertUpdate("INSERT INTO jexj_a VALUES (3, 30, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexj_b VALUES (30, 'z', '2024-01-02')", 1); + + // The actual plan has: + // 1. MV storage scan with constraint excluding stale partition + // 2. First FilterProject branch: computes EXCEPT with A JOIN B and C JOIN D + // - Left side (A JOIN B) constrained to stale partition + // - Right side (C JOIN D) scans all rows (no constraint since C/D have data for '2024-01-01') + // 3. Second FilterProject branch: computes EXCEPT with only C JOIN D + // Join structure for A JOIN B: + PlanMatchPattern leftJoinBranch = join( + exchange(constrainedTableScan("jexj_a", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("jexj_b", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))); + + // C JOIN D scans the full table (data exists for other partitions) + PlanMatchPattern rightJoinBranch = join( + exchange(tableScan("jexj_c")), + exchange(exchange(tableScan("jexj_d")))); + + // First EXCEPT branch with both sides + PlanMatchPattern exceptBranch1 = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(leftJoinBranch)))), + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(rightJoinBranch)))))))); + + // Second EXCEPT branch with only right side (C JOIN D) + PlanMatchPattern exceptBranch2 = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(rightJoinBranch))))))); + + assertPlan("SELECT * FROM mv_jexj", + output( + exchange( + constrainedTableScan("__mv_storage__mv_jexj", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + exceptBranch1, + exceptBranch2))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_jexj"); + assertUpdate("DROP TABLE IF EXISTS jexj_d"); + assertUpdate("DROP TABLE IF EXISTS jexj_c"); + assertUpdate("DROP TABLE IF EXISTS jexj_b"); + assertUpdate("DROP TABLE IF EXISTS jexj_a"); + } + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where C (right side of EXCEPT) becomes stale. + * Verifies that the MV storage reads fresh partitions while the stale branch + * computes the EXCEPT for the stale partition. C gets the stale predicate, + * D may get it via join condition, and A/B tables scan the matching partition. + */ + @Test + public void testJoinExceptJoinWithRightSideStale() + { + try { + // Create 4 tables for (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE TABLE jexjr_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO jexjr_a VALUES (1, 10, '2024-01-01'), (2, 20, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjr_b VALUES (10, 'x', '2024-01-01'), (20, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjr_c VALUES (1, 10, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexjr_d VALUES (10, 'x', '2024-01-01')", 1); + + // Create MV: (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE MATERIALIZED VIEW mv_jexjr " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexjr_a a JOIN jexjr_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexjr_c c JOIN jexjr_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexjr"); + + // Make C stale (right side of EXCEPT) + assertUpdate("INSERT INTO jexjr_c VALUES (2, 20, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjr_d VALUES (20, 'y', '2024-01-02')", 1); + + // The actual plan has: + // 1. MV storage scan with constraint excluding stale partition + // 2. First FilterProject branch: Aggregate(FINAL) -> LocalExchange -> RemoteExchange -> Aggregate(PARTIAL) -> Project -> C JOIN D + // 3. Second FilterProject branch: Same structure as first + // Each branch has only ONE Aggregate(PARTIAL) child under LocalExchange + + // C JOIN D pattern + PlanMatchPattern rightJoinBranch = join( + exchange(tableScan("jexjr_c")), + exchange(exchange(tableScan("jexjr_d")))); + + // Both EXCEPT branches have the same structure: single Aggregate(PARTIAL) child + PlanMatchPattern exceptBranch = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(rightJoinBranch))))))); + + assertPlan("SELECT * FROM mv_jexjr", + output( + exchange( + constrainedTableScan("__mv_storage__mv_jexjr", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + greaterThan(VARCHAR, utf8Slice("2024-01-02")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + exceptBranch, + exceptBranch))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_jexjr"); + assertUpdate("DROP TABLE IF EXISTS jexjr_d"); + assertUpdate("DROP TABLE IF EXISTS jexjr_c"); + assertUpdate("DROP TABLE IF EXISTS jexjr_b"); + assertUpdate("DROP TABLE IF EXISTS jexjr_a"); + } + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where both A and C become stale. + * This should create two union branches (one for A's stale partition, one for C's). + */ + @Test + public void testJoinExceptJoinWithBothSidesStale() + { + try { + // Create 4 tables for (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE TABLE jexjb_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO jexjb_a VALUES (1, 10, '2024-01-01'), (2, 20, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjb_b VALUES (10, 'x', '2024-01-01'), (20, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjb_c VALUES (1, 10, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexjb_d VALUES (10, 'x', '2024-01-01')", 1); + + // Create MV: (A JOIN B) EXCEPT (C JOIN D) + assertUpdate("CREATE MATERIALIZED VIEW mv_jexjb " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexjb_a a JOIN jexjb_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexjb_c c JOIN jexjb_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexjb"); + + // Make both A and C stale with different partitions + assertUpdate("INSERT INTO jexjb_a VALUES (3, 30, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjb_b VALUES (30, 'z', '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjb_c VALUES (4, 40, '2024-01-03')", 1); + assertUpdate("INSERT INTO jexjb_d VALUES (40, 'w', '2024-01-03')", 1); + + // The actual plan has: + // 1. MV storage scan excluding both stale partitions + // 2. First FilterProject: A JOIN B (constrained to 2024-01-02) + C JOIN D (full scan) + // 3. Second FilterProject: only C JOIN D (full scan) + // The optimizer doesn't produce Values nodes - C and D are scanned fully + PlanMatchPattern joinABForBranch1 = join( + exchange(constrainedTableScan("jexjb_a", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())), + exchange(exchange(constrainedTableScan("jexjb_b", + ImmutableMap.of("dt", singleValue(VARCHAR, utf8Slice("2024-01-02"))), + ImmutableMap.of())))); + + // C JOIN D scans full table (data exists for '2024-01-01') + PlanMatchPattern joinCD = join( + exchange(tableScan("jexjb_c")), + exchange(exchange(tableScan("jexjb_d")))); + + // First branch with A JOIN B and C JOIN D + PlanMatchPattern exceptBranch1 = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(joinABForBranch1)))), + project(exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(joinCD)))))))); + + // Second branch with only C JOIN D + PlanMatchPattern exceptBranch2 = project(filter( + aggregation( + ImmutableMap.of(), + FINAL, + exchange( + exchange( + aggregation( + ImmutableMap.of(), + PARTIAL, + project(joinCD))))))); + + assertPlan("SELECT * FROM mv_jexjb", + output( + exchange( + constrainedTableScan("__mv_storage__mv_jexjb", + ImmutableMap.of("dt", create(SortedRangeSet.copyOf(VARCHAR, ImmutableList.of( + lessThan(VARCHAR, utf8Slice("2024-01-02")), + Range.range(VARCHAR, utf8Slice("2024-01-02"), false, utf8Slice("2024-01-03"), false), + greaterThan(VARCHAR, utf8Slice("2024-01-03")))), false)), + ImmutableMap.of("dt", "dt", "id", "id", "value", "value")), + exceptBranch1, + exceptBranch2))); + } + finally { + assertUpdate("DROP MATERIALIZED VIEW IF EXISTS mv_jexjb"); + assertUpdate("DROP TABLE IF EXISTS jexjb_d"); + assertUpdate("DROP TABLE IF EXISTS jexjb_c"); + assertUpdate("DROP TABLE IF EXISTS jexjb_b"); + assertUpdate("DROP TABLE IF EXISTS jexjb_a"); + } } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViews.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViews.java deleted file mode 100644 index 0f28075e33898..0000000000000 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViews.java +++ /dev/null @@ -1,1748 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.iceberg; - -import com.facebook.airlift.http.server.testing.TestingHttpServer; -import com.facebook.presto.Session; -import com.facebook.presto.testing.QueryRunner; -import com.facebook.presto.tests.AbstractTestQueryFramework; -import com.google.common.collect.ImmutableMap; -import org.assertj.core.util.Files; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.io.File; -import java.util.Optional; - -import static com.facebook.presto.iceberg.CatalogType.REST; -import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.getRestServer; -import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.restConnectorProperties; -import static com.google.common.io.MoreFiles.deleteRecursively; -import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; - -@Test(singleThreaded = true) -public class TestIcebergMaterializedViews - extends AbstractTestQueryFramework -{ - private File warehouseLocation; - private TestingHttpServer restServer; - private String serverUri; - - @BeforeClass - @Override - public void init() - throws Exception - { - warehouseLocation = Files.newTemporaryFolder(); - - restServer = getRestServer(warehouseLocation.getAbsolutePath()); - restServer.start(); - - serverUri = restServer.getBaseUrl().toString(); - super.init(); - } - - @AfterClass(alwaysRun = true) - public void tearDown() - throws Exception - { - if (restServer != null) { - restServer.stop(); - } - deleteRecursively(warehouseLocation.toPath(), ALLOW_INSECURE); - } - - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - return IcebergQueryRunner.builder() - .setCatalogType(REST) - .setExtraConnectorProperties(restConnectorProperties(serverUri)) - .setDataDirectory(Optional.of(warehouseLocation.toPath())) - .setSchemaName("test_schema") - .setCreateTpchTables(false) - .setExtraProperties(ImmutableMap.of("experimental.legacy-materialized-views", "false")) - .build().getQueryRunner(); - } - - @Test - public void testCreateMaterializedView() - { - assertUpdate("CREATE TABLE test_mv_base (id BIGINT, name VARCHAR, value BIGINT)"); - assertUpdate("INSERT INTO test_mv_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_simple AS SELECT id, name, value FROM test_mv_base"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_simple\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_mv_simple", "SELECT 3"); - assertQuery("SELECT * FROM test_mv_simple ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_simple"); - assertUpdate("DROP TABLE test_mv_base"); - } - - @Test - public void testCreateMaterializedViewWithFilter() - { - assertUpdate("CREATE TABLE test_mv_filtered_base (id BIGINT, status VARCHAR, amount BIGINT)"); - assertUpdate("INSERT INTO test_mv_filtered_base VALUES (1, 'active', 100), (2, 'inactive', 200), (3, 'active', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_filtered AS SELECT id, amount FROM test_mv_filtered_base WHERE status = 'active'"); - - assertQuery("SELECT COUNT(*) FROM test_mv_filtered", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_filtered ORDER BY id", - "VALUES (1, 100), (3, 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_filtered"); - assertUpdate("DROP TABLE test_mv_filtered_base"); - } - - @Test - public void testCreateMaterializedViewWithAggregation() - { - assertUpdate("CREATE TABLE test_mv_sales (product_id BIGINT, category VARCHAR, revenue BIGINT)"); - assertUpdate("INSERT INTO test_mv_sales VALUES (1, 'Electronics', 1000), (2, 'Electronics', 1500), (3, 'Books', 500), (4, 'Books', 300)", 4); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_category_sales AS " + - "SELECT category, COUNT(*) as product_count, SUM(revenue) as total_revenue " + - "FROM test_mv_sales GROUP BY category"); - - assertQuery("SELECT COUNT(*) FROM test_mv_category_sales", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_category_sales ORDER BY category", - "VALUES ('Books', 2, 800), ('Electronics', 2, 2500)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_category_sales"); - assertUpdate("DROP TABLE test_mv_sales"); - } - - @Test - public void testMaterializedViewStaleness() - { - assertUpdate("CREATE TABLE test_mv_stale_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_mv_stale_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_stale AS SELECT id, value FROM test_mv_stale_base"); - - assertQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_stale ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("INSERT INTO test_mv_stale_base VALUES (3, 300)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 3"); - assertQuery("SELECT * FROM test_mv_stale ORDER BY id", - "VALUES (1, 100), (2, 200), (3, 300)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_stale", 3); - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_stale\"", "SELECT 3"); - - assertUpdate("TRUNCATE TABLE test_mv_stale_base"); - assertQuery("SELECT COUNT(*) FROM test_mv_stale_base", "SELECT 0"); - assertQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 0"); - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_stale\"", "SELECT 3"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_stale"); - assertUpdate("DROP TABLE test_mv_stale_base"); - } - - @Test - public void testDropMaterializedView() - { - assertUpdate("CREATE TABLE test_mv_drop_base (id BIGINT, value VARCHAR)"); - assertUpdate("INSERT INTO test_mv_drop_base VALUES (1, 'test')", 1); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_drop AS SELECT id, value FROM test_mv_drop_base"); - - assertQuery("SELECT COUNT(*) FROM test_mv_drop", "SELECT 1"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_drop\"", "SELECT 0"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_drop"); - - assertQueryFails("SELECT * FROM \"__mv_storage__test_mv_drop\"", ".*does not exist.*"); - - assertQuery("SELECT COUNT(*) FROM test_mv_drop_base", "SELECT 1"); - - assertUpdate("DROP TABLE test_mv_drop_base"); - } - - @Test - public void testMaterializedViewMetadata() - { - assertUpdate("CREATE TABLE test_mv_metadata_base (id BIGINT, name VARCHAR)"); - assertUpdate("INSERT INTO test_mv_metadata_base VALUES (1, 'test')", 1); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_metadata AS SELECT id, name FROM test_mv_metadata_base WHERE id > 0"); - - assertQuery("SELECT table_name, table_type FROM information_schema.tables " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_mv_metadata'", - "VALUES ('test_mv_metadata', 'MATERIALIZED VIEW')"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_metadata"); - assertUpdate("DROP TABLE test_mv_metadata_base"); - } - - @DataProvider(name = "baseTableNames") - public Object[][] baseTableNamesProvider() - { - return new Object[][] { - {"tt1"}, - {"\"tt2\""}, - {"\"tt.3\""}, - {"\"tt,4.5\""}, - {"\"tt\"\"tt,123\"\".123\""} - }; - } - - @Test(dataProvider = "baseTableNames") - public void testMaterializedViewWithSpecialBaseTableName(String tableName) - { - assertUpdate("CREATE TABLE " + tableName + " (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO " + tableName + " VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_refresh AS SELECT id, value FROM " + tableName); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("INSERT INTO " + tableName + " VALUES (3, 300)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 3"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", - "VALUES (1, 100), (2, 200), (3, 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_refresh"); - assertUpdate("DROP TABLE " + tableName); - } - - @Test - public void testRefreshMaterializedView() - { - assertUpdate("CREATE TABLE test_mv_refresh_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_mv_refresh_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_refresh AS SELECT id, value FROM test_mv_refresh_base"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); - assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("INSERT INTO test_mv_refresh_base VALUES (3, 300)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 3"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", - "VALUES (1, 100), (2, 200), (3, 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_refresh"); - assertUpdate("DROP TABLE test_mv_refresh_base"); - } - - @Test - public void testRefreshMaterializedViewWithAggregation() - { - assertUpdate("CREATE TABLE test_mv_agg_refresh_base (category VARCHAR, value BIGINT)"); - assertUpdate("INSERT INTO test_mv_agg_refresh_base VALUES ('A', 10), ('B', 20), ('A', 15)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_agg_refresh AS " + - "SELECT category, SUM(value) as total FROM test_mv_agg_refresh_base GROUP BY category"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 0"); - - assertQuery("SELECT * FROM test_mv_agg_refresh ORDER BY category", - "VALUES ('A', 25), ('B', 20)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_agg_refresh", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 2"); - - assertUpdate("INSERT INTO test_mv_agg_refresh_base VALUES ('A', 5), ('C', 30)", 2); - - assertQuery("SELECT * FROM test_mv_agg_refresh ORDER BY category", - "VALUES ('A', 30), ('B', 20), ('C', 30)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_agg_refresh", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_agg_refresh\" ORDER BY category", - "VALUES ('A', 30), ('B', 20), ('C', 30)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_agg_refresh"); - assertUpdate("DROP TABLE test_mv_agg_refresh_base"); - } - - @Test - public void testPartitionedMaterializedViewWithStaleDataConstraints() - { - assertUpdate("CREATE TABLE test_mv_partitioned_base (" + - "id BIGINT, " + - "event_date DATE, " + - "value BIGINT) " + - "WITH (partitioning = ARRAY['event_date'])"); - - assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-01', 200), " + - "(3, DATE '2024-01-02', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_partitioned AS " + - "SELECT id, event_date, value FROM test_mv_partitioned_base"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_partitioned\"", "SELECT 0"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_partitioned", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_partitioned\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_partitioned\" ORDER BY id", - "VALUES (1, DATE '2024-01-01', 100), (2, DATE '2024-01-01', 200), (3, DATE '2024-01-02', 300)"); - - assertQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 3"); - assertQuery("SELECT * FROM test_mv_partitioned ORDER BY id", - "VALUES (1, DATE '2024-01-01', 100), (2, DATE '2024-01-01', 200), (3, DATE '2024-01-02', 300)"); - - assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + - "(4, DATE '2024-01-03', 400), " + - "(5, DATE '2024-01-03', 500)", 2); - - assertQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 5"); - assertQuery("SELECT * FROM test_mv_partitioned ORDER BY id", - "VALUES (1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-01', 200), " + - "(3, DATE '2024-01-02', 300), " + - "(4, DATE '2024-01-03', 400), " + - "(5, DATE '2024-01-03', 500)"); - - assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + - "(6, DATE '2024-01-04', 600)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 6"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_partitioned"); - assertUpdate("DROP TABLE test_mv_partitioned_base"); - } - - @Test - public void testMinimalRefresh() - { - assertUpdate("CREATE TABLE minimal_table (id BIGINT)"); - assertUpdate("INSERT INTO minimal_table VALUES (1)", 1); - assertUpdate("CREATE MATERIALIZED VIEW minimal_mv AS SELECT id FROM minimal_table"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__minimal_mv\"", "SELECT 0"); - - try { - assertUpdate("REFRESH MATERIALIZED VIEW minimal_mv", 1); - } - catch (Exception e) { - System.err.println("REFRESH failed with: " + e.getMessage()); - } - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__minimal_mv\"", "SELECT 1"); - assertQuery("SELECT * FROM \"__mv_storage__minimal_mv\"", "SELECT 1"); - - assertUpdate("DROP MATERIALIZED VIEW minimal_mv"); - assertUpdate("DROP TABLE minimal_table"); - } - - @Test - public void testJoinMaterializedViewLifecycle() - { - assertUpdate("CREATE TABLE test_mv_orders (order_id BIGINT, customer_id BIGINT, amount BIGINT)"); - assertUpdate("CREATE TABLE test_mv_customers (customer_id BIGINT, customer_name VARCHAR)"); - - assertUpdate("INSERT INTO test_mv_orders VALUES (1, 100, 50), (2, 200, 75), (3, 100, 25)", 3); - assertUpdate("INSERT INTO test_mv_customers VALUES (100, 'Alice'), (200, 'Bob')", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_order_details AS " + - "SELECT o.order_id, c.customer_name, o.amount " + - "FROM test_mv_orders o JOIN test_mv_customers c ON o.customer_id = c.customer_id"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 3"); - assertQuery("SELECT * FROM test_mv_order_details ORDER BY order_id", - "VALUES (1, 'Alice', 50), (2, 'Bob', 75), (3, 'Alice', 25)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_order_details", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_order_details\" ORDER BY order_id", - "VALUES (1, 'Alice', 50), (2, 'Bob', 75), (3, 'Alice', 25)"); - - assertQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 3"); - - assertUpdate("INSERT INTO test_mv_orders VALUES (4, 200, 100)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 4"); - assertQuery("SELECT * FROM test_mv_order_details ORDER BY order_id", - "VALUES (1, 'Alice', 50), (2, 'Bob', 75), (3, 'Alice', 25), (4, 'Bob', 100)"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 3"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_order_details", 4); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 4"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_order_details\" ORDER BY order_id", - "VALUES (1, 'Alice', 50), (2, 'Bob', 75), (3, 'Alice', 25), (4, 'Bob', 100)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_order_details"); - assertUpdate("DROP TABLE test_mv_customers"); - assertUpdate("DROP TABLE test_mv_orders"); - } - - @Test - public void testPartitionedJoinMaterializedView() - { - assertUpdate("CREATE TABLE test_mv_part_orders (" + - "order_id BIGINT, " + - "customer_id BIGINT, " + - "order_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['order_date'])"); - - assertUpdate("CREATE TABLE test_mv_part_customers (customer_id BIGINT, customer_name VARCHAR)"); - - assertUpdate("INSERT INTO test_mv_part_orders VALUES " + - "(1, 100, DATE '2024-01-01', 50), " + - "(2, 200, DATE '2024-01-01', 75), " + - "(3, 100, DATE '2024-01-02', 25)", 3); - assertUpdate("INSERT INTO test_mv_part_customers VALUES (100, 'Alice'), (200, 'Bob')", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_part_join AS " + - "SELECT o.order_id, c.customer_name, o.order_date, o.amount " + - "FROM test_mv_part_orders o JOIN test_mv_part_customers c ON o.customer_id = c.customer_id"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 0"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_part_join", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_mv_part_join\" ORDER BY order_id", - "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + - "(2, 'Bob', DATE '2024-01-01', 75), " + - "(3, 'Alice', DATE '2024-01-02', 25)"); - - assertQuery("SELECT COUNT(*) FROM test_mv_part_join", "SELECT 3"); - assertQuery("SELECT * FROM test_mv_part_join ORDER BY order_id", - "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + - "(2, 'Bob', DATE '2024-01-01', 75), " + - "(3, 'Alice', DATE '2024-01-02', 25)"); - - assertUpdate("INSERT INTO test_mv_part_orders VALUES (4, 200, DATE '2024-01-03', 100)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_part_join", "SELECT 4"); - assertQuery("SELECT * FROM test_mv_part_join ORDER BY order_id", - "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + - "(2, 'Bob', DATE '2024-01-01', 75), " + - "(3, 'Alice', DATE '2024-01-02', 25), " + - "(4, 'Bob', DATE '2024-01-03', 100)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_part_join", 4); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 4"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_part_join"); - assertUpdate("DROP TABLE test_mv_part_customers"); - assertUpdate("DROP TABLE test_mv_part_orders"); - } - - @Test - public void testMultiTableStaleness_TwoTablesBothStale() - { - assertUpdate("CREATE TABLE test_mv_orders (" + - "order_id BIGINT, " + - "order_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['order_date'])"); - - assertUpdate("CREATE TABLE test_mv_customers (" + - "customer_id BIGINT, " + - "reg_date DATE, " + - "name VARCHAR) " + - "WITH (partitioning = ARRAY['reg_date'])"); - - assertUpdate("INSERT INTO test_mv_orders VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200)", 2); - assertUpdate("INSERT INTO test_mv_customers VALUES " + - "(1, DATE '2024-01-01', 'Alice'), " + - "(2, DATE '2024-01-02', 'Bob')", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_multi_stale AS " + - "SELECT o.order_id, c.name, o.order_date, c.reg_date, o.amount " + - "FROM test_mv_orders o JOIN test_mv_customers c ON o.order_id = c.customer_id"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_multi_stale", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_multi_stale\"", "SELECT 2"); - - assertQuery("SELECT COUNT(*) FROM test_mv_multi_stale", "SELECT 2"); - - assertUpdate("INSERT INTO test_mv_orders VALUES (3, DATE '2024-01-03', 300)", 1); - assertUpdate("INSERT INTO test_mv_customers VALUES (3, DATE '2024-01-03', 'Charlie')", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_multi_stale", "SELECT 3"); - assertQuery("SELECT order_id, name, order_date, reg_date, amount FROM test_mv_multi_stale ORDER BY order_id", - "VALUES (1, 'Alice', DATE '2024-01-01', DATE '2024-01-01', 100), " + - "(2, 'Bob', DATE '2024-01-02', DATE '2024-01-02', 200), " + - "(3, 'Charlie', DATE '2024-01-03', DATE '2024-01-03', 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_multi_stale"); - assertUpdate("DROP TABLE test_mv_customers"); - assertUpdate("DROP TABLE test_mv_orders"); - } - - @Test - public void testMultiTableStaleness_ThreeTablesWithTwoStale() - { - assertUpdate("CREATE TABLE test_mv_t1 (" + - "id BIGINT, " + - "date1 DATE, " + - "value1 BIGINT) " + - "WITH (partitioning = ARRAY['date1'])"); - - assertUpdate("CREATE TABLE test_mv_t2 (" + - "id BIGINT, " + - "date2 DATE, " + - "value2 BIGINT) " + - "WITH (partitioning = ARRAY['date2'])"); - - assertUpdate("CREATE TABLE test_mv_t3 (" + - "id BIGINT, " + - "date3 DATE, " + - "value3 BIGINT) " + - "WITH (partitioning = ARRAY['date3'])"); - - assertUpdate("INSERT INTO test_mv_t1 VALUES (1, DATE '2024-01-01', 100)", 1); - assertUpdate("INSERT INTO test_mv_t2 VALUES (1, DATE '2024-01-01', 200)", 1); - assertUpdate("INSERT INTO test_mv_t3 VALUES (1, DATE '2024-01-01', 300)", 1); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_three_tables AS " + - "SELECT t1.id, t1.date1, t2.date2, t3.date3, " + - " t1.value1, t2.value2, t3.value3 " + - "FROM test_mv_t1 t1 " + - "JOIN test_mv_t2 t2 ON t1.id = t2.id " + - "JOIN test_mv_t3 t3 ON t1.id = t3.id"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_three_tables", 1); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_three_tables\"", "SELECT 1"); - - assertUpdate("INSERT INTO test_mv_t1 VALUES (2, DATE '2024-01-02', 150)", 1); - assertUpdate("INSERT INTO test_mv_t2 VALUES (2, DATE '2024-01-01', 250)", 1); - assertUpdate("INSERT INTO test_mv_t3 VALUES (2, DATE '2024-01-02', 350)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_three_tables", "SELECT 2"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_three_tables"); - assertUpdate("DROP TABLE test_mv_t3"); - assertUpdate("DROP TABLE test_mv_t2"); - assertUpdate("DROP TABLE test_mv_t1"); - } - - @Test - public void testMultiTableStaleness_DifferentPartitionCounts() - { - assertUpdate("CREATE TABLE test_mv_table_a (" + - "id BIGINT, " + - "date_a DATE, " + - "value BIGINT) " + - "WITH (partitioning = ARRAY['date_a'])"); - - assertUpdate("CREATE TABLE test_mv_table_b (" + - "id BIGINT, " + - "date_b DATE, " + - "status VARCHAR) " + - "WITH (partitioning = ARRAY['date_b'])"); - - assertUpdate("INSERT INTO test_mv_table_a VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200)", 2); - assertUpdate("INSERT INTO test_mv_table_b VALUES " + - "(1, DATE '2024-01-01', 'active'), " + - "(2, DATE '2024-01-02', 'inactive')", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_diff_partitions AS " + - "SELECT a.id, a.date_a, b.date_b, a.value, b.status " + - "FROM test_mv_table_a a JOIN test_mv_table_b b ON a.id = b.id"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_diff_partitions", 2); - - assertUpdate("INSERT INTO test_mv_table_a VALUES " + - "(3, DATE '2024-01-03', 300), " + - "(4, DATE '2024-01-04', 400), " + - "(5, DATE '2024-01-05', 500)", 3); - - assertUpdate("INSERT INTO test_mv_table_b VALUES " + - "(3, DATE '2024-01-03', 'active'), " + - "(4, DATE '2024-01-04', 'active'), " + - "(5, DATE '2024-01-05', 'pending')", 3); - - assertQuery("SELECT COUNT(*) FROM test_mv_diff_partitions", "SELECT 5"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_diff_partitions"); - assertUpdate("DROP TABLE test_mv_table_b"); - assertUpdate("DROP TABLE test_mv_table_a"); - } - - @Test - public void testMultiTableStaleness_NonPartitionedAndPartitionedBothStale() - { - assertUpdate("CREATE TABLE test_mv_non_part (id BIGINT, category VARCHAR)"); - - assertUpdate("CREATE TABLE test_mv_part_sales (" + - "id BIGINT, " + - "sale_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['sale_date'])"); - - assertUpdate("INSERT INTO test_mv_non_part VALUES (1, 'Electronics'), (2, 'Books')", 2); - assertUpdate("INSERT INTO test_mv_part_sales VALUES " + - "(1, DATE '2024-01-01', 500), " + - "(2, DATE '2024-01-02', 300)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_mv_mixed_stale AS " + - "SELECT c.id, c.category, s.sale_date, s.amount " + - "FROM test_mv_non_part c JOIN test_mv_part_sales s ON c.id = s.id"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_mixed_stale\"", "SELECT 0"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_mv_mixed_stale", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_mixed_stale\"", "SELECT 2"); - assertQuery("SELECT id, category, sale_date, amount FROM \"__mv_storage__test_mv_mixed_stale\" ORDER BY id", - "VALUES (1, 'Electronics', DATE '2024-01-01', 500), (2, 'Books', DATE '2024-01-02', 300)"); - - assertUpdate("INSERT INTO test_mv_non_part VALUES (3, 'Toys')", 1); - assertUpdate("INSERT INTO test_mv_part_sales VALUES (3, DATE '2024-01-03', 700)", 1); - - assertQuery("SELECT COUNT(*) FROM test_mv_mixed_stale", "SELECT 3"); - assertQuery("SELECT id, category, sale_date, amount FROM test_mv_mixed_stale ORDER BY id", - "VALUES (1, 'Electronics', DATE '2024-01-01', 500), " + - "(2, 'Books', DATE '2024-01-02', 300), " + - "(3, 'Toys', DATE '2024-01-03', 700)"); - - assertUpdate("DROP MATERIALIZED VIEW test_mv_mixed_stale"); - assertUpdate("DROP TABLE test_mv_part_sales"); - assertUpdate("DROP TABLE test_mv_non_part"); - } - - @Test - public void testPartitionAlignment_MatchingColumns() - { - assertUpdate("CREATE TABLE test_pa_matching_base (" + - "id BIGINT, " + - "event_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['event_date'])"); - - assertUpdate("INSERT INTO test_pa_matching_base VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200), " + - "(3, DATE '2024-01-03', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_pa_matching_mv AS " + - "SELECT id, event_date, amount FROM test_pa_matching_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_pa_matching_mv", 3); - - assertUpdate("INSERT INTO test_pa_matching_base VALUES (4, DATE '2024-01-04', 400)", 1); - - assertQuery("SELECT COUNT(*) FROM test_pa_matching_mv", "SELECT 4"); - assertQuery("SELECT id, event_date, amount FROM test_pa_matching_mv ORDER BY id", - "VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200), " + - "(3, DATE '2024-01-03', 300), " + - "(4, DATE '2024-01-04', 400)"); - - assertUpdate("DROP MATERIALIZED VIEW test_pa_matching_mv"); - assertUpdate("DROP TABLE test_pa_matching_base"); - } - - @Test - public void testPartitionAlignment_MissingConstraintColumn() - { - assertUpdate("CREATE TABLE test_pa_missing_base (" + - "id BIGINT, " + - "event_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['event_date'])"); - - assertUpdate("INSERT INTO test_pa_missing_base VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200), " + - "(3, DATE '2024-01-03', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_pa_missing_mv AS " + - "SELECT id, amount FROM test_pa_missing_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_pa_missing_mv", 3); - - assertUpdate("INSERT INTO test_pa_missing_base VALUES (4, DATE '2024-01-04', 400)", 1); - - assertQuery("SELECT COUNT(*) FROM test_pa_missing_mv", "SELECT 4"); - assertQuery("SELECT id, amount FROM test_pa_missing_mv ORDER BY id", - "VALUES (1, 100), (2, 200), (3, 300), (4, 400)"); - - assertUpdate("DROP MATERIALIZED VIEW test_pa_missing_mv"); - assertUpdate("DROP TABLE test_pa_missing_base"); - } - - @Test - public void testPartitionAlignment_OverSpecifiedStorage() - { - assertUpdate("CREATE TABLE test_pa_over_table_a (" + - "id BIGINT, " + - "event_date DATE, " + - "amount BIGINT) " + - "WITH (partitioning = ARRAY['event_date'])"); - - assertUpdate("CREATE TABLE test_pa_over_table_b (" + - "customer_id BIGINT, " + - "region VARCHAR, " + - "name VARCHAR) " + - "WITH (partitioning = ARRAY['region'])"); - - assertUpdate("INSERT INTO test_pa_over_table_a VALUES " + - "(1, DATE '2024-01-01', 100), " + - "(2, DATE '2024-01-02', 200), " + - "(3, DATE '2024-01-03', 300)", 3); - - assertUpdate("INSERT INTO test_pa_over_table_b VALUES " + - "(1, 'US', 'Alice'), " + - "(2, 'US', 'Bob'), " + - "(3, 'UK', 'Charlie')", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_pa_over_mv AS " + - "SELECT a.id, a.event_date, a.amount, b.region, b.name " + - "FROM test_pa_over_table_a a " + - "JOIN test_pa_over_table_b b ON a.id = b.customer_id"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_pa_over_mv", 3); - - assertUpdate("INSERT INTO test_pa_over_table_a VALUES (1, DATE '2024-01-04', 150)", 1); - - assertQuery("SELECT COUNT(*) FROM test_pa_over_mv", "SELECT 4"); - assertQuery("SELECT id, event_date, amount, region, name FROM test_pa_over_mv ORDER BY id, event_date", - "VALUES " + - "(1, DATE '2024-01-01', 100, 'US', 'Alice'), " + - "(1, DATE '2024-01-04', 150, 'US', 'Alice'), " + - "(2, DATE '2024-01-02', 200, 'US', 'Bob'), " + - "(3, DATE '2024-01-03', 300, 'UK', 'Charlie')"); - - assertUpdate("DROP MATERIALIZED VIEW test_pa_over_mv"); - assertUpdate("DROP TABLE test_pa_over_table_b"); - assertUpdate("DROP TABLE test_pa_over_table_a"); - } - - @Test - public void testAggregationMV_MisalignedPartitioning() - { - // Bug: When GROUP BY column differs from partition column and multiple partitions - // are stale, the current implementation creates partial aggregates per partition - // and GROUP BY treats them as distinct rows instead of re-aggregating. - assertUpdate("CREATE TABLE test_agg_misaligned (" + - "id BIGINT, " + - "partition_col VARCHAR, " + - "region VARCHAR, " + - "sales BIGINT) " + - "WITH (partitioning = ARRAY['partition_col'])"); - - assertUpdate("INSERT INTO test_agg_misaligned VALUES " + - "(1, 'A', 'US', 100), " + - "(2, 'A', 'EU', 50), " + - "(3, 'B', 'US', 200), " + - "(4, 'B', 'EU', 75)", 4); - - assertUpdate("CREATE MATERIALIZED VIEW test_agg_mv AS " + - "SELECT region, SUM(sales) as total_sales " + - "FROM test_agg_misaligned " + - "GROUP BY region"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_agg_mv", 2); - - assertQuery("SELECT * FROM test_agg_mv ORDER BY region", - "VALUES ('EU', 125), ('US', 300)"); - - assertUpdate("INSERT INTO test_agg_misaligned VALUES " + - "(5, 'A', 'US', 10), " + - "(6, 'B', 'US', 20)", 2); - - assertQuery("SELECT * FROM test_agg_mv ORDER BY region", - "VALUES ('EU', 125), ('US', 330)"); - - assertUpdate("DROP MATERIALIZED VIEW test_agg_mv"); - assertUpdate("DROP TABLE test_agg_misaligned"); - } - - @Test - public void testAggregationMV_MultiTableJoin_BothStale() - { - // Bug: When both tables are stale, creates partial aggregates for each branch - // which are treated as distinct rows instead of being re-aggregated. - assertUpdate("CREATE TABLE test_multi_orders (" + - "order_id BIGINT, " + - "product_id BIGINT, " + - "order_date DATE, " + - "quantity BIGINT) " + - "WITH (partitioning = ARRAY['order_date'])"); - - assertUpdate("CREATE TABLE test_multi_products (" + - "product_id BIGINT, " + - "product_category VARCHAR, " + - "price BIGINT) " + - "WITH (partitioning = ARRAY['product_category'])"); - - assertUpdate("INSERT INTO test_multi_orders VALUES " + - "(1, 100, DATE '2024-01-01', 5), " + - "(2, 200, DATE '2024-01-01', 3)", 2); - assertUpdate("INSERT INTO test_multi_products VALUES " + - "(100, 'Electronics', 50), " + - "(200, 'Books', 20)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_multi_agg_mv AS " + - "SELECT p.product_category, SUM(o.quantity * p.price) as total_revenue " + - "FROM test_multi_orders o " + - "JOIN test_multi_products p ON o.product_id = p.product_id " + - "GROUP BY p.product_category"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_multi_agg_mv", 2); - - assertQuery("SELECT * FROM test_multi_agg_mv ORDER BY product_category", - "VALUES ('Books', 60), ('Electronics', 250)"); - - assertUpdate("INSERT INTO test_multi_orders VALUES " + - "(3, 100, DATE '2024-01-02', 2), " + - "(4, 200, DATE '2024-01-02', 4)", 2); - - assertUpdate("INSERT INTO test_multi_products VALUES " + - "(300, 'Toys', 30)", 1); - - assertUpdate("INSERT INTO test_multi_orders VALUES " + - "(5, 300, DATE '2024-01-02', 1)", 1); - - String explainResult = (String) computeScalar("EXPLAIN SELECT * FROM test_multi_agg_mv ORDER BY product_category"); - System.out.println("=== EXPLAIN PLAN ==="); - System.out.println(explainResult); - System.out.println("==================="); - - assertQuery("SELECT * FROM test_multi_agg_mv ORDER BY product_category", - "VALUES ('Books', 140), ('Electronics', 350), ('Toys', 30)"); - - assertUpdate("DROP MATERIALIZED VIEW test_multi_agg_mv"); - assertUpdate("DROP TABLE test_multi_products"); - assertUpdate("DROP TABLE test_multi_orders"); - } - - @Test - public void testMaterializedViewWithCustomStorageTableName() - { - assertUpdate("CREATE TABLE test_custom_storage_base (id BIGINT, name VARCHAR, value BIGINT)"); - assertUpdate("INSERT INTO test_custom_storage_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_custom_storage_mv " + - "WITH (storage_table = 'my_custom_storage_table') " + - "AS SELECT id, name, value FROM test_custom_storage_base"); - - assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 0"); - - assertQueryFails("SELECT * FROM \"__mv_storage__test_custom_storage_mv\"", ".*does not exist.*"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_custom_storage_mv", 2); - - assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 2"); - assertQuery("SELECT * FROM my_custom_storage_table ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertQuery("SELECT * FROM test_custom_storage_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertUpdate("INSERT INTO test_custom_storage_base VALUES (3, 'Charlie', 300)", 1); - assertUpdate("REFRESH MATERIALIZED VIEW test_custom_storage_mv", 3); - - assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 3"); - assertQuery("SELECT * FROM my_custom_storage_table ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_custom_storage_mv"); - - assertQueryFails("SELECT * FROM my_custom_storage_table", ".*does not exist.*"); - - assertUpdate("DROP TABLE test_custom_storage_base"); - } - - @Test - public void testMaterializedViewWithCustomStorageSchema() - { - assertUpdate("CREATE SCHEMA IF NOT EXISTS test_storage_schema"); - - assertUpdate("CREATE TABLE test_custom_schema_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_custom_schema_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_custom_schema_mv " + - "WITH (storage_schema = 'test_storage_schema', " + - "storage_table = 'storage_table') " + - "AS SELECT id, value FROM test_schema.test_custom_schema_base"); - - assertQuery("SELECT COUNT(*) FROM test_storage_schema.storage_table", "SELECT 0"); - - assertQueryFails("SELECT * FROM test_schema.storage_table", ".*does not exist.*"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_schema.test_custom_schema_mv", 2); - - assertQuery("SELECT COUNT(*) FROM test_storage_schema.storage_table", "SELECT 2"); - assertQuery("SELECT * FROM test_storage_schema.storage_table ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertQuery("SELECT * FROM test_custom_schema_mv ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertUpdate("DROP MATERIALIZED VIEW test_schema.test_custom_schema_mv"); - assertQueryFails("SELECT * FROM test_storage_schema.storage_table", ".*does not exist.*"); - - assertUpdate("DROP TABLE test_custom_schema_base"); - assertUpdate("DROP SCHEMA test_storage_schema"); - } - - @Test - public void testMaterializedViewWithCustomPrefix() - { - assertUpdate("CREATE TABLE test_custom_prefix_base (id BIGINT, name VARCHAR)"); - assertUpdate("INSERT INTO test_custom_prefix_base VALUES (1, 'test')", 1); - - Session sessionWithCustomPrefix = Session.builder(getSession()) - .setCatalogSessionProperty("iceberg", "materialized_view_storage_prefix", "custom_prefix_") - .build(); - - assertUpdate(sessionWithCustomPrefix, "CREATE MATERIALIZED VIEW test_custom_prefix_mv " + - "AS SELECT id, name FROM test_custom_prefix_base"); - - assertQuery("SELECT COUNT(*) FROM custom_prefix_test_custom_prefix_mv", "SELECT 0"); - - assertQueryFails("SELECT * FROM \"__mv_storage__test_custom_prefix_mv\"", ".*does not exist.*"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_custom_prefix_mv", 1); - - assertQuery("SELECT COUNT(*) FROM custom_prefix_test_custom_prefix_mv", "SELECT 1"); - assertQuery("SELECT * FROM custom_prefix_test_custom_prefix_mv", "VALUES (1, 'test')"); - - assertQuery("SELECT * FROM test_custom_prefix_mv", "VALUES (1, 'test')"); - - assertUpdate("DROP MATERIALIZED VIEW test_custom_prefix_mv"); - assertQueryFails("SELECT * FROM custom_prefix_test_custom_prefix_mv", ".*does not exist.*"); - - assertUpdate("DROP TABLE test_custom_prefix_base"); - } - - @Test - public void testMaterializedViewWithValuesOnly() - { - assertUpdate("CREATE MATERIALIZED VIEW test_values_mv AS SELECT * FROM (VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)) AS t(id, name, value)"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_values_mv\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_values_mv", "SELECT 3"); - assertQuery("SELECT * FROM test_values_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_values_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_values_mv\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_values_mv\" ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertQuery("SELECT * FROM test_values_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("DROP MATERIALIZED VIEW test_values_mv"); - assertQueryFails("SELECT * FROM \"__mv_storage__test_values_mv\"", ".*does not exist.*"); - } - - @Test - public void testMaterializedViewWithBaseTableButNoColumnsSelected() - { - assertUpdate("CREATE TABLE test_no_cols_base (id BIGINT, name VARCHAR, value BIGINT)"); - assertUpdate("INSERT INTO test_no_cols_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_no_cols_mv AS " + - "SELECT 'constant' as label, 42 as fixed_value FROM test_no_cols_base"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_no_cols_mv", "SELECT 3"); - assertQuery("SELECT * FROM test_no_cols_mv", - "VALUES ('constant', 42), ('constant', 42), ('constant', 42)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_no_cols_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_no_cols_mv\"", - "VALUES ('constant', 42), ('constant', 42), ('constant', 42)"); - - assertUpdate("INSERT INTO test_no_cols_base VALUES (4, 'Dave', 400)", 1); - - assertQuery("SELECT COUNT(*) FROM test_no_cols_mv", "SELECT 4"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_no_cols_mv", 4); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 4"); - - assertUpdate("DROP MATERIALIZED VIEW test_no_cols_mv"); - assertQueryFails("SELECT * FROM \"__mv_storage__test_no_cols_mv\"", ".*does not exist.*"); - - assertUpdate("DROP TABLE test_no_cols_base"); - } - - @Test - public void testMaterializedViewOnEmptyBaseTable() - { - assertUpdate("CREATE TABLE test_empty_base (id BIGINT, name VARCHAR, value BIGINT)"); - - assertUpdate("CREATE MATERIALIZED VIEW test_empty_mv AS SELECT id, name, value FROM test_empty_base"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 0"); - - assertQuery("SELECT COUNT(*) FROM test_empty_mv", "SELECT 0"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_empty_mv", 0); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 0"); - - assertUpdate("INSERT INTO test_empty_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); - - assertQuery("SELECT COUNT(*) FROM test_empty_mv", "SELECT 2"); - assertQuery("SELECT * FROM test_empty_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_empty_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_empty_mv\" ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertUpdate("DROP MATERIALIZED VIEW test_empty_mv"); - assertQueryFails("SELECT * FROM \"__mv_storage__test_empty_mv\"", ".*does not exist.*"); - - assertUpdate("DROP TABLE test_empty_base"); - } - - @Test - public void testRefreshFailurePreservesOldData() - { - assertUpdate("CREATE TABLE test_refresh_failure_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_refresh_failure_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_refresh_failure_mv AS " + - "SELECT id, value FROM test_refresh_failure_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_refresh_failure_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_refresh_failure_mv\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_refresh_failure_mv\" ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertUpdate("DROP TABLE test_refresh_failure_base"); - - try { - getQueryRunner().execute("REFRESH MATERIALIZED VIEW test_refresh_failure_mv"); - throw new AssertionError("Expected REFRESH to fail when base table doesn't exist"); - } - catch (Exception e) { - if (!e.getMessage().contains("does not exist") && !e.getMessage().contains("not found")) { - throw new AssertionError("Expected 'does not exist' error, got: " + e.getMessage()); - } - } - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_refresh_failure_mv\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_refresh_failure_mv\" ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertUpdate("DROP MATERIALIZED VIEW test_refresh_failure_mv"); - } - - @Test - public void testBaseTableDroppedAndRecreated() - { - assertUpdate("CREATE TABLE test_recreate_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_recreate_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_recreate_mv AS SELECT id, value FROM test_recreate_base"); - assertUpdate("REFRESH MATERIALIZED VIEW test_recreate_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_recreate_mv\" ORDER BY id", - "VALUES (1, 100), (2, 200)"); - - assertUpdate("DROP TABLE test_recreate_base"); - - assertUpdate("CREATE TABLE test_recreate_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_recreate_base VALUES (3, 300), (4, 400), (5, 500)", 3); - - assertQuery("SELECT COUNT(*) FROM test_recreate_mv", "SELECT 3"); - assertQuery("SELECT * FROM test_recreate_mv ORDER BY id", - "VALUES (3, 300), (4, 400), (5, 500)"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 2"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_recreate_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_recreate_mv\" ORDER BY id", - "VALUES (3, 300), (4, 400), (5, 500)"); - - assertUpdate("DROP MATERIALIZED VIEW test_recreate_mv"); - assertUpdate("DROP TABLE test_recreate_base"); - } - - @Test - public void testStorageTableDroppedDirectly() - { - assertUpdate("CREATE TABLE test_storage_drop_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_storage_drop_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_storage_drop_mv AS SELECT id, value FROM test_storage_drop_base"); - assertUpdate("REFRESH MATERIALIZED VIEW test_storage_drop_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_storage_drop_mv\"", "SELECT 2"); - - assertUpdate("DROP TABLE \"__mv_storage__test_storage_drop_mv\""); - - assertQueryFails("SELECT * FROM \"__mv_storage__test_storage_drop_mv\"", ".*does not exist.*"); - - assertQueryFails("SELECT * FROM test_storage_drop_mv", ".*does not exist.*"); - - assertUpdate("DROP MATERIALIZED VIEW test_storage_drop_mv"); - assertUpdate("DROP TABLE test_storage_drop_base"); - } - - @Test - public void testMaterializedViewWithRenamedColumns() - { - assertUpdate("CREATE TABLE test_renamed_base (id BIGINT, original_name VARCHAR, original_value BIGINT)"); - assertUpdate("INSERT INTO test_renamed_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_renamed_mv AS " + - "SELECT id AS person_id, original_name AS full_name, original_value AS amount " + - "FROM test_renamed_base"); - - assertQuery("SELECT COUNT(*) FROM test_renamed_mv", "SELECT 3"); - assertQuery("SELECT * FROM test_renamed_mv ORDER BY person_id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_renamed_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_renamed_mv\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_renamed_mv\" ORDER BY person_id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertQuery("SELECT * FROM test_renamed_mv ORDER BY person_id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertQuery("SELECT person_id, full_name FROM test_renamed_mv WHERE amount > 150 ORDER BY person_id", - "VALUES (2, 'Bob'), (3, 'Charlie')"); - - assertUpdate("INSERT INTO test_renamed_base VALUES (4, 'Dave', 400)", 1); - - assertQuery("SELECT COUNT(*) FROM test_renamed_mv", "SELECT 4"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_renamed_mv", 4); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_renamed_mv\"", "SELECT 4"); - assertQuery("SELECT * FROM \"__mv_storage__test_renamed_mv\" ORDER BY person_id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300), (4, 'Dave', 400)"); - - assertUpdate("DROP MATERIALIZED VIEW test_renamed_mv"); - assertUpdate("DROP TABLE test_renamed_base"); - } - - @Test - public void testMaterializedViewWithComputedColumns() - { - assertUpdate("CREATE TABLE test_computed_base (id BIGINT, quantity BIGINT, unit_price BIGINT)"); - assertUpdate("INSERT INTO test_computed_base VALUES (1, 5, 100), (2, 10, 50), (3, 3, 200)", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_computed_mv AS " + - "SELECT id, " + - "quantity, " + - "unit_price, " + - "quantity * unit_price AS total_price, " + - "quantity * 2 AS double_quantity, " + - "'Order_' || CAST(id AS VARCHAR) AS order_label " + - "FROM test_computed_base"); - - assertQuery("SELECT COUNT(*) FROM test_computed_mv", "SELECT 3"); - assertQuery("SELECT id, quantity, unit_price, total_price, double_quantity, order_label FROM test_computed_mv ORDER BY id", - "VALUES (1, 5, 100, 500, 10, 'Order_1'), " + - "(2, 10, 50, 500, 20, 'Order_2'), " + - "(3, 3, 200, 600, 6, 'Order_3')"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_computed_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_computed_mv\"", "SELECT 3"); - assertQuery("SELECT id, quantity, unit_price, total_price, double_quantity, order_label FROM \"__mv_storage__test_computed_mv\" ORDER BY id", - "VALUES (1, 5, 100, 500, 10, 'Order_1'), " + - "(2, 10, 50, 500, 20, 'Order_2'), " + - "(3, 3, 200, 600, 6, 'Order_3')"); - - assertQuery("SELECT * FROM test_computed_mv WHERE total_price > 550 ORDER BY id", - "VALUES (3, 3, 200, 600, 6, 'Order_3')"); - - assertQuery("SELECT id, order_label FROM test_computed_mv WHERE double_quantity >= 10 ORDER BY id", - "VALUES (1, 'Order_1'), (2, 'Order_2')"); - - assertUpdate("INSERT INTO test_computed_base VALUES (4, 8, 75)", 1); - - assertQuery("SELECT COUNT(*) FROM test_computed_mv", "SELECT 4"); - assertQuery("SELECT id, total_price, order_label FROM test_computed_mv WHERE id = 4", - "VALUES (4, 600, 'Order_4')"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_computed_mv", 4); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_computed_mv\"", "SELECT 4"); - assertQuery("SELECT id, quantity, unit_price, total_price, order_label FROM \"__mv_storage__test_computed_mv\" WHERE id = 4", - "VALUES (4, 8, 75, 600, 'Order_4')"); - - assertUpdate("DROP MATERIALIZED VIEW test_computed_mv"); - assertUpdate("DROP TABLE test_computed_base"); - } - - @Test - public void testMaterializedViewWithCustomTableProperties() - { - assertUpdate("CREATE TABLE test_custom_props_base (id BIGINT, name VARCHAR, region VARCHAR)"); - assertUpdate("INSERT INTO test_custom_props_base VALUES (1, 'Alice', 'US'), (2, 'Bob', 'EU'), (3, 'Charlie', 'APAC')", 3); - - assertUpdate("CREATE MATERIALIZED VIEW test_custom_props_mv " + - "WITH (" + - " partitioning = ARRAY['region'], " + - " sorted_by = ARRAY['id'], " + - " \"write.format.default\" = 'ORC'" + - ") AS " + - "SELECT id, name, region FROM test_custom_props_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_custom_props_mv", 3); - - assertQuery("SELECT COUNT(*) FROM test_custom_props_mv", "SELECT 3"); - assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'US'", "VALUES ('Alice')"); - assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'EU'", "VALUES ('Bob')"); - - String storageTableName = "__mv_storage__test_custom_props_mv"; - assertQuery("SELECT COUNT(*) FROM \"" + storageTableName + "\"", "SELECT 3"); - - assertQuery("SELECT COUNT(*) FROM \"" + storageTableName + "\" WHERE region = 'APAC'", "SELECT 1"); - - assertUpdate("INSERT INTO test_custom_props_base VALUES (4, 'David', 'US')", 1); - assertUpdate("REFRESH MATERIALIZED VIEW test_custom_props_mv", 4); - - assertQuery("SELECT COUNT(*) FROM test_custom_props_mv WHERE region = 'US'", "SELECT 2"); - assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'US' ORDER BY id", - "VALUES ('Alice'), ('David')"); - - assertUpdate("DROP MATERIALIZED VIEW test_custom_props_mv"); - assertUpdate("DROP TABLE test_custom_props_base"); - } - - @Test - public void testMaterializedViewWithNestedTypes() - { - assertUpdate("CREATE TABLE test_nested_base (" + - "id BIGINT, " + - "tags ARRAY(VARCHAR), " + - "properties MAP(VARCHAR, VARCHAR), " + - "address ROW(street VARCHAR, city VARCHAR, zipcode VARCHAR))"); - - assertUpdate("INSERT INTO test_nested_base VALUES " + - "(1, ARRAY['tag1', 'tag2'], MAP(ARRAY['key1', 'key2'], ARRAY['value1', 'value2']), ROW('123 Main St', 'NYC', '10001')), " + - "(2, ARRAY['tag3'], MAP(ARRAY['key3'], ARRAY['value3']), ROW('456 Oak Ave', 'LA', '90001'))", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_nested_mv AS " + - "SELECT id, tags, properties, address FROM test_nested_base"); - - assertQuery("SELECT COUNT(*) FROM test_nested_mv", "SELECT 2"); - assertQuery("SELECT id, cardinality(tags) FROM test_nested_mv ORDER BY id", - "VALUES (1, 2), (2, 1)"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_nested_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_nested_mv\"", "SELECT 2"); - - assertQuery("SELECT id, cardinality(tags), address.city FROM test_nested_mv ORDER BY id", - "VALUES (1, 2, 'NYC'), (2, 1, 'LA')"); - - assertQuery("SELECT id FROM test_nested_mv WHERE element_at(properties, 'key1') = 'value1'", - "VALUES (1)"); - - assertUpdate("INSERT INTO test_nested_base VALUES " + - "(3, ARRAY['tag4', 'tag5', 'tag6'], MAP(ARRAY['key4'], ARRAY['value4']), ROW('789 Elm St', 'Chicago', '60601'))", 1); - - assertQuery("SELECT COUNT(*) FROM test_nested_mv", "SELECT 3"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_nested_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_nested_mv\"", "SELECT 3"); - assertQuery("SELECT id, address.zipcode FROM test_nested_mv WHERE id = 3", - "VALUES (3, '60601')"); - - assertUpdate("DROP MATERIALIZED VIEW test_nested_mv"); - assertUpdate("DROP TABLE test_nested_base"); - } - - @Test - public void testMaterializedViewAfterColumnAdded() - { - assertUpdate("CREATE TABLE test_evolve_add_base (id BIGINT, name VARCHAR, value BIGINT)"); - assertUpdate("INSERT INTO test_evolve_add_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_evolve_add_mv AS " + - "SELECT id, name, value FROM test_evolve_add_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_add_mv", 2); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 2"); - assertQuery("SELECT * FROM test_evolve_add_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertUpdate("ALTER TABLE test_evolve_add_base ADD COLUMN region VARCHAR"); - - assertUpdate("INSERT INTO test_evolve_add_base VALUES (3, 'Charlie', 300, 'US')", 1); - - assertQuery("SELECT COUNT(*) FROM test_evolve_add_mv", "SELECT 3"); - assertQuery("SELECT * FROM test_evolve_add_mv ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 2"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_add_mv", 3); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 3"); - assertQuery("SELECT * FROM \"__mv_storage__test_evolve_add_mv\" ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertUpdate("CREATE MATERIALIZED VIEW test_evolve_add_mv2 AS " + - "SELECT id, name, value, region FROM test_evolve_add_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_add_mv2", 3); - - assertQuery("SELECT * FROM test_evolve_add_mv2 WHERE id = 3", - "VALUES (3, 'Charlie', 300, 'US')"); - assertQuery("SELECT id, region FROM test_evolve_add_mv2 WHERE id IN (1, 2) ORDER BY id", - "VALUES (1, NULL), (2, NULL)"); - - assertUpdate("DROP MATERIALIZED VIEW test_evolve_add_mv"); - assertUpdate("DROP MATERIALIZED VIEW test_evolve_add_mv2"); - assertUpdate("DROP TABLE test_evolve_add_base"); - } - - @Test - public void testMaterializedViewAfterColumnDropped() - { - assertUpdate("CREATE TABLE test_evolve_drop_base (id BIGINT, name VARCHAR, value BIGINT, status VARCHAR)"); - assertUpdate("INSERT INTO test_evolve_drop_base VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_evolve_drop_mv_all AS " + - "SELECT id, name, value, status FROM test_evolve_drop_base"); - - assertUpdate("CREATE MATERIALIZED VIEW test_evolve_drop_mv_subset AS " + - "SELECT id, name, value FROM test_evolve_drop_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_drop_mv_all", 2); - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_drop_mv_subset", 2); - - assertQuery("SELECT * FROM test_evolve_drop_mv_all ORDER BY id", - "VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')"); - assertQuery("SELECT * FROM test_evolve_drop_mv_subset ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); - - assertUpdate("ALTER TABLE test_evolve_drop_base DROP COLUMN status"); - - assertUpdate("INSERT INTO test_evolve_drop_base VALUES (3, 'Charlie', 300)", 1); - - assertQuery("SELECT COUNT(*) FROM test_evolve_drop_mv_subset", "SELECT 3"); - assertQuery("SELECT * FROM test_evolve_drop_mv_subset ORDER BY id", - "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); - - assertQueryFails("SELECT * FROM test_evolve_drop_mv_all", - ".*Column 'status' cannot be resolved.*"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_drop_mv_all\"", "SELECT 2"); - assertQuery("SELECT * FROM \"__mv_storage__test_evolve_drop_mv_all\" ORDER BY id", - "VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_evolve_drop_mv_subset", 3); - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_drop_mv_subset\"", "SELECT 3"); - - assertUpdate("DROP MATERIALIZED VIEW test_evolve_drop_mv_all"); - assertUpdate("DROP MATERIALIZED VIEW test_evolve_drop_mv_subset"); - assertUpdate("DROP TABLE test_evolve_drop_base"); - } - - @Test - public void testDropNonExistentMaterializedView() - { - assertQueryFails("DROP MATERIALIZED VIEW non_existent_mv", - ".*does not exist.*"); - } - - @Test - public void testCreateMaterializedViewWithSameNameAsExistingTable() - { - assertUpdate("CREATE TABLE existing_table_name (id BIGINT, value VARCHAR)"); - assertUpdate("INSERT INTO existing_table_name VALUES (1, 'test')", 1); - - assertQueryFails("CREATE MATERIALIZED VIEW existing_table_name AS SELECT id, value FROM existing_table_name", - ".*already exists.*"); - - assertQuery("SELECT COUNT(*) FROM existing_table_name", "SELECT 1"); - assertQuery("SELECT * FROM existing_table_name", "VALUES (1, 'test')"); - - assertUpdate("CREATE TABLE test_mv_base (id BIGINT, name VARCHAR)"); - assertUpdate("INSERT INTO test_mv_base VALUES (2, 'foo')", 1); - - assertQueryFails("CREATE MATERIALIZED VIEW existing_table_name AS SELECT id, name FROM test_mv_base", - ".*already exists.*"); - - assertUpdate("DROP TABLE existing_table_name"); - assertUpdate("DROP TABLE test_mv_base"); - } - - @Test - public void testInformationSchemaMaterializedViews() - { - assertUpdate("CREATE TABLE test_is_mv_base1 (id BIGINT, name VARCHAR, value BIGINT)"); - assertUpdate("CREATE TABLE test_is_mv_base2 (category VARCHAR, amount BIGINT)"); - - assertUpdate("INSERT INTO test_is_mv_base1 VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); - assertUpdate("INSERT INTO test_is_mv_base2 VALUES ('A', 50), ('B', 75)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_is_mv1 AS SELECT id, name, value FROM test_is_mv_base1 WHERE id > 0"); - assertUpdate("CREATE MATERIALIZED VIEW test_is_mv2 AS SELECT category, SUM(amount) as total FROM test_is_mv_base2 GROUP BY category"); - - assertQuery( - "SELECT table_name FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_mv1', 'test_is_mv2') " + - "ORDER BY table_name", - "VALUES ('test_is_mv1'), ('test_is_mv2')"); - - assertQuery( - "SELECT table_catalog, table_schema, table_name, storage_schema, storage_table_name, base_tables " + - "FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1'", - "SELECT 'iceberg', 'test_schema', 'test_is_mv1', 'test_schema', '__mv_storage__test_is_mv1', 'iceberg.test_schema.test_is_mv_base1'"); - - assertQuery( - "SELECT COUNT(*) FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + - "AND view_definition IS NOT NULL AND length(view_definition) > 0", - "SELECT 1"); - - assertQuery( - "SELECT table_name FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv2'", - "VALUES ('test_is_mv2')"); - - assertQuery( - "SELECT COUNT(*) FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + - "AND view_owner IS NOT NULL", - "SELECT 1"); - - assertQuery( - "SELECT COUNT(*) FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + - "AND view_security IS NOT NULL", - "SELECT 1"); - - assertQuery( - "SELECT base_tables FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv2'", - "VALUES ('iceberg.test_schema.test_is_mv_base2')"); - - assertUpdate("DROP MATERIALIZED VIEW test_is_mv1"); - assertUpdate("DROP MATERIALIZED VIEW test_is_mv2"); - assertUpdate("DROP TABLE test_is_mv_base1"); - assertUpdate("DROP TABLE test_is_mv_base2"); - - assertQuery( - "SELECT COUNT(*) FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_mv1', 'test_is_mv2')", - "VALUES 0"); - } - - @Test - public void testInformationSchemaTablesWithMaterializedViews() - { - assertUpdate("CREATE TABLE test_is_tables_base (id BIGINT, name VARCHAR)"); - assertUpdate("CREATE VIEW test_is_tables_view AS SELECT id, name FROM test_is_tables_base"); - assertUpdate("CREATE MATERIALIZED VIEW test_is_tables_mv AS SELECT id, name FROM test_is_tables_base"); - - assertQuery( - "SELECT table_name, table_type FROM information_schema.tables " + - "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_tables_base', 'test_is_tables_view', 'test_is_tables_mv') " + - "ORDER BY table_name", - "VALUES ('test_is_tables_base', 'BASE TABLE'), ('test_is_tables_mv', 'MATERIALIZED VIEW'), ('test_is_tables_view', 'VIEW')"); - - assertQuery( - "SELECT table_name FROM information_schema.views " + - "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_tables_view', 'test_is_tables_mv') " + - "ORDER BY table_name", - "VALUES ('test_is_tables_view')"); - - assertUpdate("DROP MATERIALIZED VIEW test_is_tables_mv"); - assertUpdate("DROP VIEW test_is_tables_view"); - assertUpdate("DROP TABLE test_is_tables_base"); - } - - @Test - public void testInformationSchemaMaterializedViewsAfterRefresh() - { - assertUpdate("CREATE TABLE test_is_mv_refresh_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (1, 100), (2, 200)", 2); - assertUpdate("CREATE MATERIALIZED VIEW test_is_mv_refresh AS SELECT id, value FROM test_is_mv_refresh_base"); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'NOT_MATERIALIZED'"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_is_mv_refresh", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (3, 300)", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertUpdate("UPDATE test_is_mv_refresh_base SET value = 250 WHERE id = 2", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertUpdate("DELETE FROM test_is_mv_refresh_base WHERE id = 1", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_is_mv_refresh", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertUpdate("DROP MATERIALIZED VIEW test_is_mv_refresh"); - assertUpdate("DROP TABLE test_is_mv_refresh_base"); - - assertQuery( - "SELECT COUNT(*) FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", - "VALUES 0"); - } - - @Test - public void testStaleReadBehaviorFail() - { - assertUpdate("CREATE TABLE test_stale_fail_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_stale_fail_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_stale_fail " + - "WITH (stale_read_behavior = 'FAIL', staleness_window = '0s') " + - "AS SELECT id, value FROM test_stale_fail_base"); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", - "SELECT 'NOT_MATERIALIZED'"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_stale_fail", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_stale_fail", "SELECT 2"); - assertQuery("SELECT * FROM test_stale_fail ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("INSERT INTO test_stale_fail_base VALUES (3, 300)", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertQueryFails("SELECT * FROM test_stale_fail", - ".*Materialized view .* is stale.*"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_stale_fail", 3); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_stale_fail", "SELECT 3"); - - assertUpdate("DROP MATERIALIZED VIEW test_stale_fail"); - assertUpdate("DROP TABLE test_stale_fail_base"); - } - - @Test - public void testStaleReadBehaviorUseViewQuery() - { - assertUpdate("CREATE TABLE test_stale_use_query_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_stale_use_query_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_stale_use_query " + - "WITH (stale_read_behavior = 'USE_VIEW_QUERY', staleness_window = '0s') " + - "AS SELECT id, value FROM test_stale_use_query_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_stale_use_query", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_use_query'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_stale_use_query", "SELECT 2"); - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_stale_use_query\"", "SELECT 2"); - - assertUpdate("INSERT INTO test_stale_use_query_base VALUES (3, 300)", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_use_query'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_stale_use_query", "SELECT 3"); - assertQuery("SELECT * FROM test_stale_use_query ORDER BY id", - "VALUES (1, 100), (2, 200), (3, 300)"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_stale_use_query\"", "SELECT 2"); - - assertUpdate("DROP MATERIALIZED VIEW test_stale_use_query"); - assertUpdate("DROP TABLE test_stale_use_query_base"); - } - - @Test - public void testMaterializedViewWithNoStaleReadBehavior() - { - assertUpdate("CREATE TABLE test_no_stale_config_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_no_stale_config_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_no_stale_config AS SELECT id, value FROM test_no_stale_config_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_no_stale_config", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_no_stale_config'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_no_stale_config", "SELECT 2"); - - assertUpdate("INSERT INTO test_no_stale_config_base VALUES (3, 300)", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_no_stale_config'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_no_stale_config", "SELECT 3"); - - assertUpdate("DROP MATERIALIZED VIEW test_no_stale_config"); - assertUpdate("DROP TABLE test_no_stale_config_base"); - } - - @Test - public void testStalenessWindowAllowsStaleReads() - { - assertUpdate("CREATE TABLE test_staleness_window_base (id BIGINT, value BIGINT)"); - assertUpdate("INSERT INTO test_staleness_window_base VALUES (1, 100), (2, 200)", 2); - - assertUpdate("CREATE MATERIALIZED VIEW test_staleness_window_mv " + - "WITH (stale_read_behavior = 'FAIL', staleness_window = '1h') " + - "AS SELECT id, value FROM test_staleness_window_base"); - - assertUpdate("REFRESH MATERIALIZED VIEW test_staleness_window_mv", 2); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_staleness_window_mv'", - "SELECT 'FULLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_staleness_window_mv", "SELECT 2"); - assertQuery("SELECT * FROM test_staleness_window_mv ORDER BY id", "VALUES (1, 100), (2, 200)"); - - assertUpdate("INSERT INTO test_staleness_window_base VALUES (3, 300)", 1); - - assertQuery( - "SELECT freshness_state FROM information_schema.materialized_views " + - "WHERE table_schema = 'test_schema' AND table_name = 'test_staleness_window_mv'", - "SELECT 'PARTIALLY_MATERIALIZED'"); - - assertQuery("SELECT COUNT(*) FROM test_staleness_window_mv", "SELECT 2"); - - assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_staleness_window_mv\"", "SELECT 2"); - - assertUpdate("DROP MATERIALIZED VIEW test_staleness_window_mv"); - assertUpdate("DROP TABLE test_staleness_window_base"); - } -} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewsBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewsBase.java new file mode 100644 index 0000000000000..1c697c5d46430 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergMaterializedViewsBase.java @@ -0,0 +1,4858 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.spi.security.ViewExpression; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Optional; + +import static com.facebook.presto.spi.StandardWarningCode.MATERIALIZED_VIEW_STALE_DATA; +import static com.facebook.presto.tests.QueryAssertions.assertEqualsIgnoreOrder; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public abstract class TestIcebergMaterializedViewsBase + extends AbstractTestQueryFramework +{ + protected File warehouseLocation; + + @Test + public void testCreateMaterializedView() + { + assertUpdate("CREATE TABLE test_mv_base (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_mv_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_simple AS SELECT id, name, value FROM test_mv_base"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_simple\"", "SELECT 0"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_simple", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_mv_simple ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_simple"); + assertUpdate("DROP TABLE test_mv_base"); + } + + @Test + public void testCreateMaterializedViewWithFilter() + { + assertUpdate("CREATE TABLE test_mv_filtered_base (id BIGINT, status VARCHAR, amount BIGINT)"); + assertUpdate("INSERT INTO test_mv_filtered_base VALUES (1, 'active', 100), (2, 'inactive', 200), (3, 'active', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_filtered AS SELECT id, amount FROM test_mv_filtered_base WHERE status = 'active'"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_filtered", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_mv_filtered ORDER BY id", + "VALUES (1, 100), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_filtered"); + assertUpdate("DROP TABLE test_mv_filtered_base"); + } + + @Test + public void testCreateMaterializedViewWithAggregation() + { + assertUpdate("CREATE TABLE test_mv_sales (product_id BIGINT, category VARCHAR, revenue BIGINT)"); + assertUpdate("INSERT INTO test_mv_sales VALUES (1, 'Electronics', 1000), (2, 'Electronics', 1500), (3, 'Books', 500), (4, 'Books', 300)", 4); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_category_sales AS " + + "SELECT category, COUNT(*) as product_count, SUM(revenue) as total_revenue " + + "FROM test_mv_sales GROUP BY category"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_category_sales", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_mv_category_sales ORDER BY category", + "VALUES ('Books', 2, 800), ('Electronics', 2, 2500)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_category_sales"); + assertUpdate("DROP TABLE test_mv_sales"); + } + + @Test + public void testMaterializedViewStaleness() + { + assertUpdate("CREATE TABLE test_mv_stale_base (id BIGINT, value BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_mv_stale_base VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_stale " + + "AS SELECT id, value, dt FROM test_mv_stale_base"); + + assertQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_mv_stale ORDER BY id", + "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')"); + + assertUpdate("INSERT INTO test_mv_stale_base VALUES (3, 300, '2024-01-02')", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_mv_stale ORDER BY id", + "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01'), (3, 300, '2024-01-02')"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_mv_stale", 3); + assertRefreshAndFullyMaterialized("test_mv_stale", 3); + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_stale\"", "SELECT 3"); + + assertUpdate("TRUNCATE TABLE test_mv_stale_base"); + assertQuery("SELECT COUNT(*) FROM test_mv_stale_base", "SELECT 0"); + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_stale", "SELECT 0"); + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_stale\"", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_stale"); + assertUpdate("DROP TABLE test_mv_stale_base"); + } + + @Test + public void testDropMaterializedView() + { + assertUpdate("CREATE TABLE test_mv_drop_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO test_mv_drop_base VALUES (1, 'test')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_drop AS SELECT id, value FROM test_mv_drop_base"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_drop", "SELECT 1"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_drop\"", "SELECT 0"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_drop"); + + assertQueryFails("SELECT * FROM \"__mv_storage__test_mv_drop\"", ".*does not exist.*"); + + assertQuery("SELECT COUNT(*) FROM test_mv_drop_base", "SELECT 1"); + + assertUpdate("DROP TABLE test_mv_drop_base"); + } + + @Test + public void testMaterializedViewMetadata() + { + assertUpdate("CREATE TABLE test_mv_metadata_base (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_mv_metadata_base VALUES (1, 'test')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_metadata AS SELECT id, name FROM test_mv_metadata_base WHERE id > 0"); + + assertQuery("SELECT table_name, table_type FROM information_schema.tables " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_mv_metadata'", + "VALUES ('test_mv_metadata', 'MATERIALIZED VIEW')"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_metadata"); + assertUpdate("DROP TABLE test_mv_metadata_base"); + } + + @DataProvider(name = "baseTableNames") + public Object[][] baseTableNamesProvider() + { + return new Object[][] { + {"tt1"}, + {"\"tt2\""}, + {"\"tt.3\""}, + {"\"tt,4.5\""}, + {"\"tt\"\"tt,123\"\".123\""} + }; + } + + @Test(dataProvider = "baseTableNames") + public void testMaterializedViewWithSpecialBaseTableName(String tableName) + { + assertUpdate("CREATE TABLE " + tableName + " (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_refresh AS SELECT id, value FROM " + tableName); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 0"); + + assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); + assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); + assertQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); + + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 300)", 1); + + assertQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 3"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_mv_refresh", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_refresh"); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testRefreshMaterializedView() + { + assertUpdate("CREATE TABLE test_mv_refresh_base (id BIGINT, value BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_mv_refresh_base VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_refresh " + + "AS SELECT id, value, dt FROM test_mv_refresh_base"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 0"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')"); + + assertRefreshAndFullyMaterialized("test_mv_refresh", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", + "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_mv_refresh ORDER BY id", "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')"); + + assertUpdate("INSERT INTO test_mv_refresh_base VALUES (3, 300, '2024-01-02')", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_refresh", "SELECT 3"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 2"); + + assertRefreshAndFullyMaterialized("test_mv_refresh", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_refresh\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_refresh\" ORDER BY id", + "VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01'), (3, 300, '2024-01-02')"); + assertMaterializedViewResultsMatch("SELECT * FROM test_mv_refresh ORDER BY id"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_refresh"); + assertUpdate("DROP TABLE test_mv_refresh_base"); + } + + @Test + public void testRefreshMaterializedViewWithAggregation() + { + assertUpdate("CREATE TABLE test_mv_agg_refresh_base (category VARCHAR, value BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_mv_agg_refresh_base VALUES ('A', 10, '2024-01-01'), ('B', 20, '2024-01-01'), ('A', 15, '2024-01-01')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_agg_refresh AS " + + "SELECT category, SUM(value) as total, dt FROM test_mv_agg_refresh_base GROUP BY category, dt"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 0"); + + assertQuery("SELECT * FROM test_mv_agg_refresh ORDER BY category", + "VALUES ('A', 25, '2024-01-01'), ('B', 20, '2024-01-01')"); + + assertRefreshAndFullyMaterialized("test_mv_agg_refresh", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 2"); + assertMaterializedViewResultsMatch("SELECT * FROM test_mv_agg_refresh ORDER BY category"); + + assertUpdate("INSERT INTO test_mv_agg_refresh_base VALUES ('A', 5, '2024-01-02'), ('C', 30, '2024-01-02')", 2); + + assertMaterializedViewQuery("SELECT * FROM test_mv_agg_refresh ORDER BY category, dt", + "VALUES ('A', 25, '2024-01-01'), ('A', 5, '2024-01-02'), ('B', 20, '2024-01-01'), ('C', 30, '2024-01-02')"); + + assertRefreshAndFullyMaterialized("test_mv_agg_refresh", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_agg_refresh\"", "SELECT 4"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_agg_refresh\" ORDER BY category, dt", + "VALUES ('A', 25, '2024-01-01'), ('A', 5, '2024-01-02'), ('B', 20, '2024-01-01'), ('C', 30, '2024-01-02')"); + assertMaterializedViewResultsMatch("SELECT * FROM test_mv_agg_refresh ORDER BY category, dt"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_agg_refresh"); + assertUpdate("DROP TABLE test_mv_agg_refresh_base"); + } + + @Test + public void testPartitionedMaterializedViewWithStaleDataConstraints() + { + assertUpdate("CREATE TABLE test_mv_partitioned_base (" + + "id BIGINT, " + + "event_date DATE, " + + "value BIGINT) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-01', 200), " + + "(3, DATE '2024-01-02', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_partitioned AS " + + "SELECT id, event_date, value FROM test_mv_partitioned_base"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_partitioned\"", "SELECT 0"); + + assertRefreshAndFullyMaterialized("test_mv_partitioned", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_partitioned\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_partitioned\" ORDER BY id", + "VALUES (1, DATE '2024-01-01', 100), (2, DATE '2024-01-01', 200), (3, DATE '2024-01-02', 300)"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_mv_partitioned ORDER BY id", + "VALUES (1, DATE '2024-01-01', 100), (2, DATE '2024-01-01', 200), (3, DATE '2024-01-02', 300)"); + + assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + + "(4, DATE '2024-01-03', 400), " + + "(5, DATE '2024-01-03', 500)", 2); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 5"); + assertMaterializedViewQuery("SELECT * FROM test_mv_partitioned ORDER BY id", + "VALUES (1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-01', 200), " + + "(3, DATE '2024-01-02', 300), " + + "(4, DATE '2024-01-03', 400), " + + "(5, DATE '2024-01-03', 500)"); + + assertUpdate("INSERT INTO test_mv_partitioned_base VALUES " + + "(6, DATE '2024-01-04', 600)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_partitioned", "SELECT 6"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_partitioned"); + assertUpdate("DROP TABLE test_mv_partitioned_base"); + } + + @Test + public void testMinimalRefresh() + { + assertUpdate("CREATE TABLE minimal_table (id BIGINT, dt VARCHAR) WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO minimal_table VALUES (1, '2024-01-01')", 1); + assertUpdate("CREATE MATERIALIZED VIEW minimal_mv AS SELECT id, dt FROM minimal_table"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__minimal_mv\"", "SELECT 0"); + + assertRefreshAndFullyMaterialized("minimal_mv", 1); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__minimal_mv\"", "SELECT 1"); + assertQuery("SELECT * FROM \"__mv_storage__minimal_mv\"", "VALUES (1, '2024-01-01')"); + + assertUpdate("DROP MATERIALIZED VIEW minimal_mv"); + assertUpdate("DROP TABLE minimal_table"); + } + + @Test + public void testJoinMaterializedViewLifecycle() + { + assertUpdate("CREATE TABLE test_mv_orders (order_id BIGINT, customer_id BIGINT, amount BIGINT, order_date VARCHAR) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE test_mv_customers (customer_id BIGINT, customer_name VARCHAR)"); + + assertUpdate("INSERT INTO test_mv_orders VALUES (1, 100, 50, '2024-01-01'), (2, 200, 75, '2024-01-01'), (3, 100, 25, '2024-01-01')", 3); + assertUpdate("INSERT INTO test_mv_customers VALUES (100, 'Alice'), (200, 'Bob')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_order_details AS " + + "SELECT o.order_id, c.customer_name, o.amount, o.order_date " + + "FROM test_mv_orders o JOIN test_mv_customers c ON o.customer_id = c.customer_id"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 0"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_mv_order_details ORDER BY order_id", + "VALUES (1, 'Alice', 50, '2024-01-01'), (2, 'Bob', 75, '2024-01-01'), (3, 'Alice', 25, '2024-01-01')"); + + assertRefreshAndFullyMaterialized("test_mv_order_details", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_order_details\" ORDER BY order_id", + "VALUES (1, 'Alice', 50, '2024-01-01'), (2, 'Bob', 75, '2024-01-01'), (3, 'Alice', 25, '2024-01-01')"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 3"); + + assertUpdate("INSERT INTO test_mv_orders VALUES (4, 200, 100, '2024-01-02')", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_order_details", "SELECT 4"); + assertMaterializedViewQuery("SELECT * FROM test_mv_order_details ORDER BY order_id", + "VALUES (1, 'Alice', 50, '2024-01-01'), (2, 'Bob', 75, '2024-01-01'), (3, 'Alice', 25, '2024-01-01'), (4, 'Bob', 100, '2024-01-02')"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 3"); + + assertRefreshAndFullyMaterialized("test_mv_order_details", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_order_details\"", "SELECT 4"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_order_details\" ORDER BY order_id", + "VALUES (1, 'Alice', 50, '2024-01-01'), (2, 'Bob', 75, '2024-01-01'), (3, 'Alice', 25, '2024-01-01'), (4, 'Bob', 100, '2024-01-02')"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_order_details"); + assertUpdate("DROP TABLE test_mv_customers"); + assertUpdate("DROP TABLE test_mv_orders"); + } + + @Test + public void testPartitionedJoinMaterializedView() + { + assertUpdate("CREATE TABLE test_mv_part_orders (" + + "order_id BIGINT, " + + "customer_id BIGINT, " + + "order_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_mv_part_customers (customer_id BIGINT, customer_name VARCHAR)"); + + assertUpdate("INSERT INTO test_mv_part_orders VALUES " + + "(1, 100, DATE '2024-01-01', 50), " + + "(2, 200, DATE '2024-01-01', 75), " + + "(3, 100, DATE '2024-01-02', 25)", 3); + assertUpdate("INSERT INTO test_mv_part_customers VALUES (100, 'Alice'), (200, 'Bob')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_part_join AS " + + "SELECT o.order_id, c.customer_name, o.order_date, o.amount " + + "FROM test_mv_part_orders o JOIN test_mv_part_customers c ON o.customer_id = c.customer_id"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 0"); + + assertRefreshAndFullyMaterialized("test_mv_part_join", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_mv_part_join\" ORDER BY order_id", + "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + + "(2, 'Bob', DATE '2024-01-01', 75), " + + "(3, 'Alice', DATE '2024-01-02', 25)"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_part_join", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_mv_part_join ORDER BY order_id", + "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + + "(2, 'Bob', DATE '2024-01-01', 75), " + + "(3, 'Alice', DATE '2024-01-02', 25)"); + + assertUpdate("INSERT INTO test_mv_part_orders VALUES (4, 200, DATE '2024-01-03', 100)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_part_join", "SELECT 4"); + assertMaterializedViewQuery("SELECT * FROM test_mv_part_join ORDER BY order_id", + "VALUES (1, 'Alice', DATE '2024-01-01', 50), " + + "(2, 'Bob', DATE '2024-01-01', 75), " + + "(3, 'Alice', DATE '2024-01-02', 25), " + + "(4, 'Bob', DATE '2024-01-03', 100)"); + + assertRefreshAndFullyMaterialized("test_mv_part_join", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_part_join\"", "SELECT 4"); + assertMaterializedViewResultsMatch("SELECT * FROM test_mv_part_join ORDER BY order_id"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_part_join"); + assertUpdate("DROP TABLE test_mv_part_customers"); + assertUpdate("DROP TABLE test_mv_part_orders"); + } + + @Test + public void testMultiTableStaleness_TwoTablesBothStale() + { + assertUpdate("CREATE TABLE test_mv_orders (" + + "order_id BIGINT, " + + "order_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_mv_customers (" + + "customer_id BIGINT, " + + "reg_date DATE, " + + "name VARCHAR) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO test_mv_orders VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200)", 2); + assertUpdate("INSERT INTO test_mv_customers VALUES " + + "(1, DATE '2024-01-01', 'Alice'), " + + "(2, DATE '2024-01-02', 'Bob')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_multi_stale AS " + + "SELECT o.order_id, c.name, o.order_date, c.reg_date, o.amount " + + "FROM test_mv_orders o JOIN test_mv_customers c ON o.order_id = c.customer_id"); + + assertRefreshAndFullyMaterialized("test_mv_multi_stale", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_multi_stale\"", "SELECT 2"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_multi_stale", "SELECT 2"); + + assertUpdate("INSERT INTO test_mv_orders VALUES (3, DATE '2024-01-03', 300)", 1); + assertUpdate("INSERT INTO test_mv_customers VALUES (3, DATE '2024-01-03', 'Charlie')", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_multi_stale", "SELECT 3"); + assertMaterializedViewQuery("SELECT order_id, name, order_date, reg_date, amount FROM test_mv_multi_stale ORDER BY order_id", + "VALUES (1, 'Alice', DATE '2024-01-01', DATE '2024-01-01', 100), " + + "(2, 'Bob', DATE '2024-01-02', DATE '2024-01-02', 200), " + + "(3, 'Charlie', DATE '2024-01-03', DATE '2024-01-03', 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_multi_stale"); + assertUpdate("DROP TABLE test_mv_customers"); + assertUpdate("DROP TABLE test_mv_orders"); + } + + @Test + public void testMultiTableStaleness_ThreeTablesWithTwoStale() + { + assertUpdate("CREATE TABLE test_mv_t1 (" + + "id BIGINT, " + + "date1 DATE, " + + "value1 BIGINT) " + + "WITH (partitioning = ARRAY['date1'])"); + + assertUpdate("CREATE TABLE test_mv_t2 (" + + "id BIGINT, " + + "date2 DATE, " + + "value2 BIGINT) " + + "WITH (partitioning = ARRAY['date2'])"); + + assertUpdate("CREATE TABLE test_mv_t3 (" + + "id BIGINT, " + + "date3 DATE, " + + "value3 BIGINT) " + + "WITH (partitioning = ARRAY['date3'])"); + + assertUpdate("INSERT INTO test_mv_t1 VALUES (1, DATE '2024-01-01', 100)", 1); + assertUpdate("INSERT INTO test_mv_t2 VALUES (1, DATE '2024-01-01', 200)", 1); + assertUpdate("INSERT INTO test_mv_t3 VALUES (1, DATE '2024-01-01', 300)", 1); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_three_tables AS " + + "SELECT t1.id, t1.date1, t2.date2, t3.date3, " + + " t1.value1, t2.value2, t3.value3 " + + "FROM test_mv_t1 t1 " + + "JOIN test_mv_t2 t2 ON t1.id = t2.id " + + "JOIN test_mv_t3 t3 ON t1.id = t3.id"); + + assertRefreshAndFullyMaterialized("test_mv_three_tables", 1); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_three_tables\"", "SELECT 1"); + + assertUpdate("INSERT INTO test_mv_t1 VALUES (2, DATE '2024-01-02', 150)", 1); + assertUpdate("INSERT INTO test_mv_t2 VALUES (2, DATE '2024-01-01', 250)", 1); + assertUpdate("INSERT INTO test_mv_t3 VALUES (2, DATE '2024-01-02', 350)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_three_tables", "SELECT 2"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_three_tables"); + assertUpdate("DROP TABLE test_mv_t3"); + assertUpdate("DROP TABLE test_mv_t2"); + assertUpdate("DROP TABLE test_mv_t1"); + } + + @Test + public void testMultiTableStaleness_DifferentPartitionCounts() + { + assertUpdate("CREATE TABLE test_mv_table_a (" + + "id BIGINT, " + + "date_a DATE, " + + "value BIGINT) " + + "WITH (partitioning = ARRAY['date_a'])"); + + assertUpdate("CREATE TABLE test_mv_table_b (" + + "id BIGINT, " + + "date_b DATE, " + + "status VARCHAR) " + + "WITH (partitioning = ARRAY['date_b'])"); + + assertUpdate("INSERT INTO test_mv_table_a VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200)", 2); + assertUpdate("INSERT INTO test_mv_table_b VALUES " + + "(1, DATE '2024-01-01', 'active'), " + + "(2, DATE '2024-01-02', 'inactive')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_diff_partitions AS " + + "SELECT a.id, a.date_a, b.date_b, a.value, b.status " + + "FROM test_mv_table_a a JOIN test_mv_table_b b ON a.id = b.id"); + + assertRefreshAndFullyMaterialized("test_mv_diff_partitions", 2); + + assertUpdate("INSERT INTO test_mv_table_a VALUES " + + "(3, DATE '2024-01-03', 300), " + + "(4, DATE '2024-01-04', 400), " + + "(5, DATE '2024-01-05', 500)", 3); + + assertUpdate("INSERT INTO test_mv_table_b VALUES " + + "(3, DATE '2024-01-03', 'active'), " + + "(4, DATE '2024-01-04', 'active'), " + + "(5, DATE '2024-01-05', 'pending')", 3); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_diff_partitions", "SELECT 5"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_diff_partitions"); + assertUpdate("DROP TABLE test_mv_table_b"); + assertUpdate("DROP TABLE test_mv_table_a"); + } + + @Test + public void testMultiTableStaleness_NonPartitionedAndPartitionedBothStale() + { + assertUpdate("CREATE TABLE test_mv_non_part (id BIGINT, category VARCHAR, created_date DATE) " + + "WITH (partitioning = ARRAY['created_date'])"); + + assertUpdate("CREATE TABLE test_mv_part_sales (" + + "id BIGINT, " + + "sale_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['sale_date'])"); + + assertUpdate("INSERT INTO test_mv_non_part VALUES (1, 'Electronics', DATE '2024-01-01'), (2, 'Books', DATE '2024-01-02')", 2); + assertUpdate("INSERT INTO test_mv_part_sales VALUES " + + "(1, DATE '2024-01-01', 500), " + + "(2, DATE '2024-01-02', 300)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_mixed_stale AS " + + "SELECT c.id, c.category, s.sale_date, s.amount " + + "FROM test_mv_non_part c JOIN test_mv_part_sales s ON c.id = s.id"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_mixed_stale\"", "SELECT 0"); + + assertRefreshAndFullyMaterialized("test_mv_mixed_stale", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_mv_mixed_stale\"", "SELECT 2"); + assertQuery("SELECT id, category, sale_date, amount FROM \"__mv_storage__test_mv_mixed_stale\" ORDER BY id", + "VALUES (1, 'Electronics', DATE '2024-01-01', 500), (2, 'Books', DATE '2024-01-02', 300)"); + + assertUpdate("INSERT INTO test_mv_non_part VALUES (3, 'Toys', DATE '2024-01-03')", 1); + assertUpdate("INSERT INTO test_mv_part_sales VALUES (3, DATE '2024-01-03', 700)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_mv_mixed_stale", "SELECT 3"); + assertMaterializedViewQuery("SELECT id, category, sale_date, amount FROM test_mv_mixed_stale ORDER BY id", + "VALUES (1, 'Electronics', DATE '2024-01-01', 500), " + + "(2, 'Books', DATE '2024-01-02', 300), " + + "(3, 'Toys', DATE '2024-01-03', 700)"); + + assertUpdate("DROP MATERIALIZED VIEW test_mv_mixed_stale"); + assertUpdate("DROP TABLE test_mv_part_sales"); + assertUpdate("DROP TABLE test_mv_non_part"); + } + + @Test + public void testPartitionAlignment_MatchingColumns() + { + assertUpdate("CREATE TABLE test_pa_matching_base (" + + "id BIGINT, " + + "event_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_pa_matching_base VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200), " + + "(3, DATE '2024-01-03', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_pa_matching_mv AS " + + "SELECT id, event_date, amount FROM test_pa_matching_base"); + + assertRefreshAndFullyMaterialized("test_pa_matching_mv", 3); + + assertUpdate("INSERT INTO test_pa_matching_base VALUES (4, DATE '2024-01-04', 400)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_pa_matching_mv", "SELECT 4"); + assertMaterializedViewQuery("SELECT id, event_date, amount FROM test_pa_matching_mv ORDER BY id", + "VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200), " + + "(3, DATE '2024-01-03', 300), " + + "(4, DATE '2024-01-04', 400)"); + + assertUpdate("DROP MATERIALIZED VIEW test_pa_matching_mv"); + assertUpdate("DROP TABLE test_pa_matching_base"); + } + + @Test + public void testPartitionAlignment_MissingConstraintColumn() + { + assertUpdate("CREATE TABLE test_pa_missing_base (" + + "id BIGINT, " + + "event_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_pa_missing_base VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200), " + + "(3, DATE '2024-01-03', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_pa_missing_mv AS " + + "SELECT id, amount FROM test_pa_missing_base"); + + assertRefreshAndFullyMaterialized("test_pa_missing_mv", 3); + + assertUpdate("INSERT INTO test_pa_missing_base VALUES (4, DATE '2024-01-04', 400)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_pa_missing_mv", "SELECT 4"); + assertMaterializedViewQuery("SELECT id, amount FROM test_pa_missing_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300), (4, 400)"); + + assertUpdate("DROP MATERIALIZED VIEW test_pa_missing_mv"); + assertUpdate("DROP TABLE test_pa_missing_base"); + } + + @Test + public void testPartitionAlignment_OverSpecifiedStorage() + { + assertUpdate("CREATE TABLE test_pa_over_table_a (" + + "id BIGINT, " + + "event_date DATE, " + + "amount BIGINT) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("CREATE TABLE test_pa_over_table_b (" + + "customer_id BIGINT, " + + "region VARCHAR, " + + "name VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + + assertUpdate("INSERT INTO test_pa_over_table_a VALUES " + + "(1, DATE '2024-01-01', 100), " + + "(2, DATE '2024-01-02', 200), " + + "(3, DATE '2024-01-03', 300)", 3); + + assertUpdate("INSERT INTO test_pa_over_table_b VALUES " + + "(1, 'US', 'Alice'), " + + "(2, 'US', 'Bob'), " + + "(3, 'UK', 'Charlie')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_pa_over_mv AS " + + "SELECT a.id, a.event_date, a.amount, b.region, b.name " + + "FROM test_pa_over_table_a a " + + "JOIN test_pa_over_table_b b ON a.id = b.customer_id"); + + assertRefreshAndFullyMaterialized("test_pa_over_mv", 3); + + assertUpdate("INSERT INTO test_pa_over_table_a VALUES (1, DATE '2024-01-04', 150)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_pa_over_mv", "SELECT 4"); + assertMaterializedViewQuery("SELECT id, event_date, amount, region, name FROM test_pa_over_mv ORDER BY id, event_date", + "VALUES " + + "(1, DATE '2024-01-01', 100, 'US', 'Alice'), " + + "(1, DATE '2024-01-04', 150, 'US', 'Alice'), " + + "(2, DATE '2024-01-02', 200, 'US', 'Bob'), " + + "(3, DATE '2024-01-03', 300, 'UK', 'Charlie')"); + + assertUpdate("DROP MATERIALIZED VIEW test_pa_over_mv"); + assertUpdate("DROP TABLE test_pa_over_table_b"); + assertUpdate("DROP TABLE test_pa_over_table_a"); + } + + @Test + public void testAggregationMV_MisalignedPartitioning() + { + assertUpdate("CREATE TABLE test_agg_misaligned (" + + "id BIGINT, " + + "partition_col VARCHAR, " + + "region VARCHAR, " + + "sales BIGINT) " + + "WITH (partitioning = ARRAY['partition_col'])"); + + assertUpdate("INSERT INTO test_agg_misaligned VALUES " + + "(1, 'A', 'US', 100), " + + "(2, 'A', 'EU', 50), " + + "(3, 'B', 'US', 200), " + + "(4, 'B', 'EU', 75)", 4); + + assertUpdate("CREATE MATERIALIZED VIEW test_agg_mv AS " + + "SELECT region, SUM(sales) as total_sales " + + "FROM test_agg_misaligned " + + "GROUP BY region"); + + assertRefreshAndFullyMaterialized("test_agg_mv", 2); + + assertMaterializedViewQuery("SELECT * FROM test_agg_mv ORDER BY region", + "VALUES ('EU', 125), ('US', 300)"); + + assertUpdate("INSERT INTO test_agg_misaligned VALUES " + + "(5, 'A', 'US', 10), " + + "(6, 'B', 'US', 20)", 2); + + assertMaterializedViewQuery("SELECT * FROM test_agg_mv ORDER BY region", + "VALUES ('EU', 125), ('US', 330)"); + + assertUpdate("DROP MATERIALIZED VIEW test_agg_mv"); + assertUpdate("DROP TABLE test_agg_misaligned"); + } + + @Test + public void testAggregationMV_MultiTableJoin_BothStale() + { + assertUpdate("CREATE TABLE test_multi_orders (" + + "order_id BIGINT, " + + "product_id BIGINT, " + + "order_date DATE, " + + "quantity BIGINT) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_multi_products (" + + "product_id BIGINT, " + + "product_category VARCHAR, " + + "price BIGINT) " + + "WITH (partitioning = ARRAY['product_category'])"); + + assertUpdate("INSERT INTO test_multi_orders VALUES " + + "(1, 100, DATE '2024-01-01', 5), " + + "(2, 200, DATE '2024-01-01', 3)", 2); + assertUpdate("INSERT INTO test_multi_products VALUES " + + "(100, 'Electronics', 50), " + + "(200, 'Books', 20)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_multi_agg_mv AS " + + "SELECT p.product_category, SUM(o.quantity * p.price) as total_revenue " + + "FROM test_multi_orders o " + + "JOIN test_multi_products p ON o.product_id = p.product_id " + + "GROUP BY p.product_category"); + + assertRefreshAndFullyMaterialized("test_multi_agg_mv", 2); + + assertMaterializedViewQuery("SELECT * FROM test_multi_agg_mv ORDER BY product_category", + "VALUES ('Books', 60), ('Electronics', 250)"); + + assertUpdate("INSERT INTO test_multi_orders VALUES " + + "(3, 100, DATE '2024-01-02', 2), " + + "(4, 200, DATE '2024-01-02', 4)", 2); + + assertUpdate("INSERT INTO test_multi_products VALUES " + + "(300, 'Toys', 30)", 1); + + assertUpdate("INSERT INTO test_multi_orders VALUES " + + "(5, 300, DATE '2024-01-02', 1)", 1); + + assertMaterializedViewQuery("SELECT * FROM test_multi_agg_mv ORDER BY product_category", + "VALUES ('Books', 140), ('Electronics', 350), ('Toys', 30)"); + + assertUpdate("DROP MATERIALIZED VIEW test_multi_agg_mv"); + assertUpdate("DROP TABLE test_multi_products"); + assertUpdate("DROP TABLE test_multi_orders"); + } + + @Test + public void testMaterializedViewJoinAggregationWithMultipleStalePartitions() + { + assertUpdate("CREATE TABLE orders_bug (" + + "order_id BIGINT, " + + "product_id BIGINT, " + + "quantity BIGINT, " + + "order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE products_bug (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "price BIGINT) " + + "WITH (partitioning = ARRAY['category'])"); + + assertUpdate("INSERT INTO orders_bug VALUES (1, 200, 5, DATE '2024-01-01')", 1); + assertUpdate("INSERT INTO products_bug VALUES (200, 'Books', 50)", 1); + + // Create aggregation MV + assertUpdate("CREATE MATERIALIZED VIEW mv_revenue AS " + + "SELECT p.category, SUM(o.quantity * p.price) as total_revenue " + + "FROM orders_bug o " + + "JOIN products_bug p ON o.product_id = p.product_id " + + "GROUP BY p.category"); + + assertRefreshAndFullyMaterialized("mv_revenue", 1); + assertMaterializedViewQuery("SELECT * FROM mv_revenue ORDER BY category", + "VALUES ('Books', 250)"); + + // Make both base tables stale + assertUpdate("INSERT INTO products_bug VALUES (300, 'Electronics', 10)", 1); + assertUpdate("INSERT INTO orders_bug VALUES (2, 200, 3, DATE '2024-01-02'), (3, 300, 4, DATE '2024-01-02')", 2); + + assertMaterializedViewQuery("SELECT * FROM mv_revenue ORDER BY category", + "VALUES ('Books', 400), ('Electronics', 40)"); + assertMaterializedViewQuery("SELECT COUNT(*) FROM mv_revenue WHERE category = 'Books'", "SELECT 1"); + assertMaterializedViewQuery("SELECT COUNT(*) FROM mv_revenue WHERE category = 'Electronics'", "SELECT 1"); + + assertUpdate("DROP MATERIALIZED VIEW mv_revenue"); + assertUpdate("DROP TABLE products_bug"); + assertUpdate("DROP TABLE orders_bug"); + } + + @Test + public void testMaterializedViewWithCustomStorageTableName() + { + assertUpdate("CREATE TABLE test_custom_storage_base (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_custom_storage_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_custom_storage_mv " + + "WITH (storage_table = 'my_custom_storage_table') " + + "AS SELECT id, name, value FROM test_custom_storage_base"); + + assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 0"); + + assertQueryFails("SELECT * FROM \"__mv_storage__test_custom_storage_mv\"", ".*does not exist.*"); + + assertRefreshAndFullyMaterialized("test_custom_storage_mv", 2); + + assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 2"); + assertQuery("SELECT * FROM my_custom_storage_table ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertMaterializedViewQuery("SELECT * FROM test_custom_storage_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertUpdate("INSERT INTO test_custom_storage_base VALUES (3, 'Charlie', 300)", 1); + assertRefreshAndFullyMaterialized("test_custom_storage_mv", 3); + + assertQuery("SELECT COUNT(*) FROM my_custom_storage_table", "SELECT 3"); + assertQuery("SELECT * FROM my_custom_storage_table ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_custom_storage_mv"); + + assertQueryFails("SELECT * FROM my_custom_storage_table", ".*does not exist.*"); + + assertUpdate("DROP TABLE test_custom_storage_base"); + } + + @Test + public void testMaterializedViewWithCustomStorageSchema() + { + assertUpdate("CREATE SCHEMA IF NOT EXISTS test_storage_schema"); + + assertUpdate("CREATE TABLE test_custom_schema_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_custom_schema_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_custom_schema_mv " + + "WITH (storage_schema = 'test_storage_schema', " + + "storage_table = 'storage_table') " + + "AS SELECT id, value FROM test_schema.test_custom_schema_base"); + + assertQuery("SELECT COUNT(*) FROM test_storage_schema.storage_table", "SELECT 0"); + + assertQueryFails("SELECT * FROM test_schema.storage_table", ".*does not exist.*"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_schema.test_custom_schema_mv", 2); + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_custom_schema_mv'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_storage_schema.storage_table", "SELECT 2"); + assertQuery("SELECT * FROM test_storage_schema.storage_table ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertMaterializedViewQuery("SELECT * FROM test_custom_schema_mv ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertUpdate("DROP MATERIALIZED VIEW test_schema.test_custom_schema_mv"); + assertQueryFails("SELECT * FROM test_storage_schema.storage_table", ".*does not exist.*"); + + assertUpdate("DROP TABLE test_custom_schema_base"); + assertUpdate("DROP SCHEMA test_storage_schema"); + } + + @Test + public void testMaterializedViewWithCustomPrefix() + { + assertUpdate("CREATE TABLE test_custom_prefix_base (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_custom_prefix_base VALUES (1, 'test')", 1); + + Session sessionWithCustomPrefix = Session.builder(getSession()) + .setCatalogSessionProperty("iceberg", "materialized_view_storage_prefix", "custom_prefix_") + .build(); + + assertUpdate(sessionWithCustomPrefix, "CREATE MATERIALIZED VIEW test_custom_prefix_mv " + + "AS SELECT id, name FROM test_custom_prefix_base"); + + assertQuery("SELECT COUNT(*) FROM custom_prefix_test_custom_prefix_mv", "SELECT 0"); + + assertQueryFails("SELECT * FROM \"__mv_storage__test_custom_prefix_mv\"", ".*does not exist.*"); + + assertRefreshAndFullyMaterialized("test_custom_prefix_mv", 1); + + assertQuery("SELECT COUNT(*) FROM custom_prefix_test_custom_prefix_mv", "SELECT 1"); + assertQuery("SELECT * FROM custom_prefix_test_custom_prefix_mv", "VALUES (1, 'test')"); + + assertMaterializedViewQuery("SELECT * FROM test_custom_prefix_mv", "VALUES (1, 'test')"); + + assertUpdate("DROP MATERIALIZED VIEW test_custom_prefix_mv"); + assertQueryFails("SELECT * FROM custom_prefix_test_custom_prefix_mv", ".*does not exist.*"); + + assertUpdate("DROP TABLE test_custom_prefix_base"); + } + + @Test + public void testMaterializedViewWithValuesOnly() + { + assertUpdate("CREATE MATERIALIZED VIEW test_values_mv AS SELECT * FROM (VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)) AS t(id, name, value)"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_values_mv\"", "SELECT 0"); + + assertQuery("SELECT COUNT(*) FROM test_values_mv", "SELECT 3"); + assertQuery("SELECT * FROM test_values_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertRefreshAndFullyMaterialized("test_values_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_values_mv\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_values_mv\" ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertMaterializedViewQuery("SELECT * FROM test_values_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_values_mv"); + assertQueryFails("SELECT * FROM \"__mv_storage__test_values_mv\"", ".*does not exist.*"); + } + + @Test + public void testMaterializedViewWithBaseTableButNoColumnsSelected() + { + assertUpdate("CREATE TABLE test_no_cols_base (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_no_cols_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_no_cols_mv AS " + + "SELECT 'constant' as label, 42 as fixed_value FROM test_no_cols_base"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 0"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_no_cols_mv", "SELECT 3"); + assertMaterializedViewQuery("SELECT * FROM test_no_cols_mv", + "VALUES ('constant', 42), ('constant', 42), ('constant', 42)"); + + assertRefreshAndFullyMaterialized("test_no_cols_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_no_cols_mv\"", + "VALUES ('constant', 42), ('constant', 42), ('constant', 42)"); + + assertUpdate("INSERT INTO test_no_cols_base VALUES (4, 'Dave', 400)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_no_cols_mv", "SELECT 4"); + + assertRefreshAndFullyMaterialized("test_no_cols_mv", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_no_cols_mv\"", "SELECT 4"); + + assertUpdate("DROP MATERIALIZED VIEW test_no_cols_mv"); + assertQueryFails("SELECT * FROM \"__mv_storage__test_no_cols_mv\"", ".*does not exist.*"); + + assertUpdate("DROP TABLE test_no_cols_base"); + } + + @Test + public void testMaterializedViewOnEmptyBaseTable() + { + assertUpdate("CREATE TABLE test_empty_base (id BIGINT, name VARCHAR, value BIGINT)"); + + assertUpdate("CREATE MATERIALIZED VIEW test_empty_mv AS SELECT id, name, value FROM test_empty_base"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 0"); + + assertQuery("SELECT COUNT(*) FROM test_empty_mv", "SELECT 0"); + + assertRefreshAndFullyMaterialized("test_empty_mv", 0); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 0"); + + assertUpdate("INSERT INTO test_empty_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_empty_mv", "SELECT 2"); + assertMaterializedViewQuery("SELECT * FROM test_empty_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertRefreshAndFullyMaterialized("test_empty_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_empty_mv\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_empty_mv\" ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertUpdate("DROP MATERIALIZED VIEW test_empty_mv"); + assertQueryFails("SELECT * FROM \"__mv_storage__test_empty_mv\"", ".*does not exist.*"); + + assertUpdate("DROP TABLE test_empty_base"); + } + + @Test + public void testRefreshFailurePreservesOldData() + { + assertUpdate("CREATE TABLE test_refresh_failure_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_refresh_failure_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_refresh_failure_mv AS " + + "SELECT id, value FROM test_refresh_failure_base"); + + assertRefreshAndFullyMaterialized("test_refresh_failure_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_refresh_failure_mv\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_refresh_failure_mv\" ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertUpdate("DROP TABLE test_refresh_failure_base"); + + try { + getQueryRunner().execute("REFRESH MATERIALIZED VIEW test_refresh_failure_mv"); + throw new AssertionError("Expected REFRESH to fail when base table doesn't exist"); + } + catch (Exception e) { + if (!e.getMessage().contains("does not exist") && !e.getMessage().contains("not found")) { + throw new AssertionError("Expected 'does not exist' error, got: " + e.getMessage()); + } + } + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_refresh_failure_mv\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_refresh_failure_mv\" ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertUpdate("DROP MATERIALIZED VIEW test_refresh_failure_mv"); + } + + @Test + public void testBaseTableDroppedAndRecreated() + { + assertUpdate("CREATE TABLE test_recreate_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_recreate_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_recreate_mv AS SELECT id, value FROM test_recreate_base"); + assertRefreshAndFullyMaterialized("test_recreate_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_recreate_mv\" ORDER BY id", + "VALUES (1, 100), (2, 200)"); + + assertUpdate("DROP TABLE test_recreate_base"); + + assertUpdate("CREATE TABLE test_recreate_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_recreate_base VALUES (3, 300), (4, 400), (5, 500)", 3); + + assertQuery("SELECT COUNT(*) FROM test_recreate_mv", "SELECT 3"); + assertQuery("SELECT * FROM test_recreate_mv ORDER BY id", + "VALUES (3, 300), (4, 400), (5, 500)"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 2"); + + assertRefreshAndFullyMaterialized("test_recreate_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_recreate_mv\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_recreate_mv\" ORDER BY id", + "VALUES (3, 300), (4, 400), (5, 500)"); + + assertUpdate("DROP MATERIALIZED VIEW test_recreate_mv"); + assertUpdate("DROP TABLE test_recreate_base"); + } + + @Test + public void testStorageTableDroppedDirectly() + { + assertUpdate("CREATE TABLE test_storage_drop_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_storage_drop_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_storage_drop_mv AS SELECT id, value FROM test_storage_drop_base"); + assertRefreshAndFullyMaterialized("test_storage_drop_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_storage_drop_mv\"", "SELECT 2"); + + assertUpdate("DROP TABLE \"__mv_storage__test_storage_drop_mv\""); + + assertQueryFails("SELECT * FROM \"__mv_storage__test_storage_drop_mv\"", ".*does not exist.*"); + + assertQueryFails("SELECT * FROM test_storage_drop_mv", ".*does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW test_storage_drop_mv"); + assertUpdate("DROP TABLE test_storage_drop_base"); + } + + @Test + public void testMaterializedViewWithRenamedColumns() + { + assertUpdate("CREATE TABLE test_renamed_base (id BIGINT, original_name VARCHAR, original_value BIGINT)"); + assertUpdate("INSERT INTO test_renamed_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_renamed_mv AS " + + "SELECT id AS person_id, original_name AS full_name, original_value AS amount " + + "FROM test_renamed_base"); + + assertQuery("SELECT COUNT(*) FROM test_renamed_mv", "SELECT 3"); + assertQuery("SELECT * FROM test_renamed_mv ORDER BY person_id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertRefreshAndFullyMaterialized("test_renamed_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_renamed_mv\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_renamed_mv\" ORDER BY person_id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertQuery("SELECT * FROM test_renamed_mv ORDER BY person_id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertQuery("SELECT person_id, full_name FROM test_renamed_mv WHERE amount > 150 ORDER BY person_id", + "VALUES (2, 'Bob'), (3, 'Charlie')"); + + assertUpdate("INSERT INTO test_renamed_base VALUES (4, 'Dave', 400)", 1); + + assertQuery("SELECT COUNT(*) FROM test_renamed_mv", "SELECT 4"); + + assertRefreshAndFullyMaterialized("test_renamed_mv", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_renamed_mv\"", "SELECT 4"); + assertQuery("SELECT * FROM \"__mv_storage__test_renamed_mv\" ORDER BY person_id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300), (4, 'Dave', 400)"); + + assertUpdate("DROP MATERIALIZED VIEW test_renamed_mv"); + assertUpdate("DROP TABLE test_renamed_base"); + } + + @Test + public void testMaterializedViewWithComputedColumns() + { + assertUpdate("CREATE TABLE test_computed_base (id BIGINT, quantity BIGINT, unit_price BIGINT)"); + assertUpdate("INSERT INTO test_computed_base VALUES (1, 5, 100), (2, 10, 50), (3, 3, 200)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_computed_mv AS " + + "SELECT id, " + + "quantity, " + + "unit_price, " + + "quantity * unit_price AS total_price, " + + "quantity * 2 AS double_quantity, " + + "'Order_' || CAST(id AS VARCHAR) AS order_label " + + "FROM test_computed_base"); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_computed_mv", "SELECT 3"); + assertMaterializedViewQuery("SELECT id, quantity, unit_price, total_price, double_quantity, order_label FROM test_computed_mv ORDER BY id", + "VALUES (1, 5, 100, 500, 10, 'Order_1'), " + + "(2, 10, 50, 500, 20, 'Order_2'), " + + "(3, 3, 200, 600, 6, 'Order_3')"); + + assertRefreshAndFullyMaterialized("test_computed_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_computed_mv\"", "SELECT 3"); + assertQuery("SELECT id, quantity, unit_price, total_price, double_quantity, order_label FROM \"__mv_storage__test_computed_mv\" ORDER BY id", + "VALUES (1, 5, 100, 500, 10, 'Order_1'), " + + "(2, 10, 50, 500, 20, 'Order_2'), " + + "(3, 3, 200, 600, 6, 'Order_3')"); + + assertMaterializedViewQuery("SELECT * FROM test_computed_mv WHERE total_price > 550 ORDER BY id", + "VALUES (3, 3, 200, 600, 6, 'Order_3')"); + + assertMaterializedViewQuery("SELECT id, order_label FROM test_computed_mv WHERE double_quantity >= 10 ORDER BY id", + "VALUES (1, 'Order_1'), (2, 'Order_2')"); + + assertUpdate("INSERT INTO test_computed_base VALUES (4, 8, 75)", 1); + + assertMaterializedViewQuery("SELECT COUNT(*) FROM test_computed_mv", "SELECT 4"); + assertMaterializedViewQuery("SELECT id, total_price, order_label FROM test_computed_mv WHERE id = 4", + "VALUES (4, 600, 'Order_4')"); + + assertRefreshAndFullyMaterialized("test_computed_mv", 4); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_computed_mv\"", "SELECT 4"); + assertQuery("SELECT id, quantity, unit_price, total_price, order_label FROM \"__mv_storage__test_computed_mv\" WHERE id = 4", + "VALUES (4, 8, 75, 600, 'Order_4')"); + + assertUpdate("DROP MATERIALIZED VIEW test_computed_mv"); + assertUpdate("DROP TABLE test_computed_base"); + } + + @Test + public void testMaterializedViewWithCustomTableProperties() + { + assertUpdate("CREATE TABLE test_custom_props_base (id BIGINT, name VARCHAR, region VARCHAR)"); + assertUpdate("INSERT INTO test_custom_props_base VALUES (1, 'Alice', 'US'), (2, 'Bob', 'EU'), (3, 'Charlie', 'APAC')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_custom_props_mv " + + "WITH (" + + " partitioning = ARRAY['region'], " + + " sorted_by = ARRAY['id'], " + + " \"write.format.default\" = 'PARQUET'" + + ") AS " + + "SELECT id, name, region FROM test_custom_props_base"); + + assertRefreshAndFullyMaterialized("test_custom_props_mv", 3); + + assertQuery("SELECT COUNT(*) FROM test_custom_props_mv", "SELECT 3"); + assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'US'", "VALUES ('Alice')"); + assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'EU'", "VALUES ('Bob')"); + + String storageTableName = "__mv_storage__test_custom_props_mv"; + assertQuery("SELECT COUNT(*) FROM \"" + storageTableName + "\"", "SELECT 3"); + + assertQuery("SELECT COUNT(*) FROM \"" + storageTableName + "\" WHERE region = 'APAC'", "SELECT 1"); + + assertUpdate("INSERT INTO test_custom_props_base VALUES (4, 'David', 'US')", 1); + assertRefreshAndFullyMaterialized("test_custom_props_mv", 4); + + assertQuery("SELECT COUNT(*) FROM test_custom_props_mv WHERE region = 'US'", "SELECT 2"); + assertQuery("SELECT name FROM test_custom_props_mv WHERE region = 'US' ORDER BY id", + "VALUES ('Alice'), ('David')"); + + assertUpdate("DROP MATERIALIZED VIEW test_custom_props_mv"); + assertUpdate("DROP TABLE test_custom_props_base"); + } + + @Test + public void testMaterializedViewWithNestedTypes() + { + assertUpdate("CREATE TABLE test_nested_base (" + + "id BIGINT, " + + "tags ARRAY(VARCHAR), " + + "properties MAP(VARCHAR, VARCHAR), " + + "address ROW(street VARCHAR, city VARCHAR, zipcode VARCHAR))"); + + assertUpdate("INSERT INTO test_nested_base VALUES " + + "(1, ARRAY['tag1', 'tag2'], MAP(ARRAY['key1', 'key2'], ARRAY['value1', 'value2']), ROW('123 Main St', 'NYC', '10001')), " + + "(2, ARRAY['tag3'], MAP(ARRAY['key3'], ARRAY['value3']), ROW('456 Oak Ave', 'LA', '90001'))", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_nested_mv AS " + + "SELECT id, tags, properties, address FROM test_nested_base"); + + assertQuery("SELECT COUNT(*) FROM test_nested_mv", "SELECT 2"); + assertQuery("SELECT id, cardinality(tags) FROM test_nested_mv ORDER BY id", + "VALUES (1, 2), (2, 1)"); + + assertRefreshAndFullyMaterialized("test_nested_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_nested_mv\"", "SELECT 2"); + + assertQuery("SELECT id, cardinality(tags) FROM test_nested_mv ORDER BY id", + "VALUES (1, 2), (2, 1)"); + + assertQuery("SELECT id FROM test_nested_mv WHERE element_at(properties, 'key1') = 'value1'", + "VALUES (1)"); + + assertUpdate("INSERT INTO test_nested_base VALUES " + + "(3, ARRAY['tag4', 'tag5', 'tag6'], MAP(ARRAY['key4'], ARRAY['value4']), ROW('789 Elm St', 'Chicago', '60601'))", 1); + + assertQuery("SELECT COUNT(*) FROM test_nested_mv", "SELECT 3"); + + assertRefreshAndFullyMaterialized("test_nested_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_nested_mv\"", "SELECT 3"); + assertQuery("SELECT id, cardinality(tags) FROM test_nested_mv WHERE id = 3", + "VALUES (3, 3)"); + + assertUpdate("DROP MATERIALIZED VIEW test_nested_mv"); + assertUpdate("DROP TABLE test_nested_base"); + } + + @Test + public void testMaterializedViewAfterColumnAdded() + { + assertUpdate("CREATE TABLE test_evolve_add_base (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_evolve_add_base VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_evolve_add_mv AS " + + "SELECT id, name, value FROM test_evolve_add_base"); + + assertRefreshAndFullyMaterialized("test_evolve_add_mv", 2); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 2"); + assertQuery("SELECT * FROM test_evolve_add_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertUpdate("ALTER TABLE test_evolve_add_base ADD COLUMN region VARCHAR"); + + assertUpdate("INSERT INTO test_evolve_add_base VALUES (3, 'Charlie', 300, 'US')", 1); + + assertQuery("SELECT COUNT(*) FROM test_evolve_add_mv", "SELECT 3"); + assertQuery("SELECT * FROM test_evolve_add_mv ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 2"); + + assertRefreshAndFullyMaterialized("test_evolve_add_mv", 3); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_add_mv\"", "SELECT 3"); + assertQuery("SELECT * FROM \"__mv_storage__test_evolve_add_mv\" ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertUpdate("CREATE MATERIALIZED VIEW test_evolve_add_mv2 AS " + + "SELECT id, name, value, region FROM test_evolve_add_base"); + + assertRefreshAndFullyMaterialized("test_evolve_add_mv2", 3); + + assertQuery("SELECT * FROM test_evolve_add_mv2 WHERE id = 3", + "VALUES (3, 'Charlie', 300, 'US')"); + assertQuery("SELECT id, region FROM test_evolve_add_mv2 WHERE id IN (1, 2) ORDER BY id", + "VALUES (1, NULL), (2, NULL)"); + + assertUpdate("DROP MATERIALIZED VIEW test_evolve_add_mv"); + assertUpdate("DROP MATERIALIZED VIEW test_evolve_add_mv2"); + assertUpdate("DROP TABLE test_evolve_add_base"); + } + + @Test + public void testMaterializedViewAfterColumnDropped() + { + assertUpdate("CREATE TABLE test_evolve_drop_base (id BIGINT, name VARCHAR, value BIGINT, status VARCHAR)"); + assertUpdate("INSERT INTO test_evolve_drop_base VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_evolve_drop_mv_all AS " + + "SELECT id, name, value, status FROM test_evolve_drop_base"); + + assertUpdate("CREATE MATERIALIZED VIEW test_evolve_drop_mv_subset AS " + + "SELECT id, name, value FROM test_evolve_drop_base"); + + assertRefreshAndFullyMaterialized("test_evolve_drop_mv_all", 2); + assertRefreshAndFullyMaterialized("test_evolve_drop_mv_subset", 2); + + assertQuery("SELECT * FROM test_evolve_drop_mv_all ORDER BY id", + "VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')"); + assertQuery("SELECT * FROM test_evolve_drop_mv_subset ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + assertUpdate("ALTER TABLE test_evolve_drop_base DROP COLUMN status"); + + assertUpdate("INSERT INTO test_evolve_drop_base VALUES (3, 'Charlie', 300)", 1); + + assertQuery("SELECT COUNT(*) FROM test_evolve_drop_mv_subset", "SELECT 3"); + assertQuery("SELECT * FROM test_evolve_drop_mv_subset ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertQueryFails("SELECT * FROM test_evolve_drop_mv_all", + ".*Column 'status' cannot be resolved.*"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_drop_mv_all\"", "SELECT 2"); + assertQuery("SELECT * FROM \"__mv_storage__test_evolve_drop_mv_all\" ORDER BY id", + "VALUES (1, 'Alice', 100, 'active'), (2, 'Bob', 200, 'inactive')"); + + assertRefreshAndFullyMaterialized("test_evolve_drop_mv_subset", 3); + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_evolve_drop_mv_subset\"", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW test_evolve_drop_mv_all"); + assertUpdate("DROP MATERIALIZED VIEW test_evolve_drop_mv_subset"); + assertUpdate("DROP TABLE test_evolve_drop_base"); + } + + @Test + public void testDropNonExistentMaterializedView() + { + assertQueryFails("DROP MATERIALIZED VIEW non_existent_mv", + ".*does not exist.*"); + } + + @Test + public void testCreateMaterializedViewWithSameNameAsExistingTable() + { + assertUpdate("CREATE TABLE existing_table_name (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO existing_table_name VALUES (1, 'test')", 1); + + assertQueryFails("CREATE MATERIALIZED VIEW existing_table_name AS SELECT id, value FROM existing_table_name", + ".*already exists.*"); + + assertQuery("SELECT COUNT(*) FROM existing_table_name", "SELECT 1"); + assertQuery("SELECT * FROM existing_table_name", "VALUES (1, 'test')"); + + assertUpdate("CREATE TABLE test_mv_base (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_mv_base VALUES (2, 'foo')", 1); + + assertQueryFails("CREATE MATERIALIZED VIEW existing_table_name AS SELECT id, name FROM test_mv_base", + ".*already exists.*"); + + assertUpdate("DROP TABLE existing_table_name"); + assertUpdate("DROP TABLE test_mv_base"); + } + + @Test + public void testInformationSchemaMaterializedViews() + { + assertUpdate("CREATE TABLE test_is_mv_base1 (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("CREATE TABLE test_is_mv_base2 (category VARCHAR, amount BIGINT)"); + + assertUpdate("INSERT INTO test_is_mv_base1 VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + assertUpdate("INSERT INTO test_is_mv_base2 VALUES ('A', 50), ('B', 75)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_is_mv1 AS SELECT id, name, value FROM test_is_mv_base1 WHERE id > 0"); + assertUpdate("CREATE MATERIALIZED VIEW test_is_mv2 AS SELECT category, SUM(amount) as total FROM test_is_mv_base2 GROUP BY category"); + + assertQuery( + "SELECT table_name FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_mv1', 'test_is_mv2') " + + "ORDER BY table_name", + "VALUES ('test_is_mv1'), ('test_is_mv2')"); + + assertQuery( + "SELECT table_catalog, table_schema, table_name, storage_schema, storage_table_name, base_tables " + + "FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1'", + "SELECT 'iceberg', 'test_schema', 'test_is_mv1', 'test_schema', '__mv_storage__test_is_mv1', 'iceberg.test_schema.test_is_mv_base1'"); + + assertQuery( + "SELECT COUNT(*) FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + + "AND view_definition IS NOT NULL AND length(view_definition) > 0", + "SELECT 1"); + + assertQuery( + "SELECT table_name FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv2'", + "VALUES ('test_is_mv2')"); + + assertQuery( + "SELECT COUNT(*) FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + + "AND view_owner IS NOT NULL", + "SELECT 1"); + + assertQuery( + "SELECT COUNT(*) FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv1' " + + "AND view_security IS NOT NULL", + "SELECT 1"); + + assertQuery( + "SELECT base_tables FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv2'", + "VALUES ('iceberg.test_schema.test_is_mv_base2')"); + + assertUpdate("DROP MATERIALIZED VIEW test_is_mv1"); + assertUpdate("DROP MATERIALIZED VIEW test_is_mv2"); + assertUpdate("DROP TABLE test_is_mv_base1"); + assertUpdate("DROP TABLE test_is_mv_base2"); + + assertQuery( + "SELECT COUNT(*) FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_mv1', 'test_is_mv2')", + "VALUES 0"); + } + + @Test + public void testInformationSchemaTablesWithMaterializedViews() + { + assertUpdate("CREATE TABLE test_is_tables_base (id BIGINT, name VARCHAR)"); + assertUpdate("CREATE VIEW test_is_tables_view AS SELECT id, name FROM test_is_tables_base"); + assertUpdate("CREATE MATERIALIZED VIEW test_is_tables_mv AS SELECT id, name FROM test_is_tables_base"); + + assertQuery( + "SELECT table_name, table_type FROM information_schema.tables " + + "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_tables_base', 'test_is_tables_view', 'test_is_tables_mv') " + + "ORDER BY table_name", + "VALUES ('test_is_tables_base', 'BASE TABLE'), ('test_is_tables_mv', 'MATERIALIZED VIEW'), ('test_is_tables_view', 'VIEW')"); + + assertQuery( + "SELECT table_name FROM information_schema.views " + + "WHERE table_schema = 'test_schema' AND table_name IN ('test_is_tables_view', 'test_is_tables_mv') " + + "ORDER BY table_name", + "VALUES ('test_is_tables_view')"); + + assertUpdate("DROP MATERIALIZED VIEW test_is_tables_mv"); + assertUpdate("DROP VIEW test_is_tables_view"); + assertUpdate("DROP TABLE test_is_tables_base"); + } + + @Test + public void testInformationSchemaMaterializedViewsAfterRefresh() + { + assertUpdate("CREATE TABLE test_is_mv_refresh_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (1, 100), (2, 200)", 2); + assertUpdate("CREATE MATERIALIZED VIEW test_is_mv_refresh AS SELECT id, value FROM test_is_mv_refresh_base"); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'NOT_MATERIALIZED'"); + + assertRefreshAndFullyMaterialized("test_is_mv_refresh", 2); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (3, 300)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (4, 400)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertUpdate("INSERT INTO test_is_mv_refresh_base VALUES (5, 500)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertRefreshAndFullyMaterialized("test_is_mv_refresh", 5); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertUpdate("DROP MATERIALIZED VIEW test_is_mv_refresh"); + assertUpdate("DROP TABLE test_is_mv_refresh_base"); + + assertQuery( + "SELECT COUNT(*) FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_is_mv_refresh'", + "VALUES 0"); + } + + @Test + public void testStaleReadBehaviorFail() + { + assertUpdate("CREATE TABLE test_stale_fail_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_stale_fail_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_stale_fail " + + "WITH (stale_read_behavior = 'FAIL', staleness_window = '0s') " + + "AS SELECT id, value FROM test_stale_fail_base"); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", + "SELECT 'NOT_MATERIALIZED'"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_stale_fail", 2); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_stale_fail", "SELECT 2"); + assertQuery("SELECT * FROM test_stale_fail ORDER BY id", "VALUES (1, 100), (2, 200)"); + + assertUpdate("INSERT INTO test_stale_fail_base VALUES (3, 300)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertQueryFails("SELECT * FROM test_stale_fail", + ".*Materialized view .* is stale.*"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_stale_fail", 3); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_fail'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_stale_fail", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW test_stale_fail"); + assertUpdate("DROP TABLE test_stale_fail_base"); + } + + @Test + public void testStaleReadBehaviorUseViewQuery() + { + assertUpdate("CREATE TABLE test_stale_use_query_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_stale_use_query_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_stale_use_query " + + "WITH (stale_read_behavior = 'USE_VIEW_QUERY', staleness_window = '0s') " + + "AS SELECT id, value FROM test_stale_use_query_base"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_stale_use_query", 2); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_use_query'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_stale_use_query", "SELECT 2"); + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_stale_use_query\"", "SELECT 2"); + + assertUpdate("INSERT INTO test_stale_use_query_base VALUES (3, 300)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_stale_use_query'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_stale_use_query", "SELECT 3"); + assertQuery("SELECT * FROM test_stale_use_query ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_stale_use_query\"", "SELECT 2"); + + assertUpdate("DROP MATERIALIZED VIEW test_stale_use_query"); + assertUpdate("DROP TABLE test_stale_use_query_base"); + } + + @Test + public void testMaterializedViewWithNoStaleReadBehavior() + { + assertUpdate("CREATE TABLE test_no_stale_config_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_no_stale_config_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_no_stale_config AS SELECT id, value FROM test_no_stale_config_base"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_no_stale_config", 2); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_no_stale_config'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_no_stale_config", "SELECT 2"); + + assertUpdate("INSERT INTO test_no_stale_config_base VALUES (3, 300)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_no_stale_config'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_no_stale_config", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW test_no_stale_config"); + assertUpdate("DROP TABLE test_no_stale_config_base"); + } + + @Test + public void testStalenessWindowAllowsStaleReads() + { + assertUpdate("CREATE TABLE test_staleness_window_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO test_staleness_window_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_staleness_window_mv " + + "WITH (stale_read_behavior = 'FAIL', staleness_window = '1h') " + + "AS SELECT id, value FROM test_staleness_window_base"); + + assertUpdate("REFRESH MATERIALIZED VIEW test_staleness_window_mv", 2); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_staleness_window_mv'", + "SELECT 'FULLY_MATERIALIZED'"); + + assertQuery("SELECT COUNT(*) FROM test_staleness_window_mv", "SELECT 2"); + assertQuery("SELECT * FROM test_staleness_window_mv ORDER BY id", "VALUES (1, 100), (2, 200)"); + + assertUpdate("INSERT INTO test_staleness_window_base VALUES (3, 300)", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'test_staleness_window_mv'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + MaterializedResult staleResult = getQueryRunner().execute(getSession(), "SELECT COUNT(*) FROM test_staleness_window_mv"); + assertEquals((long) staleResult.getMaterializedRows().get(0).getField(0), 2L); + assertTrue(staleResult.getWarnings().stream() + .anyMatch(warning -> warning.getWarningCode().equals(MATERIALIZED_VIEW_STALE_DATA.toWarningCode()))); + + assertQuery("SELECT COUNT(*) FROM \"__mv_storage__test_staleness_window_mv\"", "SELECT 2"); + + assertUpdate("DROP MATERIALIZED VIEW test_staleness_window_mv"); + assertUpdate("DROP TABLE test_staleness_window_base"); + } + + @Test + public void testInsertAndCtasFromMaterializedView() + { + assertUpdate("CREATE TABLE test_mv_insert_base (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_mv_insert_base VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_insert_mv AS SELECT id, name, value FROM test_mv_insert_base"); + + try { + // CTAS from MV should succeed (no longer blanket-blocked) + assertQuerySucceeds("CREATE TABLE test_mv_insert_ctas AS SELECT * FROM test_mv_insert_mv"); + + // INSERT from MV into a non-base-table should succeed + assertUpdate("CREATE TABLE test_mv_insert_target (id BIGINT, name VARCHAR, value BIGINT)"); + assertQuerySucceeds("INSERT INTO test_mv_insert_target SELECT * FROM test_mv_insert_mv"); + + // INSERT from MV into its base table should fail (circular dependency) + assertQueryFails("INSERT INTO test_mv_insert_base SELECT * FROM test_mv_insert_mv", + ".*INSERT into table .* by selecting from materialized view .* is not supported because .* is a base table of the materialized view.*"); + } + finally { + getQueryRunner().execute("DROP TABLE IF EXISTS test_mv_insert_ctas"); + getQueryRunner().execute("DROP TABLE IF EXISTS test_mv_insert_target"); + getQueryRunner().execute("DROP MATERIALIZED VIEW IF EXISTS test_mv_insert_mv"); + getQueryRunner().execute("DROP TABLE IF EXISTS test_mv_insert_base"); + } + } + + @Test + public void testInsertFromMaterializedViewTransitiveBaseTables() + { + // Create base table -> view -> materialized view chain to test transitive base table resolution + assertUpdate("CREATE TABLE test_mv_transitive_base (id BIGINT, category VARCHAR, amount BIGINT)"); + assertUpdate("INSERT INTO test_mv_transitive_base VALUES (1, 'A', 100), (2, 'B', 200), (3, 'A', 300)", 3); + + assertUpdate("CREATE VIEW test_mv_transitive_view AS SELECT id, category, amount FROM test_mv_transitive_base"); + + assertUpdate("CREATE MATERIALIZED VIEW test_mv_transitive_mv AS SELECT id, category, amount FROM test_mv_transitive_view"); + + try { + // INSERT from MV into a non-base-table should succeed + assertUpdate("CREATE TABLE test_mv_transitive_target (id BIGINT, category VARCHAR, amount BIGINT)"); + assertQuerySucceeds("INSERT INTO test_mv_transitive_target SELECT * FROM test_mv_transitive_mv"); + + // INSERT from MV into the transitive base table (underlying table of the view) should fail + assertQueryFails("INSERT INTO test_mv_transitive_base SELECT * FROM test_mv_transitive_mv", + ".*INSERT into table .* by selecting from materialized view .* is not supported because .* is a base table of the materialized view.*"); + } + finally { + getQueryRunner().execute("DROP TABLE IF EXISTS test_mv_transitive_target"); + getQueryRunner().execute("DROP MATERIALIZED VIEW IF EXISTS test_mv_transitive_mv"); + getQueryRunner().execute("DROP VIEW IF EXISTS test_mv_transitive_view"); + getQueryRunner().execute("DROP TABLE IF EXISTS test_mv_transitive_base"); + } + } + + @Test + public void testMaterializedViewStitchingForTimestamp() + { + assertUpdate("CREATE TABLE test_ts_stitch_base (a INTEGER, b TIMESTAMP) WITH (partitioning = ARRAY['b'])"); + assertUpdate("INSERT INTO test_ts_stitch_base VALUES (1, TIMESTAMP '1984-12-08 00:00:10')", 1); + assertUpdate("INSERT INTO test_ts_stitch_base VALUES (2, TIMESTAMP '2001-09-10 00:10:00')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW test_ts_stitch_mv AS SELECT * FROM test_ts_stitch_base"); + assertUpdate("REFRESH MATERIALIZED VIEW test_ts_stitch_mv", 2); + assertRefreshAndFullyMaterialized("test_ts_stitch_mv", 2); + + assertUpdate("INSERT INTO test_ts_stitch_base VALUES (3, TIMESTAMP '1984-12-08 00:00:10')", 1); + + assertMaterializedViewQuery("SELECT * FROM test_ts_stitch_mv ORDER BY a", + "VALUES (1, TIMESTAMP '1984-12-08 00:00:10'), " + + "(2, TIMESTAMP '2001-09-10 00:10:00'), " + + "(3, TIMESTAMP '1984-12-08 00:00:10')"); + + assertUpdate("DROP MATERIALIZED VIEW test_ts_stitch_mv"); + assertUpdate("DROP TABLE test_ts_stitch_base"); + } + + @Test + public void testMultipleAggregatesWithSingleStaleTable() + { + assertUpdate("CREATE TABLE test_agg_single_stale (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_agg_single_stale VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'B', 150, DATE '2024-01-01'), " + + "(4, 'B', 250, DATE '2024-01-01')", 4); + + assertUpdate("CREATE MATERIALIZED VIEW test_multi_agg_mv AS " + + "SELECT category, " + + " COUNT(*) as cnt, " + + " SUM(value) as total, " + + " AVG(value) as average, " + + " MIN(value) as minimum, " + + " MAX(value) as maximum " + + "FROM test_agg_single_stale " + + "GROUP BY category"); + + assertRefreshAndFullyMaterialized("test_multi_agg_mv", 2); + + // Verify initial state + assertMaterializedViewQuery("SELECT * FROM test_multi_agg_mv ORDER BY category", + "VALUES ('A', 2, 300, 150.0, 100, 200), " + + " ('B', 2, 400, 200.0, 150, 250)"); + + // Insert new data in new partition - makes table stale + assertUpdate("INSERT INTO test_agg_single_stale VALUES " + + "(5, 'A', 50, DATE '2024-01-02'), " + + "(6, 'B', 300, DATE '2024-01-02'), " + + "(7, 'C', 175, DATE '2024-01-02')", 3); + + // Expected results after stitching: + // A: cnt=3, total=350, avg=116.67, min=50, max=200 + // B: cnt=3, total=700, avg=233.33, min=150, max=300 + // C: cnt=1, total=175, avg=175, min=175, max=175 + assertMaterializedViewQuery("SELECT category, cnt, total, minimum, maximum FROM test_multi_agg_mv ORDER BY category", + "VALUES ('A', 3, 350, 50, 200), " + + " ('B', 3, 700, 150, 300), " + + " ('C', 1, 175, 175, 175)"); + + assertUpdate("DROP MATERIALIZED VIEW test_multi_agg_mv"); + assertUpdate("DROP TABLE test_agg_single_stale"); + } + + @Test + public void testMultipleAggregatesWithMultipleStaleTables() + { + assertUpdate("CREATE TABLE test_agg_orders (" + + "order_id BIGINT, " + + "product_id BIGINT, " + + "quantity BIGINT, " + + "price BIGINT, " + + "order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_agg_products (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "region VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + + assertUpdate("INSERT INTO test_agg_orders VALUES " + + "(1, 100, 5, 10, DATE '2024-01-01'), " + + "(2, 200, 3, 20, DATE '2024-01-01'), " + + "(3, 100, 2, 10, DATE '2024-01-01')", 3); + + assertUpdate("INSERT INTO test_agg_products VALUES " + + "(100, 'Electronics', 'US'), " + + "(200, 'Books', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_multi_agg_join_mv AS " + + "SELECT p.category, " + + " COUNT(*) as order_count, " + + " SUM(o.quantity) as total_quantity, " + + " AVG(o.quantity) as avg_quantity, " + + " MIN(o.price) as min_price, " + + " MAX(o.price) as max_price, " + + " SUM(o.quantity * o.price) as revenue " + + "FROM test_agg_orders o " + + "JOIN test_agg_products p ON o.product_id = p.product_id " + + "GROUP BY p.category"); + + assertRefreshAndFullyMaterialized("test_multi_agg_join_mv", 2); + + // Initial: Electronics: 2 orders, 7 qty, avg 3.5, min 10, max 10, revenue 70 + // Books: 1 order, 3 qty, avg 3, min 20, max 20, revenue 60 + assertMaterializedViewQuery("SELECT category, order_count, total_quantity, min_price, max_price, revenue FROM test_multi_agg_join_mv ORDER BY category", + "VALUES ('Books', 1, 3, 20, 20, 60), " + + " ('Electronics', 2, 7, 10, 10, 70)"); + + // Make orders table stale - add order in new date partition + assertUpdate("INSERT INTO test_agg_orders VALUES (4, 200, 10, 25, DATE '2024-01-02')", 1); + + // Books should now have: 2 orders, 13 qty, min 20, max 25, revenue 310 + assertMaterializedViewQuery("SELECT category, order_count, total_quantity, min_price, max_price, revenue FROM test_multi_agg_join_mv ORDER BY category", + "VALUES ('Books', 2, 13, 20, 25, 310), " + + " ('Electronics', 2, 7, 10, 10, 70)"); + + // Make products table also stale - add product in new region + assertUpdate("INSERT INTO test_agg_products VALUES (300, 'Clothing', 'EU')", 1); + assertUpdate("INSERT INTO test_agg_orders VALUES (5, 300, 8, 30, DATE '2024-01-02')", 1); + + // Clothing should now appear: 1 order, 8 qty, min 30, max 30, revenue 240 + assertMaterializedViewQuery("SELECT category, order_count, total_quantity, min_price, max_price, revenue FROM test_multi_agg_join_mv ORDER BY category", + "VALUES ('Books', 2, 13, 20, 25, 310), " + + " ('Clothing', 1, 8, 30, 30, 240), " + + " ('Electronics', 2, 7, 10, 10, 70)"); + + assertUpdate("DROP MATERIALIZED VIEW test_multi_agg_join_mv"); + assertUpdate("DROP TABLE test_agg_products"); + assertUpdate("DROP TABLE test_agg_orders"); + } + + @Test + public void testAggregationMVWithMultipleStaleTables() + { + // This test verifies that aggregation MVs with multiple stale tables produce + // correct results by comparing stitched results with full recomputation. + // This is a regression test for the MarkDistinct optimization which skips + // deduplication for aggregation queries that already produce unique rows. + assertUpdate("CREATE TABLE test_agg_multi_stale_t1 (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("CREATE TABLE test_agg_multi_stale_t2 (" + + "id BIGINT, " + + "region VARCHAR, " + + "multiplier BIGINT, " + + "reg_date DATE) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO test_agg_multi_stale_t1 VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'B', 200, DATE '2024-01-01')", 2); + + assertUpdate("INSERT INTO test_agg_multi_stale_t2 VALUES " + + "(1, 'US', 2, DATE '2024-01-01'), " + + "(2, 'EU', 3, DATE '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_agg_multi_stale_mv AS " + + "SELECT t1.category, " + + " SUM(t1.value * t2.multiplier) as total " + + "FROM test_agg_multi_stale_t1 t1 " + + "JOIN test_agg_multi_stale_t2 t2 ON t1.id = t2.id " + + "GROUP BY t1.category"); + + assertRefreshAndFullyMaterialized("test_agg_multi_stale_mv", 2); + + // Initial: A: 100*2=200, B: 200*3=600 + assertMaterializedViewQuery("SELECT * FROM test_agg_multi_stale_mv ORDER BY category", + "VALUES ('A', 200), ('B', 600)"); + + // Make t1 stale by inserting into new partition + assertUpdate("INSERT INTO test_agg_multi_stale_t1 VALUES " + + "(1, 'A', 150, DATE '2024-01-02'), " + + "(3, 'C', 300, DATE '2024-01-02')", 2); + + // Make t2 stale by inserting into new partition + assertUpdate("INSERT INTO test_agg_multi_stale_t2 VALUES " + + "(3, 'APAC', 4, DATE '2024-01-02')", 1); + + // Both tables are now stale. Verify results match full recomputation. + // Expected: A: (100*2)+(150*2)=500, B: 200*3=600, C: 300*4=1200 + assertMaterializedViewResultsMatch("SELECT * FROM test_agg_multi_stale_mv ORDER BY category"); + + assertMaterializedViewQuery("SELECT * FROM test_agg_multi_stale_mv ORDER BY category", + "VALUES ('A', 500), ('B', 600), ('C', 1200)"); + + assertUpdate("DROP MATERIALIZED VIEW test_agg_multi_stale_mv"); + assertUpdate("DROP TABLE test_agg_multi_stale_t2"); + assertUpdate("DROP TABLE test_agg_multi_stale_t1"); + } + + @Test + public void testSelectDistinctMVWithMultipleStaleTables() + { + // This test verifies SELECT DISTINCT MVs with multiple stale tables produce correct results. + // SELECT DISTINCT creates an AggregationNode in the plan, so the MarkDistinct optimization + // should skip deduplication. This tests that correctness is maintained. + assertUpdate("CREATE TABLE test_distinct_t1 (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("CREATE TABLE test_distinct_t2 (" + + "id BIGINT, " + + "region VARCHAR, " + + "code VARCHAR, " + + "reg_date DATE) " + + "WITH (partitioning = ARRAY['reg_date'])"); + + assertUpdate("INSERT INTO test_distinct_t1 VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'B', 200, DATE '2024-01-01')", 2); + + assertUpdate("INSERT INTO test_distinct_t2 VALUES " + + "(1, 'US', 'X', DATE '2024-01-01'), " + + "(2, 'EU', 'Y', DATE '2024-01-01')", 2); + + // Create MV with SELECT DISTINCT on a JOIN + assertUpdate("CREATE MATERIALIZED VIEW test_distinct_mv " + + "WITH (partitioning = ARRAY['event_date', 'reg_date']) AS " + + "SELECT DISTINCT t1.category, t2.region, t1.event_date, t2.reg_date " + + "FROM test_distinct_t1 t1 " + + "JOIN test_distinct_t2 t2 ON t1.id = t2.id"); + + assertRefreshAndFullyMaterialized("test_distinct_mv", 2); + + // Initial: (A, US), (B, EU) + assertMaterializedViewQuery("SELECT * FROM test_distinct_mv ORDER BY category", + "VALUES ('A', 'US', DATE '2024-01-01', DATE '2024-01-01'), " + + " ('B', 'EU', DATE '2024-01-01', DATE '2024-01-01')"); + + // Make t1 stale by inserting into new partition + assertUpdate("INSERT INTO test_distinct_t1 VALUES " + + "(1, 'A', 150, DATE '2024-01-02'), " + // Same category as before, should deduplicate + "(3, 'C', 300, DATE '2024-01-02')", 2); + + // Make t2 stale by inserting into new partition + assertUpdate("INSERT INTO test_distinct_t2 VALUES " + + "(3, 'APAC', 'Z', DATE '2024-01-02')", 1); + + // Both tables are now stale. Verify results match full recomputation. + // The DISTINCT should ensure no duplicates even with multiple stale branches. + assertMaterializedViewResultsMatch("SELECT * FROM test_distinct_mv ORDER BY category, region"); + + // Expected: (A, US, 01-01, 01-01) from old data + // (A, US, 01-02, 01-01) from new t1 row (id=1, event_date=01-02) joining old t2 row (id=1, reg_date=01-01) + // (B, EU, 01-01, 01-01) from old data + // (C, APAC, 01-02, 01-02) from new data on both sides + assertMaterializedViewQuery("SELECT * FROM test_distinct_mv ORDER BY category, region, event_date", + "VALUES ('A', 'US', DATE '2024-01-01', DATE '2024-01-01'), " + + " ('A', 'US', DATE '2024-01-02', DATE '2024-01-01'), " + + " ('B', 'EU', DATE '2024-01-01', DATE '2024-01-01'), " + + " ('C', 'APAC', DATE '2024-01-02', DATE '2024-01-02')"); + + assertUpdate("DROP MATERIALIZED VIEW test_distinct_mv"); + assertUpdate("DROP TABLE test_distinct_t2"); + assertUpdate("DROP TABLE test_distinct_t1"); + } + + @Test + public void testAggregatesWithNullValues() + { + assertUpdate("CREATE TABLE test_agg_nulls (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_agg_nulls VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', NULL, DATE '2024-01-01'), " + + "(3, 'B', 200, DATE '2024-01-01'), " + + "(4, 'B', NULL, DATE '2024-01-01')", 4); + + assertUpdate("CREATE MATERIALIZED VIEW test_agg_nulls_mv AS " + + "SELECT category, " + + " COUNT(*) as total_rows, " + + " COUNT(value) as non_null_count, " + + " SUM(value) as total, " + + " AVG(value) as average, " + + " MIN(value) as minimum, " + + " MAX(value) as maximum " + + "FROM test_agg_nulls " + + "GROUP BY category"); + + assertRefreshAndFullyMaterialized("test_agg_nulls_mv", 2); + + // Initial: A has 2 rows, 1 non-null, sum=100, avg=100, min=100, max=100 + // B has 2 rows, 1 non-null, sum=200, avg=200, min=200, max=200 + assertMaterializedViewQuery("SELECT category, total_rows, non_null_count, total, minimum, maximum FROM test_agg_nulls_mv ORDER BY category", + "VALUES ('A', 2, 1, 100, 100, 100), " + + " ('B', 2, 1, 200, 200, 200)"); + + // Insert more data with NULLs in new partition + assertUpdate("INSERT INTO test_agg_nulls VALUES " + + "(5, 'A', 150, DATE '2024-01-02'), " + + "(6, 'A', NULL, DATE '2024-01-02'), " + + "(7, 'B', NULL, DATE '2024-01-02')", 3); + + // A: 4 total, 2 non-null, sum=250, avg=125, min=100, max=150 + // B: 3 total, 1 non-null, sum=200, avg=200, min=200, max=200 + assertMaterializedViewQuery("SELECT category, total_rows, non_null_count, total, minimum, maximum FROM test_agg_nulls_mv ORDER BY category", + "VALUES ('A', 4, 2, 250, 100, 150), " + + " ('B', 3, 1, 200, 200, 200)"); + + assertUpdate("DROP MATERIALIZED VIEW test_agg_nulls_mv"); + assertUpdate("DROP TABLE test_agg_nulls"); + } + + @Test + public void testCountDistinctWithStaleTables() + { + assertUpdate("CREATE TABLE test_count_distinct (" + + "id BIGINT, " + + "category VARCHAR, " + + "user_id BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_count_distinct VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 100, DATE '2024-01-01'), " + + "(3, 'A', 200, DATE '2024-01-01'), " + + "(4, 'B', 300, DATE '2024-01-01'), " + + "(5, 'B', 300, DATE '2024-01-01')", 5); + + assertUpdate("CREATE MATERIALIZED VIEW test_count_distinct_mv AS " + + "SELECT category, " + + " COUNT(*) as total_events, " + + " COUNT(DISTINCT user_id) as unique_users " + + "FROM test_count_distinct " + + "GROUP BY category"); + + assertRefreshAndFullyMaterialized("test_count_distinct_mv", 2); + + // Initial: A has 3 events, 2 unique users; B has 2 events, 1 unique user + assertMaterializedViewQuery("SELECT * FROM test_count_distinct_mv ORDER BY category", + "VALUES ('A', 3, 2), " + + " ('B', 2, 1)"); + + // Insert new events with some duplicate users and some new users + assertUpdate("INSERT INTO test_count_distinct VALUES " + + "(6, 'A', 100, DATE '2024-01-02'), " + // Duplicate user 100 + "(7, 'A', 300, DATE '2024-01-02'), " + // New user 300 + "(8, 'B', 300, DATE '2024-01-02'), " + // Duplicate user 300 + "(9, 'B', 400, DATE '2024-01-02')", 4); // New user 400 + + // Expected: A has 5 events, 3 unique users (100, 200, 300) + // B has 4 events, 2 unique users (300, 400) + assertMaterializedViewQuery("SELECT * FROM test_count_distinct_mv ORDER BY category", + "VALUES ('A', 5, 3), " + + " ('B', 4, 2)"); + + assertUpdate("DROP MATERIALIZED VIEW test_count_distinct_mv"); + assertUpdate("DROP TABLE test_count_distinct"); + } + + @Test + public void testWindowFunctionSumPartitionNoStaleTables() + { + assertUpdate("CREATE TABLE test_window_base (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_window_base VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'B', 150, DATE '2024-01-01'), " + + "(4, 'B', 250, DATE '2024-01-01')", 4); + + assertUpdate("CREATE MATERIALIZED VIEW test_window_sum_mv AS " + + "SELECT id, category, value, " + + " SUM(value) OVER (PARTITION BY category) as category_total, " + + " COUNT(*) OVER (PARTITION BY category) as category_count " + + "FROM test_window_base"); + + assertRefreshAndFullyMaterialized("test_window_sum_mv", 4); + + // With no staleness, should return data from storage + // A: total=300, count=2 + // B: total=400, count=2 + assertMaterializedViewQuery("SELECT id, category, value, category_total, category_count FROM test_window_sum_mv ORDER BY id", + "VALUES (1, 'A', 100, 300, 2), " + + " (2, 'A', 200, 300, 2), " + + " (3, 'B', 150, 400, 2), " + + " (4, 'B', 250, 400, 2)"); + + assertUpdate("DROP MATERIALIZED VIEW test_window_sum_mv"); + assertUpdate("DROP TABLE test_window_base"); + } + + @Test + public void testWindowFunctionSumPartitionWithSingleStaleTable() + { + assertUpdate("CREATE TABLE test_window_stale1 (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_window_stale1 VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'B', 150, DATE '2024-01-01')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_window_stale1_mv AS " + + "SELECT id, category, value, " + + " SUM(value) OVER (PARTITION BY category) as category_total, " + + " COUNT(*) OVER (PARTITION BY category) as category_count " + + "FROM test_window_stale1"); + + assertRefreshAndFullyMaterialized("test_window_stale1_mv", 3); + + // Initial state: A total=300, count=2; B total=150, count=1 + assertMaterializedViewQuery("SELECT id, category, category_total, category_count FROM test_window_stale1_mv ORDER BY id", + "VALUES (1, 'A', 300, 2), (2, 'A', 300, 2), (3, 'B', 150, 1)"); + + // Insert into new partition to make table stale + assertUpdate("INSERT INTO test_window_stale1 VALUES " + + "(4, 'B', 75, DATE '2024-01-02'), " + + "(5, 'A', 300, DATE '2024-01-02')", 2); + + // Expected: A total=600, count=3; B total=225, count=2 + assertMaterializedViewQuery("SELECT id, category, category_total, category_count FROM test_window_stale1_mv ORDER BY id", + "VALUES (1, 'A', 600, 3), (2, 'A', 600, 3), (3, 'B', 225, 2), (4, 'B', 225, 2), (5, 'A', 600, 3)"); + + assertUpdate("DROP MATERIALIZED VIEW test_window_stale1_mv"); + assertUpdate("DROP TABLE test_window_stale1"); + } + + @Test + public void testWindowFunctionSumPartitionWithMultipleStaleTables() + { + assertUpdate("CREATE TABLE test_window_orders (" + + "order_id BIGINT, " + + "product_id BIGINT, " + + "amount BIGINT, " + + "order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_window_products (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "region VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + + assertUpdate("INSERT INTO test_window_orders VALUES " + + "(1, 100, 500, DATE '2024-01-01'), " + + "(2, 200, 300, DATE '2024-01-01'), " + + "(3, 100, 700, DATE '2024-01-01')", 3); + + assertUpdate("INSERT INTO test_window_products VALUES " + + "(100, 'Electronics', 'US'), " + + "(200, 'Books', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_window_join_mv AS " + + "SELECT o.order_id, p.category, o.amount, " + + " SUM(o.amount) OVER (PARTITION BY p.category) as category_total, " + + " AVG(o.amount) OVER (PARTITION BY p.category) as category_avg " + + "FROM test_window_orders o " + + "JOIN test_window_products p ON o.product_id = p.product_id"); + + assertRefreshAndFullyMaterialized("test_window_join_mv", 3); + + // Initial: Electronics total=1200, avg=600; Books total=300, avg=300 + assertMaterializedViewQuery("SELECT order_id, category, amount, category_total, category_avg FROM test_window_join_mv ORDER BY order_id", + "VALUES (1, 'Electronics', 500, 1200, 600.0), " + + " (2, 'Books', 300, 300, 300.0), " + + " (3, 'Electronics', 700, 1200, 600.0)"); + + // Make orders table stale + assertUpdate("INSERT INTO test_window_orders VALUES (4, 200, 250, DATE '2024-01-02')", 1); + + // Expected: Electronics total=1200, avg=600; Books total=550, avg=275 + assertMaterializedViewQuery("SELECT order_id, category, amount, category_total, category_avg FROM test_window_join_mv ORDER BY order_id", + "VALUES (1, 'Electronics', 500, 1200, 600.0), " + + " (2, 'Books', 300, 550, 275.0), " + + " (3, 'Electronics', 700, 1200, 600.0), " + + " (4, 'Books', 250, 550, 275.0)"); + + // Make products table also stale + assertUpdate("INSERT INTO test_window_products VALUES (300, 'Clothing', 'EU')", 1); + assertUpdate("INSERT INTO test_window_orders VALUES (5, 300, 400, DATE '2024-01-03')", 1); + + // Expected: Electronics total=1200, avg=600; Books total=550, avg=275; Clothing total=400, avg=400 + assertMaterializedViewQuery("SELECT order_id, category, amount, category_total, category_avg FROM test_window_join_mv ORDER BY order_id", + "VALUES (1, 'Electronics', 500, 1200, 600.0), " + + " (2, 'Books', 300, 550, 275.0), " + + " (3, 'Electronics', 700, 1200, 600.0), " + + " (4, 'Books', 250, 550, 275.0), " + + " (5, 'Clothing', 400, 400, 400.0)"); + + assertUpdate("DROP MATERIALIZED VIEW test_window_join_mv"); + assertUpdate("DROP TABLE test_window_products"); + assertUpdate("DROP TABLE test_window_orders"); + } + + @Test + public void testLeftJoinWithNoStaleTables() + { + assertUpdate("CREATE TABLE test_left_employees (" + + "emp_id BIGINT, " + + "emp_name VARCHAR, " + + "dept_id BIGINT, " + + "hire_date DATE) " + + "WITH (partitioning = ARRAY['hire_date'])"); + + assertUpdate("CREATE TABLE test_left_departments (" + + "dept_id BIGINT, " + + "dept_name VARCHAR, " + + "location VARCHAR) " + + "WITH (partitioning = ARRAY['location'])"); + + assertUpdate("INSERT INTO test_left_employees VALUES " + + "(1, 'Alice', 100, DATE '2024-01-01'), " + + "(2, 'Bob', 200, DATE '2024-01-01'), " + + "(3, 'Charlie', 300, DATE '2024-01-01'), " + + "(4, 'Dave', NULL, DATE '2024-01-01')", 4); + + assertUpdate("INSERT INTO test_left_departments VALUES " + + "(100, 'Engineering', 'US'), " + + "(200, 'Sales', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_left_join_mv AS " + + "SELECT e.emp_id, e.emp_name, e.dept_id, d.dept_name, d.location " + + "FROM test_left_employees e " + + "LEFT JOIN test_left_departments d ON e.dept_id = d.dept_id"); + + assertRefreshAndFullyMaterialized("test_left_join_mv", 4); + + // Verify results: Alice->Engineering, Bob->Sales, Charlie->NULL (dept 300 doesn't exist), Dave->NULL (NULL dept_id) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_id, dept_name, location FROM test_left_join_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 100, 'Engineering', 'US'), " + + " (2, 'Bob', 200, 'Sales', 'US'), " + + " (3, 'Charlie', 300, NULL, NULL), " + + " (4, 'Dave', NULL, NULL, NULL)"); + + assertUpdate("DROP MATERIALIZED VIEW test_left_join_mv"); + assertUpdate("DROP TABLE test_left_departments"); + assertUpdate("DROP TABLE test_left_employees"); + } + + @Test + public void testLeftJoinWithLeftTableStale() + { + assertUpdate("CREATE TABLE test_left_stale_employees (" + + "emp_id BIGINT, " + + "emp_name VARCHAR, " + + "dept_id BIGINT, " + + "hire_date DATE) " + + "WITH (partitioning = ARRAY['hire_date'])"); + + assertUpdate("CREATE TABLE test_left_stale_departments (" + + "dept_id BIGINT, " + + "dept_name VARCHAR, " + + "location VARCHAR) " + + "WITH (partitioning = ARRAY['location'])"); + + assertUpdate("INSERT INTO test_left_stale_employees VALUES " + + "(1, 'Alice', 100, DATE '2024-01-01'), " + + "(2, 'Bob', 200, DATE '2024-01-01')", 2); + + assertUpdate("INSERT INTO test_left_stale_departments VALUES " + + "(100, 'Engineering', 'US'), " + + "(200, 'Sales', 'US'), " + + "(300, 'Marketing', 'EU')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_left_stale_left_mv AS " + + "SELECT e.emp_id, e.emp_name, e.dept_id, e.hire_date, d.dept_name, d.location " + + "FROM test_left_stale_employees e " + + "LEFT JOIN test_left_stale_departments d ON e.dept_id = d.dept_id"); + + assertRefreshAndFullyMaterialized("test_left_stale_left_mv", 2); + + // Initial state: Alice->Engineering, Bob->Sales + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name, location FROM test_left_stale_left_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering', 'US'), " + + " (2, 'Bob', 'Sales', 'US')"); + + // Make left table (employees) stale by adding new employee in new partition + assertUpdate("INSERT INTO test_left_stale_employees VALUES " + + "(3, 'Charlie', 300, DATE '2024-01-02'), " + + "(4, 'Dave', NULL, DATE '2024-01-02'), " + + "(5, 'Eve', 400, DATE '2024-01-02')", 3); + + // Expected: Charlie joins with Marketing, Dave has NULL dept (no match), Eve has NULL (dept 400 doesn't exist) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_id, dept_name, location FROM test_left_stale_left_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 100, 'Engineering', 'US'), " + + " (2, 'Bob', 200, 'Sales', 'US'), " + + " (3, 'Charlie', 300, 'Marketing', 'EU'), " + + " (4, 'Dave', NULL, NULL, NULL), " + + " (5, 'Eve', 400, NULL, NULL)"); + + assertUpdate("DROP MATERIALIZED VIEW test_left_stale_left_mv"); + assertUpdate("DROP TABLE test_left_stale_departments"); + assertUpdate("DROP TABLE test_left_stale_employees"); + } + + @Test + public void testLeftJoinWithRightTableStale() + { + assertUpdate("CREATE TABLE test_right_stale_employees (" + + "emp_id BIGINT, " + + "emp_name VARCHAR, " + + "dept_id BIGINT, " + + "hire_date DATE) " + + "WITH (partitioning = ARRAY['hire_date'])"); + + assertUpdate("CREATE TABLE test_right_stale_departments (" + + "dept_id BIGINT, " + + "dept_name VARCHAR, " + + "location VARCHAR) " + + "WITH (partitioning = ARRAY['location'])"); + + assertUpdate("INSERT INTO test_right_stale_employees VALUES " + + "(1, 'Alice', 100, DATE '2024-01-01'), " + + "(2, 'Bob', 200, DATE '2024-01-01'), " + + "(3, 'Charlie', 300, DATE '2024-01-01')", 3); + + assertUpdate("INSERT INTO test_right_stale_departments VALUES " + + "(100, 'Engineering', 'US'), " + + "(200, 'Sales', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_right_stale_mv AS " + + "SELECT e.emp_id, e.emp_name, e.dept_id, e.hire_date, d.dept_name, d.location " + + "FROM test_right_stale_employees e " + + "LEFT JOIN test_right_stale_departments d ON e.dept_id = d.dept_id"); + + assertRefreshAndFullyMaterialized("test_right_stale_mv", 3); + + // Initial state: Alice->Engineering, Bob->Sales, Charlie->NULL (dept 300 doesn't exist) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name, location FROM test_right_stale_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering', 'US'), " + + " (2, 'Bob', 'Sales', 'US'), " + + " (3, 'Charlie', NULL, NULL)"); + + // Make right table (departments) stale by adding new department in new partition + assertUpdate("INSERT INTO test_right_stale_departments VALUES " + + "(300, 'Marketing', 'EU'), " + + "(400, 'HR', 'EU')", 2); + + // Expected: Charlie now joins with Marketing; HR dept has no matching employees (LEFT JOIN preserves left table) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_id, dept_name, location FROM test_right_stale_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 100, 'Engineering', 'US'), " + + " (2, 'Bob', 200, 'Sales', 'US'), " + + " (3, 'Charlie', 300, 'Marketing', 'EU')"); + + assertUpdate("DROP MATERIALIZED VIEW test_right_stale_mv"); + assertUpdate("DROP TABLE test_right_stale_departments"); + assertUpdate("DROP TABLE test_right_stale_employees"); + } + + @Test + public void testLeftJoinWithBothTablesStale() + { + assertUpdate("CREATE TABLE test_both_stale_employees (" + + "emp_id BIGINT, " + + "emp_name VARCHAR, " + + "dept_id BIGINT, " + + "hire_date DATE) " + + "WITH (partitioning = ARRAY['hire_date'])"); + + assertUpdate("CREATE TABLE test_both_stale_departments (" + + "dept_id BIGINT, " + + "dept_name VARCHAR, " + + "location VARCHAR) " + + "WITH (partitioning = ARRAY['location'])"); + + assertUpdate("INSERT INTO test_both_stale_employees VALUES " + + "(1, 'Alice', 100, DATE '2024-01-01'), " + + "(2, 'Bob', 200, DATE '2024-01-01')", 2); + + assertUpdate("INSERT INTO test_both_stale_departments VALUES " + + "(100, 'Engineering', 'US'), " + + "(200, 'Sales', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_both_stale_mv AS " + + "SELECT e.emp_id, e.emp_name, e.dept_id, e.hire_date, d.dept_name, d.location " + + "FROM test_both_stale_employees e " + + "LEFT JOIN test_both_stale_departments d ON e.dept_id = d.dept_id"); + + assertRefreshAndFullyMaterialized("test_both_stale_mv", 2); + + // Initial state: Alice->Engineering, Bob->Sales + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name, location FROM test_both_stale_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering', 'US'), " + + " (2, 'Bob', 'Sales', 'US')"); + + // Make both tables stale + assertUpdate("INSERT INTO test_both_stale_employees VALUES " + + "(3, 'Charlie', 300, DATE '2024-01-02'), " + + "(4, 'Dave', NULL, DATE '2024-01-02')", 2); + + assertUpdate("INSERT INTO test_both_stale_departments VALUES " + + "(300, 'Marketing', 'EU'), " + + "(400, 'HR', 'EU')", 2); + + // Expected: Charlie joins with Marketing, Dave has NULL (no match), HR dept has no employees + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_id, dept_name, location FROM test_both_stale_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 100, 'Engineering', 'US'), " + + " (2, 'Bob', 200, 'Sales', 'US'), " + + " (3, 'Charlie', 300, 'Marketing', 'EU'), " + + " (4, 'Dave', NULL, NULL, NULL)"); + + assertUpdate("DROP MATERIALIZED VIEW test_both_stale_mv"); + assertUpdate("DROP TABLE test_both_stale_departments"); + assertUpdate("DROP TABLE test_both_stale_employees"); + } + + @Test + public void testLeftJoinWithAggregationAndMultipleStaleTables() + { + assertUpdate("CREATE TABLE test_agg_orders (" + + "order_id BIGINT, " + + "customer_id BIGINT, " + + "amount BIGINT, " + + "order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_agg_customers (" + + "customer_id BIGINT, " + + "customer_name VARCHAR, " + + "region VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + + assertUpdate("INSERT INTO test_agg_orders VALUES " + + "(1, 100, 500, DATE '2024-01-01'), " + + "(2, 100, 300, DATE '2024-01-01'), " + + "(3, 200, 700, DATE '2024-01-01'), " + + "(4, 300, 150, DATE '2024-01-01')", 4); + + assertUpdate("INSERT INTO test_agg_customers VALUES " + + "(100, 'Alice', 'US'), " + + "(200, 'Bob', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_left_agg_mv AS " + + "SELECT c.customer_id, c.customer_name, c.region, " + + " COUNT(o.order_id) as order_count, " + + " SUM(o.amount) as total_amount, " + + " AVG(o.amount) as avg_amount " + + "FROM test_agg_customers c " + + "LEFT JOIN test_agg_orders o ON c.customer_id = o.customer_id " + + "GROUP BY c.customer_id, c.customer_name, c.region"); + + assertRefreshAndFullyMaterialized("test_left_agg_mv", 2); + + // Initial: Alice has 2 orders (800 total, 400 avg), Bob has 1 order (700 total, 700 avg) + // Customer 300 doesn't exist in customers table, so order 4 doesn't appear in LEFT JOIN result + assertMaterializedViewQuery("SELECT customer_id, customer_name, order_count, total_amount, avg_amount FROM test_left_agg_mv ORDER BY customer_id", + "VALUES (100, 'Alice', 2, 800, 400.0), " + + " (200, 'Bob', 1, 700, 700.0)"); + + // Make orders table stale by adding new orders + assertUpdate("INSERT INTO test_agg_orders VALUES " + + "(5, 100, 200, DATE '2024-01-02'), " + + "(6, 300, 150, DATE '2024-01-02')", 2); + + // Alice now has 3 orders (1000 total), Bob still has 1 order + // Customer 300's order is still excluded (customer 300 doesn't exist yet) + assertMaterializedViewQuery("SELECT customer_id, customer_name, order_count, total_amount FROM test_left_agg_mv ORDER BY customer_id", + "VALUES (100, 'Alice', 3, 1000), " + + " (200, 'Bob', 1, 700)"); + + // Make customers table also stale - add customers 300 and 400 + assertUpdate("INSERT INTO test_agg_customers VALUES " + + "(300, 'Charlie', 'EU'), " + + "(400, 'Dave', 'EU')", 2); + + // Charlie now appears with 2 orders (150 from order 4 + 150 from order 6 = 300 total) + // Dave appears with 0 orders (no matching orders) + assertMaterializedViewQuery("SELECT customer_id, customer_name, order_count, total_amount FROM test_left_agg_mv ORDER BY customer_id", + "VALUES (100, 'Alice', 3, 1000), " + + " (200, 'Bob', 1, 700), " + + " (300, 'Charlie', 2, 300), " + + " (400, 'Dave', 0, NULL)"); + + assertUpdate("DROP MATERIALIZED VIEW test_left_agg_mv"); + assertUpdate("DROP TABLE test_agg_customers"); + assertUpdate("DROP TABLE test_agg_orders"); + } + + @Test + public void testLeftJoinWithWhereClauseFilteringNulls() + { + assertUpdate("CREATE TABLE test_filter_employees (" + + "emp_id BIGINT, " + + "emp_name VARCHAR, " + + "dept_id BIGINT, " + + "hire_date DATE) " + + "WITH (partitioning = ARRAY['hire_date'])"); + + assertUpdate("CREATE TABLE test_filter_departments (" + + "dept_id BIGINT, " + + "dept_name VARCHAR, " + + "location VARCHAR) " + + "WITH (partitioning = ARRAY['location'])"); + + assertUpdate("INSERT INTO test_filter_employees VALUES " + + "(1, 'Alice', 100, DATE '2024-01-01'), " + + "(2, 'Bob', 200, DATE '2024-01-01'), " + + "(3, 'Charlie', 300, DATE '2024-01-01'), " + + "(4, 'Dave', NULL, DATE '2024-01-01')", 4); + + assertUpdate("INSERT INTO test_filter_departments VALUES " + + "(100, 'Engineering', 'US'), " + + "(200, 'Sales', 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_filter_nulls_mv AS " + + "SELECT e.emp_id, e.emp_name, e.dept_id, e.hire_date, d.dept_name, d.location " + + "FROM test_filter_employees e " + + "LEFT JOIN test_filter_departments d ON e.dept_id = d.dept_id " + + "WHERE d.dept_name IS NOT NULL"); + + assertRefreshAndFullyMaterialized("test_filter_nulls_mv", 2); + + // Initial: Only Alice and Bob (Charlie and Dave filtered out by WHERE clause) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name FROM test_filter_nulls_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering'), " + + " (2, 'Bob', 'Sales')"); + + // Make employees table stale + assertUpdate("INSERT INTO test_filter_employees VALUES " + + "(5, 'Eve', 100, DATE '2024-01-02'), " + + "(6, 'Frank', 400, DATE '2024-01-02')", 2); + + // Eve joins with Engineering, Frank is filtered out (dept 400 doesn't exist) + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name FROM test_filter_nulls_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering'), " + + " (2, 'Bob', 'Sales'), " + + " (5, 'Eve', 'Engineering')"); + + // Make departments table stale + assertUpdate("INSERT INTO test_filter_departments VALUES " + + "(300, 'Marketing', 'EU')", 1); + + // Charlie now joins with Marketing and appears in results + assertMaterializedViewQuery("SELECT emp_id, emp_name, dept_name FROM test_filter_nulls_mv ORDER BY emp_id", + "VALUES (1, 'Alice', 'Engineering'), " + + " (2, 'Bob', 'Sales'), " + + " (3, 'Charlie', 'Marketing'), " + + " (5, 'Eve', 'Engineering')"); + + assertUpdate("DROP MATERIALIZED VIEW test_filter_nulls_mv"); + assertUpdate("DROP TABLE test_filter_departments"); + assertUpdate("DROP TABLE test_filter_employees"); + } + + @Test + public void testGroupByHavingWithNoStaleTables() + { + assertUpdate("CREATE TABLE test_having_sales (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "revenue BIGINT, " + + "sale_date DATE) " + + "WITH (partitioning = ARRAY['sale_date'])"); + + assertUpdate("INSERT INTO test_having_sales VALUES " + + "(1, 'Electronics', 1000, DATE '2024-01-01'), " + + "(2, 'Electronics', 1500, DATE '2024-01-01'), " + + "(3, 'Books', 300, DATE '2024-01-01'), " + + "(4, 'Books', 200, DATE '2024-01-01'), " + + "(5, 'Clothing', 800, DATE '2024-01-01')", 5); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_mv AS " + + "SELECT category, SUM(revenue) as total_revenue, COUNT(*) as sale_count " + + "FROM test_having_sales " + + "GROUP BY category " + + "HAVING SUM(revenue) > 500"); + + assertRefreshAndFullyMaterialized("test_having_mv", 2); + + // Only Electronics (2500) and Clothing (800) pass the HAVING filter + // Books (500) is filtered out + assertMaterializedViewQuery("SELECT category, total_revenue, sale_count FROM test_having_mv ORDER BY category", + "VALUES ('Clothing', 800, 1), " + + " ('Electronics', 2500, 2)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_mv"); + assertUpdate("DROP TABLE test_having_sales"); + } + + @Test + public void testGroupByHavingWithSingleStaleTable() + { + assertUpdate("CREATE TABLE test_having_stale1 (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "revenue BIGINT, " + + "sale_date DATE) " + + "WITH (partitioning = ARRAY['sale_date'])"); + + assertUpdate("INSERT INTO test_having_stale1 VALUES " + + "(1, 'Electronics', 600, DATE '2024-01-01'), " + + "(2, 'Books', 400, DATE '2024-01-01'), " + + "(3, 'Clothing', 300, DATE '2024-01-01')", 3); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_stale1_mv AS " + + "SELECT category, SUM(revenue) as total_revenue, COUNT(*) as sale_count " + + "FROM test_having_stale1 " + + "GROUP BY category " + + "HAVING SUM(revenue) >= 500"); + + assertRefreshAndFullyMaterialized("test_having_stale1_mv", 1); + + // Initial: Only Electronics (600) passes HAVING filter + assertMaterializedViewQuery("SELECT category, total_revenue, sale_count FROM test_having_stale1_mv ORDER BY category", + "VALUES ('Electronics', 600, 1)"); + + // Make table stale - add sales that change HAVING results + assertUpdate("INSERT INTO test_having_stale1 VALUES " + + "(4, 'Books', 200, DATE '2024-01-02'), " + + "(5, 'Clothing', 400, DATE '2024-01-02'), " + + "(6, 'Toys', 800, DATE '2024-01-02')", 3); + + // Expected: Books now 600 (passes), Clothing now 700 (passes), Toys 800 (passes), Electronics still 600 + assertMaterializedViewQuery("SELECT category, total_revenue, sale_count FROM test_having_stale1_mv ORDER BY category", + "VALUES ('Books', 600, 2), " + + " ('Clothing', 700, 2), " + + " ('Electronics', 600, 1), " + + " ('Toys', 800, 1)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_stale1_mv"); + assertUpdate("DROP TABLE test_having_stale1"); + } + + @Test + public void testGroupByHavingWithMultipleStaleTables() + { + assertUpdate("CREATE TABLE test_having_orders (" + + "order_id BIGINT, " + + "product_id BIGINT, " + + "quantity BIGINT, " + + "order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + + assertUpdate("CREATE TABLE test_having_products (" + + "product_id BIGINT, " + + "category VARCHAR, " + + "price BIGINT, " + + "region VARCHAR) " + + "WITH (partitioning = ARRAY['region'])"); + + assertUpdate("INSERT INTO test_having_orders VALUES " + + "(1, 100, 10, DATE '2024-01-01'), " + + "(2, 200, 5, DATE '2024-01-01'), " + + "(3, 100, 8, DATE '2024-01-01')", 3); + + assertUpdate("INSERT INTO test_having_products VALUES " + + "(100, 'Electronics', 50, 'US'), " + + "(200, 'Books', 20, 'US')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_join_mv AS " + + "SELECT p.category, SUM(o.quantity * p.price) as total_revenue, COUNT(*) as order_count " + + "FROM test_having_orders o " + + "JOIN test_having_products p ON o.product_id = p.product_id " + + "GROUP BY p.category " + + "HAVING SUM(o.quantity * p.price) > 200"); + + assertRefreshAndFullyMaterialized("test_having_join_mv", 1); + + // Initial: Electronics has revenue 900 (10*50 + 8*50), Books has 100 (5*20) + // Only Electronics passes HAVING > 200 + assertMaterializedViewQuery("SELECT category, total_revenue, order_count FROM test_having_join_mv ORDER BY category", + "VALUES ('Electronics', 900, 2)"); + + // Make orders table stale + assertUpdate("INSERT INTO test_having_orders VALUES " + + "(4, 200, 15, DATE '2024-01-02')", 1); + + // Books now has revenue 400 (5*20 + 15*20), now passes HAVING filter + assertMaterializedViewQuery("SELECT category, total_revenue, order_count FROM test_having_join_mv ORDER BY category", + "VALUES ('Books', 400, 2), " + + " ('Electronics', 900, 2)"); + + // Make products table also stale + assertUpdate("INSERT INTO test_having_products VALUES " + + "(300, 'Toys', 30, 'EU')", 1); + assertUpdate("INSERT INTO test_having_orders VALUES " + + "(5, 300, 10, DATE '2024-01-03')", 1); + + // Toys has revenue 300 (10*30), passes HAVING filter + assertMaterializedViewQuery("SELECT category, total_revenue, order_count FROM test_having_join_mv ORDER BY category", + "VALUES ('Books', 400, 2), " + + " ('Electronics', 900, 2), " + + " ('Toys', 300, 1)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_join_mv"); + assertUpdate("DROP TABLE test_having_products"); + assertUpdate("DROP TABLE test_having_orders"); + } + + @Test + public void testGroupByHavingWithSumCount() + { + assertUpdate("CREATE TABLE test_having_sumcount (" + + "id BIGINT, " + + "category VARCHAR, " + + "amount BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_having_sumcount VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'A', 300, DATE '2024-01-01'), " + + "(4, 'B', 500, DATE '2024-01-01'), " + + "(5, 'B', 600, DATE '2024-01-01'), " + + "(6, 'C', 1000, DATE '2024-01-01')", 6); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_sumcount_mv AS " + + "SELECT category, SUM(amount) as total, COUNT(*) as cnt " + + "FROM test_having_sumcount " + + "GROUP BY category " + + "HAVING SUM(amount) > 500 AND COUNT(*) >= 2"); + + assertRefreshAndFullyMaterialized("test_having_sumcount_mv", 2); + + // A: sum=600, count=3 (passes both conditions) + // B: sum=1100, count=2 (passes both conditions) + // C: sum=1000, count=1 (fails count condition) + assertMaterializedViewQuery("SELECT category, total, cnt FROM test_having_sumcount_mv ORDER BY category", + "VALUES ('A', 600, 3), " + + " ('B', 1100, 2)"); + + // Make table stale + assertUpdate("INSERT INTO test_having_sumcount VALUES " + + "(7, 'C', 100, DATE '2024-01-02'), " + + "(8, 'D', 300, DATE '2024-01-02'), " + + "(9, 'D', 400, DATE '2024-01-02')", 3); + + // C: sum=1100, count=2 (now passes both conditions) + // D: sum=700, count=2 (passes both conditions) + assertMaterializedViewQuery("SELECT category, total, cnt FROM test_having_sumcount_mv ORDER BY category", + "VALUES ('A', 600, 3), " + + " ('B', 1100, 2), " + + " ('C', 1100, 2), " + + " ('D', 700, 2)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_sumcount_mv"); + assertUpdate("DROP TABLE test_having_sumcount"); + } + + @Test + public void testGroupByHavingWithAvg() + { + assertUpdate("CREATE TABLE test_having_avg (" + + "id BIGINT, " + + "category VARCHAR, " + + "value BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_having_avg VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'A', 300, DATE '2024-01-01'), " + + "(4, 'B', 50, DATE '2024-01-01'), " + + "(5, 'B', 100, DATE '2024-01-01'), " + + "(6, 'C', 500, DATE '2024-01-01')", 6); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_avg_mv AS " + + "SELECT category, AVG(value) as avg_value, COUNT(*) as cnt " + + "FROM test_having_avg " + + "GROUP BY category " + + "HAVING AVG(value) > 100"); + + assertRefreshAndFullyMaterialized("test_having_avg_mv", 2); + + // A: avg=200 (passes) + // B: avg=75 (fails) + // C: avg=500 (passes) + assertMaterializedViewQuery("SELECT category, avg_value, cnt FROM test_having_avg_mv ORDER BY category", + "VALUES ('A', 200.0, 3), " + + " ('C', 500.0, 1)"); + + // Make table stale - add values that change averages + assertUpdate("INSERT INTO test_having_avg VALUES " + + "(7, 'B', 150, DATE '2024-01-02'), " + + "(8, 'B', 200, DATE '2024-01-02'), " + + "(9, 'D', 250, DATE '2024-01-02')", 3); + + // B: avg=(50+100+150+200)/4=125 (now passes) + // D: avg=250 (passes) + assertMaterializedViewQuery("SELECT category, avg_value, cnt FROM test_having_avg_mv ORDER BY category", + "VALUES ('A', 200.0, 3), " + + " ('B', 125.0, 4), " + + " ('C', 500.0, 1), " + + " ('D', 250.0, 1)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_avg_mv"); + assertUpdate("DROP TABLE test_having_avg"); + } + + @Test + public void testGroupByHavingWithCountDistinct() + { + assertUpdate("CREATE TABLE test_having_distinct (" + + "id BIGINT, " + + "category VARCHAR, " + + "user_id BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_having_distinct VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 100, DATE '2024-01-01'), " + + "(3, 'A', 200, DATE '2024-01-01'), " + + "(4, 'A', 200, DATE '2024-01-01'), " + + "(5, 'B', 300, DATE '2024-01-01'), " + + "(6, 'B', 300, DATE '2024-01-01'), " + + "(7, 'C', 400, DATE '2024-01-01'), " + + "(8, 'C', 500, DATE '2024-01-01'), " + + "(9, 'C', 600, DATE '2024-01-01')", 9); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_distinct_mv AS " + + "SELECT category, COUNT(DISTINCT user_id) as unique_users, COUNT(*) as total_events " + + "FROM test_having_distinct " + + "GROUP BY category " + + "HAVING COUNT(DISTINCT user_id) >= 2"); + + assertRefreshAndFullyMaterialized("test_having_distinct_mv", 2); + + // A: 2 unique users (passes) + // B: 1 unique user (fails) + // C: 3 unique users (passes) + assertMaterializedViewQuery("SELECT category, unique_users, total_events FROM test_having_distinct_mv ORDER BY category", + "VALUES ('A', 2, 4), " + + " ('C', 3, 3)"); + + // Make table stale + assertUpdate("INSERT INTO test_having_distinct VALUES " + + "(10, 'B', 400, DATE '2024-01-02'), " + + "(11, 'D', 700, DATE '2024-01-02'), " + + "(12, 'D', 800, DATE '2024-01-02')", 3); + + // B: 2 unique users (now passes) + // D: 2 unique users (passes) + assertMaterializedViewQuery("SELECT category, unique_users, total_events FROM test_having_distinct_mv ORDER BY category", + "VALUES ('A', 2, 4), " + + " ('B', 2, 3), " + + " ('C', 3, 3), " + + " ('D', 2, 2)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_distinct_mv"); + assertUpdate("DROP TABLE test_having_distinct"); + } + + @Test + public void testGroupByHavingWithMultipleConditions() + { + assertUpdate("CREATE TABLE test_having_multi (" + + "id BIGINT, " + + "category VARCHAR, " + + "amount BIGINT, " + + "event_date DATE) " + + "WITH (partitioning = ARRAY['event_date'])"); + + assertUpdate("INSERT INTO test_having_multi VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'A', 300, DATE '2024-01-01'), " + + "(4, 'A', 400, DATE '2024-01-01'), " + + "(5, 'B', 800, DATE '2024-01-01'), " + + "(6, 'B', 900, DATE '2024-01-01'), " + + "(7, 'C', 150, DATE '2024-01-01'), " + + "(8, 'C', 250, DATE '2024-01-01'), " + + "(9, 'D', 2000, DATE '2024-01-01')", 9); + + assertUpdate("CREATE MATERIALIZED VIEW test_having_multi_mv AS " + + "SELECT category, " + + " SUM(amount) as total, " + + " AVG(amount) as average, " + + " COUNT(*) as cnt, " + + " MIN(amount) as minimum, " + + " MAX(amount) as maximum " + + "FROM test_having_multi " + + "GROUP BY category " + + "HAVING SUM(amount) > 500 AND AVG(amount) >= 200 AND COUNT(*) >= 2"); + + assertRefreshAndFullyMaterialized("test_having_multi_mv", 2); + + // A: sum=1000, avg=250, cnt=4, min=100, max=400 (passes all) + // B: sum=1700, avg=850, cnt=2, min=800, max=900 (passes all) + // C: sum=400, avg=200, cnt=2 (fails sum condition) + // D: sum=2000, avg=2000, cnt=1 (fails cnt condition) + assertMaterializedViewQuery("SELECT category, total, average, cnt, minimum, maximum FROM test_having_multi_mv ORDER BY category", + "VALUES ('A', 1000, 250.0, 4, 100, 400), " + + " ('B', 1700, 850.0, 2, 800, 900)"); + + // Make table stale + assertUpdate("INSERT INTO test_having_multi VALUES " + + "(10, 'C', 350, DATE '2024-01-02'), " + + "(11, 'D', 500, DATE '2024-01-02'), " + + "(12, 'E', 300, DATE '2024-01-02'), " + + "(13, 'E', 400, DATE '2024-01-02')", 4); + + // C: sum=750, avg=250, cnt=3 (now passes all) + // D: sum=2500, avg=1250, cnt=2 (now passes all) + // E: sum=700, avg=350, cnt=2 (passes all) + assertMaterializedViewQuery("SELECT category, total, average, cnt, minimum, maximum FROM test_having_multi_mv ORDER BY category", + "VALUES ('A', 1000, 250.0, 4, 100, 400), " + + " ('B', 1700, 850.0, 2, 800, 900), " + + " ('C', 750, 250.0, 3, 150, 350), " + + " ('D', 2500, 1250.0, 2, 500, 2000), " + + " ('E', 700, 350.0, 2, 300, 400)"); + + assertUpdate("DROP MATERIALIZED VIEW test_having_multi_mv"); + assertUpdate("DROP TABLE test_having_multi"); + } + + /** + * Data provider for non-deterministic function tests. + * Returns: [function expression, description, needs base tables] + */ + @DataProvider(name = "nonDeterministicFunctions") + public Object[][] nonDeterministicFunctions() + { + return new Object[][] { + {"RAND()", "RAND() in SELECT", true}, + {"NOW()", "NOW() in SELECT", true}, + {"CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP in SELECT", true}, + }; + } + + /** + * Test that non-deterministic functions in SELECT cause fallback to full recompute. + * Materialized views with non-deterministic functions should never use stitching because: + * - Fresh branch would compute RAND()/NOW()/UUID() at read time + * - Stale branch would compute different values + * - Results would be inconsistent + */ + @Test(dataProvider = "nonDeterministicFunctions") + public void testNonDeterministicFunctionInSelect(String functionExpr, String description, boolean needsBaseTables) + { + // Create base table + assertUpdate("CREATE TABLE test_nondeterministic_base (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_nondeterministic_base VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + + // Create MV with non-deterministic function + assertUpdate("CREATE MATERIALIZED VIEW test_nondeterministic_mv AS " + + "SELECT id, value, " + functionExpr + " as nondeterministic_value, dt FROM test_nondeterministic_base"); + + // Initial refresh + assertRefreshAndFullyMaterialized("test_nondeterministic_mv", 2); + + // Introduce staleness + assertUpdate("INSERT INTO test_nondeterministic_base VALUES (3, 300, DATE '2024-01-02')", 1); + + // Query should fall back to full recompute (not use stitching) + // We verify this by checking that the query succeeds and returns correct row count + // The optimizer will automatically use full recompute instead of stitching + assertQuery("SELECT id, value FROM test_nondeterministic_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_nondeterministic_mv"); + assertUpdate("DROP TABLE test_nondeterministic_base"); + } + + /** + * Test that non-deterministic functions in WHERE clause cause fallback to full recompute. + */ + @Test(dataProvider = "nonDeterministicFunctions") + public void testNonDeterministicFunctionInWhere(String functionExpr, String description, boolean needsBaseTables) + { + // Skip functions that can't be used in WHERE clause (UUID returns VARCHAR, can't compare with numbers easily) + if (functionExpr.equals("UUID()")) { + return; + } + + // Create base table + assertUpdate("CREATE TABLE test_nondeterministic_where (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_nondeterministic_where VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + + // Create MV with non-deterministic function in WHERE clause + // Use a condition that's always true but contains the non-deterministic function + String whereClause = functionExpr.contains("RAND") ? "RAND() >= 0" : functionExpr + " IS NOT NULL"; + assertUpdate("CREATE MATERIALIZED VIEW test_nondeterministic_where_mv AS " + + "SELECT id, value, dt FROM test_nondeterministic_where WHERE " + whereClause); + + // Initial refresh + assertRefreshAndFullyMaterialized("test_nondeterministic_where_mv", 2); + + // Introduce staleness + assertUpdate("INSERT INTO test_nondeterministic_where VALUES (3, 300, DATE '2024-01-02')", 1); + + // Query should fall back to full recompute + assertQuery("SELECT id, value FROM test_nondeterministic_where_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW test_nondeterministic_where_mv"); + assertUpdate("DROP TABLE test_nondeterministic_where"); + } + + /** + * Test that non-deterministic functions in aggregation cause fallback to full recompute. + */ + @Test + public void testNonDeterministicFunctionInAggregation() + { + // Create base table + assertUpdate("CREATE TABLE test_nondeterministic_agg (id INTEGER, category VARCHAR, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("INSERT INTO test_nondeterministic_agg VALUES " + + "(1, 'A', 100, DATE '2024-01-01'), " + + "(2, 'A', 200, DATE '2024-01-01'), " + + "(3, 'B', 300, DATE '2024-01-01')", 3); + + // Create MV with non-deterministic function in aggregation + // Use RAND() to create a non-deterministic computed value before aggregation + assertUpdate("CREATE MATERIALIZED VIEW test_nondeterministic_agg_mv AS " + + "SELECT category, SUM(value * (1 + RAND())) as total FROM test_nondeterministic_agg GROUP BY category"); + + // Initial refresh + assertRefreshAndFullyMaterialized("test_nondeterministic_agg_mv", 2); + + // Introduce staleness + assertUpdate("INSERT INTO test_nondeterministic_agg VALUES (4, 'A', 400, DATE '2024-01-02')", 1); + + // Query should fall back to full recompute + // We just verify it returns the expected categories (values will vary due to RAND()) + assertQuery("SELECT category FROM test_nondeterministic_agg_mv ORDER BY category", + "VALUES ('A'), ('B')"); + + assertUpdate("DROP MATERIALIZED VIEW test_nondeterministic_agg_mv"); + assertUpdate("DROP TABLE test_nondeterministic_agg"); + } + + @Test + public void testUnionInMVDefinitionWithNoStaleTables() + { + assertUpdate("CREATE TABLE test_union_base1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_union_base2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_union_base1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO test_union_base2 VALUES (3, 300, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 2); + + // Create MV with UNION + assertUpdate("CREATE MATERIALIZED VIEW test_union_mv AS " + + "SELECT id, value, dt FROM test_union_base1 " + + "UNION " + + "SELECT id, value, dt FROM test_union_base2"); + + assertRefreshAndFullyMaterialized("test_union_mv", 4); + + // Query MV - all data is fresh, no stitching needed + assertQuery("SELECT id, value FROM test_union_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300), (4, 400)"); + + assertUpdate("DROP MATERIALIZED VIEW test_union_mv"); + assertUpdate("DROP TABLE test_union_base1"); + assertUpdate("DROP TABLE test_union_base2"); + } + + @Test + public void testUnionInMVDefinitionWithSingleStaleTable() + { + assertUpdate("CREATE TABLE test_union_stale1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_union_stale2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_union_stale1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO test_union_stale2 VALUES (3, 300, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 2); + + // Create MV with UNION + assertUpdate("CREATE MATERIALIZED VIEW test_union_stale_mv AS " + + "SELECT id, value, dt FROM test_union_stale1 " + + "UNION " + + "SELECT id, value, dt FROM test_union_stale2"); + + assertRefreshAndFullyMaterialized("test_union_stale_mv", 4); + + // Insert into one table to make it stale + assertUpdate("INSERT INTO test_union_stale1 VALUES (5, 500, DATE '2024-01-02')", 1); + + // Query MV - should stitch fresh from MV storage with stale from base table + assertQuery("SELECT id, value FROM test_union_stale_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300), (4, 400), (5, 500)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_union_stale_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_union_stale_mv"); + assertUpdate("DROP TABLE test_union_stale1"); + assertUpdate("DROP TABLE test_union_stale2"); + } + + @Test + public void testUnionInMVDefinitionWithMultipleStaleTables() + { + assertUpdate("CREATE TABLE test_union_multi1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_union_multi2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_union_multi1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO test_union_multi2 VALUES (3, 300, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 2); + + // Create MV with UNION + assertUpdate("CREATE MATERIALIZED VIEW test_union_multi_mv AS " + + "SELECT id, value, dt FROM test_union_multi1 " + + "UNION " + + "SELECT id, value, dt FROM test_union_multi2"); + + assertRefreshAndFullyMaterialized("test_union_multi_mv", 4); + + // Insert into both tables to make both stale + assertUpdate("INSERT INTO test_union_multi1 VALUES (5, 500, DATE '2024-01-02')", 1); + assertUpdate("INSERT INTO test_union_multi2 VALUES (6, 600, DATE '2024-01-02')", 1); + + // Query MV - should stitch fresh from MV storage with stale from both base tables + assertQuery("SELECT id, value FROM test_union_multi_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300), (4, 400), (5, 500), (6, 600)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_union_multi_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_union_multi_mv"); + assertUpdate("DROP TABLE test_union_multi1"); + assertUpdate("DROP TABLE test_union_multi2"); + } + + @Test + public void testUnionAllVsUnionDistinct() + { + assertUpdate("CREATE TABLE test_union_type1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_union_type2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Insert data with potential duplicates + assertUpdate("INSERT INTO test_union_type1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO test_union_type2 VALUES (1, 100, DATE '2024-01-01'), (3, 300, DATE '2024-01-01')", 2); + + // Test UNION (implicit DISTINCT) + assertUpdate("CREATE MATERIALIZED VIEW test_union_distinct_mv AS " + + "SELECT id, value, dt FROM test_union_type1 " + + "UNION " + + "SELECT id, value, dt FROM test_union_type2"); + + assertRefreshAndFullyMaterialized("test_union_distinct_mv", 3); + + // Test UNION ALL (keeps duplicates) + assertUpdate("CREATE MATERIALIZED VIEW test_union_all_mv AS " + + "SELECT id, value, dt FROM test_union_type1 " + + "UNION ALL " + + "SELECT id, value, dt FROM test_union_type2"); + + assertRefreshAndFullyMaterialized("test_union_all_mv", 4); + + // UNION should deduplicate (1, 100) appears only once + assertQuery("SELECT id, value FROM test_union_distinct_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + // UNION ALL should keep duplicates (1, 100) appears twice + assertQuery("SELECT id, value FROM test_union_all_mv ORDER BY id", + "VALUES (1, 100), (1, 100), (2, 200), (3, 300)"); + + // Insert into one table to make it stale + assertUpdate("INSERT INTO test_union_type1 VALUES (4, 400, DATE '2024-01-02')", 1); + + // Verify stitching preserves UNION semantics (deduplication) + assertQuery("SELECT id, value FROM test_union_distinct_mv ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300), (4, 400)"); + + // Verify stitching preserves UNION ALL semantics (no deduplication) + assertQuery("SELECT id, value FROM test_union_all_mv ORDER BY id", + "VALUES (1, 100), (1, 100), (2, 200), (3, 300), (4, 400)"); + + // Verify correctness + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_union_distinct_mv ORDER BY id", + true); + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_union_all_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_union_distinct_mv"); + assertUpdate("DROP MATERIALIZED VIEW test_union_all_mv"); + assertUpdate("DROP TABLE test_union_type1"); + assertUpdate("DROP TABLE test_union_type2"); + } + + @Test + public void testIntersectInMVDefinition() + { + assertUpdate("CREATE TABLE test_intersect_base1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_intersect_base2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_intersect_base1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01')", 3); + assertUpdate("INSERT INTO test_intersect_base2 VALUES (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 3); + + // Create MV with INTERSECT - should return only common rows + assertUpdate("CREATE MATERIALIZED VIEW test_intersect_mv AS " + + "SELECT id, value, dt FROM test_intersect_base1 " + + "INTERSECT " + + "SELECT id, value, dt FROM test_intersect_base2"); + + assertRefreshAndFullyMaterialized("test_intersect_mv", 2); + + // Only (2, 200) and (3, 300) should appear + assertQuery("SELECT id, value FROM test_intersect_mv ORDER BY id", + "VALUES (2, 200), (3, 300)"); + + // Insert into BOTH tables with same dt to create a true intersection + assertUpdate("INSERT INTO test_intersect_base1 VALUES (4, 400, DATE '2024-01-02')", 1); + assertUpdate("INSERT INTO test_intersect_base2 VALUES (4, 400, DATE '2024-01-02')", 1); + + // Now (4, 400, '2024-01-02') exists in both tables, creating a true intersection + assertQuery("SELECT id, value FROM test_intersect_mv ORDER BY id", + "VALUES (2, 200), (3, 300), (4, 400)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_intersect_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_intersect_mv"); + assertUpdate("DROP TABLE test_intersect_base1"); + assertUpdate("DROP TABLE test_intersect_base2"); + } + + @Test + public void testIntersectInMVDefinitionOneSideStale() + { + assertUpdate("CREATE TABLE test_intersect_one_stale_base1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_intersect_one_stale_base2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_intersect_one_stale_base1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01')", 3); + assertUpdate("INSERT INTO test_intersect_one_stale_base2 VALUES (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 3); + + // Create MV with INTERSECT - should return only common rows + assertUpdate("CREATE MATERIALIZED VIEW test_intersect_one_stale_mv AS " + + "SELECT id, value, dt FROM test_intersect_one_stale_base1 " + + "INTERSECT " + + "SELECT id, value, dt FROM test_intersect_one_stale_base2"); + + assertRefreshAndFullyMaterialized("test_intersect_one_stale_mv", 2); + + // Only (2, 200) and (3, 300) should appear + assertQuery("SELECT id, value FROM test_intersect_one_stale_mv ORDER BY id", + "VALUES (2, 200), (3, 300)"); + + // Insert into ONLY base1 (left side becomes stale, right side stays fresh) + assertUpdate("INSERT INTO test_intersect_one_stale_base1 VALUES (5, 500, DATE '2024-01-02')", 1); + + // MV should not change since there's no matching row in base2 + assertQuery("SELECT id, value FROM test_intersect_one_stale_mv ORDER BY id", + "VALUES (2, 200), (3, 300)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_intersect_one_stale_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_intersect_one_stale_mv"); + assertUpdate("DROP TABLE test_intersect_one_stale_base1"); + assertUpdate("DROP TABLE test_intersect_one_stale_base2"); + } + + @Test + public void testExceptInMVDefinition() + { + assertUpdate("CREATE TABLE test_except_base1 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE test_except_base2 (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + assertUpdate("INSERT INTO test_except_base1 VALUES (1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01')", 3); + assertUpdate("INSERT INTO test_except_base2 VALUES (2, 200, DATE '2024-01-01'), (4, 400, DATE '2024-01-01')", 2); + + // Create MV with EXCEPT - returns rows in base1 but not in base2 + assertUpdate("CREATE MATERIALIZED VIEW test_except_mv AS " + + "SELECT id, value, dt FROM test_except_base1 " + + "EXCEPT " + + "SELECT id, value, dt FROM test_except_base2"); + + assertRefreshAndFullyMaterialized("test_except_mv", 2); + + // Only (1, 100) and (3, 300) should appear (2, 200) is excluded + assertQuery("SELECT id, value FROM test_except_mv ORDER BY id", + "VALUES (1, 100), (3, 300)"); + + // Insert into first table to make it stale + assertUpdate("INSERT INTO test_except_base1 VALUES (5, 500, DATE '2024-01-02')", 1); + + // (5, 500) is in base1 but not base2, so it should appear + assertQuery("SELECT id, value FROM test_except_mv ORDER BY id", + "VALUES (1, 100), (3, 300), (5, 500)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM test_except_mv ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW test_except_mv"); + assertUpdate("DROP TABLE test_except_base1"); + assertUpdate("DROP TABLE test_except_base2"); + } + + /** + * Test UNION of two tables, then those unioned results are joined with a third table. + * This tests that fresh tables in UNION get FALSE predicate, and that predicate + * doesn't interfere with the JOIN above it. + * + * Pattern: (T1 UNION T2) JOIN T3 + * Multi-column partitioning ensures proper constraint handling. + */ + @Test + public void testUnionThenJoinWithMultiPartitioning() + { + assertUpdate("CREATE TABLE union_join_t1 (id BIGINT, key BIGINT, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + assertUpdate("CREATE TABLE union_join_t2 (id BIGINT, key BIGINT, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + assertUpdate("CREATE TABLE union_join_t3 (key BIGINT, name VARCHAR, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + + // Insert initial data across multiple partitions + assertUpdate("INSERT INTO union_join_t1 VALUES (1, 100, '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO union_join_t1 VALUES (2, 101, '2024-01-01', 'EU')", 1); + assertUpdate("INSERT INTO union_join_t2 VALUES (3, 100, '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO union_join_t2 VALUES (4, 101, '2024-01-01', 'EU')", 1); + assertUpdate("INSERT INTO union_join_t3 VALUES (100, 'Alice', '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO union_join_t3 VALUES (101, 'Bob', '2024-01-01', 'EU')", 1); + + // Create MV: (T1 UNION T2) JOIN T3 with multi-column partitioning + assertUpdate("CREATE MATERIALIZED VIEW mv_union_join " + + "WITH (partitioning = ARRAY['date', 'region']) AS " + + "SELECT u.id, t3.name, u.date, u.region " + + "FROM (SELECT id, key, date, region FROM union_join_t1 " + + " UNION ALL " + + " SELECT id, key, date, region FROM union_join_t2) u " + + "JOIN union_join_t3 t3 ON u.key = t3.key AND u.date = t3.date AND u.region = t3.region"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_union_join"); + + // Make T1 stale in ONLY ONE partition: (date='2024-01-02', region='US') + // This leaves other partitions fresh, ensuring data table is used for stitching + assertUpdate("INSERT INTO union_join_t1 VALUES (5, 200, '2024-01-02', 'US')", 1); + assertUpdate("INSERT INTO union_join_t3 VALUES (200, 'Charlie', '2024-01-02', 'US')", 1); + + // Query the MV - should use UNION stitching + // Expected behavior: + // - Storage scan with filter: NOT (date='2024-01-02' AND region='US') + // - Stale branch recomputes only (date='2024-01-02', region='US'): + // - T1 with date='2024-01-02' AND region='US' (stale table) + // - T2 gets FALSE predicate (fresh table in UNION with stale branch) + // - T3 sees all data (not in UNION, participating in JOIN) + assertQuery("SELECT * FROM mv_union_join ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01', 'US'), " + + "(2, 'Bob', '2024-01-01', 'EU'), " + + "(3, 'Alice', '2024-01-01', 'US'), " + + "(4, 'Bob', '2024-01-01', 'EU'), " + + "(5, 'Charlie', '2024-01-02', 'US')"); + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_union_join ORDER BY id"); + + assertUpdate("DROP MATERIALIZED VIEW mv_union_join"); + assertUpdate("DROP TABLE union_join_t3"); + assertUpdate("DROP TABLE union_join_t2"); + assertUpdate("DROP TABLE union_join_t1"); + } + + /** + * Test JOIN of two tables, then those joined results are unioned with a third table. + * This tests that the fresh table in UNION gets FALSE predicate correctly, + * and that tables in the JOIN (not in UNION) see all data. + * + * Pattern: (T1 JOIN T2) UNION T3 + * Multi-column partitioning with partial staleness ensures data table usage. + */ + @Test + public void testJoinThenUnionWithMultiPartitioning() + { + assertUpdate("CREATE TABLE join_union_t1 (id BIGINT, key BIGINT, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + assertUpdate("CREATE TABLE join_union_t2 (key BIGINT, name VARCHAR, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + assertUpdate("CREATE TABLE join_union_t3 (id BIGINT, name VARCHAR, date VARCHAR, region VARCHAR) " + + "WITH (partitioning = ARRAY['date', 'region'])"); + + // Insert initial data across multiple partitions + assertUpdate("INSERT INTO join_union_t1 VALUES (1, 100, '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO join_union_t1 VALUES (2, 101, '2024-01-01', 'EU')", 1); + assertUpdate("INSERT INTO join_union_t2 VALUES (100, 'Alice', '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO join_union_t2 VALUES (101, 'Bob', '2024-01-01', 'EU')", 1); + assertUpdate("INSERT INTO join_union_t3 VALUES (3, 'Charlie', '2024-01-01', 'US')", 1); + assertUpdate("INSERT INTO join_union_t3 VALUES (4, 'David', '2024-01-01', 'EU')", 1); + + // Create MV: (T1 JOIN T2) UNION T3 with multi-column partitioning + assertUpdate("CREATE MATERIALIZED VIEW mv_join_union " + + "WITH (partitioning = ARRAY['date', 'region']) AS " + + "SELECT j.id, j.name, j.date, j.region FROM " + + " (SELECT t1.id, t2.name, t1.date, t1.region " + + " FROM join_union_t1 t1 " + + " JOIN join_union_t2 t2 ON t1.key = t2.key AND t1.date = t2.date AND t1.region = t2.region) j " + + "UNION ALL " + + "SELECT id, name, date, region FROM join_union_t3"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_join_union"); + + // Make T1 stale in ONLY ONE partition: (date='2024-01-02', region='EU') + // This leaves (date='2024-01-01', region='US') and (date='2024-01-01', region='EU') fresh + assertUpdate("INSERT INTO join_union_t1 VALUES (5, 200, '2024-01-02', 'EU')", 1); + assertUpdate("INSERT INTO join_union_t2 VALUES (200, 'Eve', '2024-01-02', 'EU')", 1); + + // Query the MV - should use UNION stitching + // Expected behavior: + // - Storage scan with filter: NOT (date='2024-01-02' AND region='EU') + // - Stale branch recomputes only (date='2024-01-02', region='EU'): + // - T1 with date='2024-01-02' AND region='EU' (stale table) + // - T2 sees all data (not in UNION, participating in JOIN with stale table) + // - T3 gets FALSE predicate (fresh table in UNION with stale branch) + assertQuery("SELECT * FROM mv_join_union ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01', 'US'), " + + "(2, 'Bob', '2024-01-01', 'EU'), " + + "(3, 'Charlie', '2024-01-01', 'US'), " + + "(4, 'David', '2024-01-01', 'EU'), " + + "(5, 'Eve', '2024-01-02', 'EU')"); + + assertUpdate("DROP MATERIALIZED VIEW mv_join_union"); + assertUpdate("DROP TABLE join_union_t3"); + assertUpdate("DROP TABLE join_union_t2"); + assertUpdate("DROP TABLE join_union_t1"); + } + + /** + * Edge case: Nested UNIONs with JOIN in between. + * Pattern: (T1 UNION T2) JOIN (T3 UNION T4) + * All tables inside the JOINs should see all data, even though they're in inner UNIONs. + */ + @Test + public void testUnionJoinUnionNesting() + { + assertUpdate("CREATE TABLE unju_t1 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE unju_t2 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE unju_t3 (key BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE unju_t4 (key BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + + // Initial data + assertUpdate("INSERT INTO unju_t1 VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO unju_t2 VALUES (2, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO unju_t3 VALUES (100, 'Alice', '2024-01-01')", 1); + assertUpdate("INSERT INTO unju_t4 VALUES (100, 'Bob', '2024-01-01')", 1); + + // MV: (T1 UNION T2) JOIN (T3 UNION T4) + assertUpdate("CREATE MATERIALIZED VIEW mv_unju AS " + + "SELECT u1.id, u2.name, u1.date " + + "FROM (SELECT id, key, date FROM unju_t1 UNION ALL SELECT id, key, date FROM unju_t2) u1 " + + "JOIN (SELECT key, name, date FROM unju_t3 UNION ALL SELECT key, name, date FROM unju_t4) u2 " + + "ON u1.key = u2.key AND u1.date = u2.date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_unju"); + + // Make T1 stale + assertUpdate("INSERT INTO unju_t1 VALUES (3, 200, '2024-01-02')", 1); + assertUpdate("INSERT INTO unju_t3 VALUES (200, 'Charlie', '2024-01-02')", 1); + + // Expected: T1 sees stale predicate, T2/T3/T4 all see ALL data (inside JOINs) + // Result should include new row from recompute + assertQuery("SELECT * FROM mv_unju ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01'), " + + "(1, 'Bob', '2024-01-01'), " + + "(2, 'Alice', '2024-01-01'), " + + "(2, 'Bob', '2024-01-01'), " + + "(3, 'Charlie', '2024-01-02')"); + + assertUpdate("DROP MATERIALIZED VIEW mv_unju"); + assertUpdate("DROP TABLE unju_t4"); + assertUpdate("DROP TABLE unju_t3"); + assertUpdate("DROP TABLE unju_t2"); + assertUpdate("DROP TABLE unju_t1"); + } + + /** + * Test INTERSECT inside JOIN. + * Pattern: (T1 INTERSECT T2) JOIN (T3 INTERSECT T4) + * When T1 becomes stale: + * - T1 sees stale predicate (dt='2024-01-02') + * - T2, T3, T4 all see ALL data (because INTERSECT needs complete data from both sides) + * This verifies that INTERSECT is treated like JOIN, not like UNION. + */ + @Test + public void testIntersectJoinIntersectNesting() + { + assertUpdate("CREATE TABLE inji_t1 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE inji_t2 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE inji_t3 (key BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE inji_t4 (key BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + + // Initial data: Create intersections in both T1/T2 and T3/T4 + assertUpdate("INSERT INTO inji_t1 VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO inji_t2 VALUES (1, 100, '2024-01-01'), (3, 300, '2024-01-01')", 2); // (1, 100) is common + assertUpdate("INSERT INTO inji_t3 VALUES (100, 'Alice', '2024-01-01'), (200, 'Bob', '2024-01-01')", 2); + assertUpdate("INSERT INTO inji_t4 VALUES (100, 'Alice', '2024-01-01'), (300, 'Charlie', '2024-01-01')", 2); // (100, 'Alice') is common + + // MV: (T1 INTERSECT T2) JOIN (T3 INTERSECT T4) + // After initial data, LEFT side produces (1, 100, '2024-01-01'), RIGHT side produces (100, 'Alice', '2024-01-01') + // JOIN produces (1, 'Alice', '2024-01-01') + assertUpdate("CREATE MATERIALIZED VIEW mv_inji AS " + + "SELECT u1.id, u2.name, u1.date " + + "FROM (SELECT id, key, date FROM inji_t1 INTERSECT SELECT id, key, date FROM inji_t2) u1 " + + "JOIN (SELECT key, name, date FROM inji_t3 INTERSECT SELECT key, name, date FROM inji_t4) u2 " + + "ON u1.key = u2.key AND u1.date = u2.date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_inji"); + + // Verify initial state + assertQuery("SELECT * FROM mv_inji ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01')"); + + // Make T1 and T3 stale by inserting into new partition + // New intersection on LEFT: (4, 400) appears in both T1 and T2 + // New intersection on RIGHT: (400, 'David') appears in both T3 and T4 + assertUpdate("INSERT INTO inji_t1 VALUES (4, 400, '2024-01-02')", 1); + assertUpdate("INSERT INTO inji_t2 VALUES (4, 400, '2024-01-02')", 1); + assertUpdate("INSERT INTO inji_t3 VALUES (400, 'David', '2024-01-02')", 1); + assertUpdate("INSERT INTO inji_t4 VALUES (400, 'David', '2024-01-02')", 1); + + // Expected: Both old and new rows + // With partition stitching: + // - T1 sees stale predicate (dt='2024-01-02') + // - T2, T3, T4 must see ALL data for INTERSECT to work correctly + // Result should include both (1, 'Alice', '2024-01-01') from storage and (4, 'David', '2024-01-02') from recompute + assertQuery("SELECT * FROM mv_inji ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01'), (4, 'David', '2024-01-02')"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_inji ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_inji"); + assertUpdate("DROP TABLE inji_t4"); + assertUpdate("DROP TABLE inji_t3"); + assertUpdate("DROP TABLE inji_t2"); + assertUpdate("DROP TABLE inji_t1"); + } + + /** + * Edge case: UNION inside JOIN inside UNION. + * Pattern: ((T1 UNION T2) JOIN T3) UNION T4 + * - T1, T2 inside inner UNION but also inside JOIN → should see all data (JOIN barrier) + * - T3 inside JOIN → should see all data + * - T4 in outer UNION → should get FALSE predicate + */ + @Test + public void testUnionJoinUnionTripleNesting() + { + assertUpdate("CREATE TABLE ujut_t1 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE ujut_t2 (id BIGINT, key BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE ujut_t3 (key BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE ujut_t4 (id BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + + // Initial data + assertUpdate("INSERT INTO ujut_t1 VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO ujut_t2 VALUES (2, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO ujut_t3 VALUES (100, 'Alice', '2024-01-01')", 1); + assertUpdate("INSERT INTO ujut_t4 VALUES (4, 'Bob', '2024-01-01')", 1); + + // MV: ((T1 UNION T2) JOIN T3) UNION T4 + assertUpdate("CREATE MATERIALIZED VIEW mv_ujut AS " + + "SELECT j.id, j.name, j.date FROM " + + " (SELECT u.id, t3.name, u.date FROM " + + " (SELECT id, key, date FROM ujut_t1 UNION ALL SELECT id, key, date FROM ujut_t2) u " + + " JOIN ujut_t3 t3 ON u.key = t3.key AND u.date = t3.date) j " + + "UNION ALL " + + "SELECT id, name, date FROM ujut_t4"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_ujut"); + + // Make T1 stale + assertUpdate("INSERT INTO ujut_t1 VALUES (5, 200, '2024-01-02')", 1); + assertUpdate("INSERT INTO ujut_t3 VALUES (200, 'Charlie', '2024-01-02')", 1); + + // Expected behavior: + // - T1 gets stale predicate (date='2024-01-02') + // - T2 sees all data (inside JOIN, even though also in inner UNION) + // - T3 sees all data (inside JOIN with stale T1) + // - T4 gets FALSE predicate (fresh table in outer UNION) + assertQuery("SELECT * FROM mv_ujut ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01'), " + + "(2, 'Alice', '2024-01-01'), " + + "(4, 'Bob', '2024-01-01'), " + + "(5, 'Charlie', '2024-01-02')"); + + assertUpdate("DROP MATERIALIZED VIEW mv_ujut"); + assertUpdate("DROP TABLE ujut_t4"); + assertUpdate("DROP TABLE ujut_t3"); + assertUpdate("DROP TABLE ujut_t2"); + assertUpdate("DROP TABLE ujut_t1"); + } + + /** + * Edge case: Multiple JOIN layers. + * Pattern: ((T1 JOIN T2) JOIN T3) UNION T4 + * - T1, T2, T3 all inside nested JOINs → should see all data + * - T4 in UNION → should get FALSE predicate + */ + @Test + public void testNestedJoinsWithUnion() + { + assertUpdate("CREATE TABLE njwu_t1 (id BIGINT, key1 BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE njwu_t2 (key1 BIGINT, key2 BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE njwu_t3 (key2 BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE njwu_t4 (id BIGINT, name VARCHAR, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + + // Initial data + assertUpdate("INSERT INTO njwu_t1 VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO njwu_t2 VALUES (100, 200, '2024-01-01')", 1); + assertUpdate("INSERT INTO njwu_t3 VALUES (200, 'Alice', '2024-01-01')", 1); + assertUpdate("INSERT INTO njwu_t4 VALUES (4, 'Bob', '2024-01-01')", 1); + + // MV: ((T1 JOIN T2) JOIN T3) UNION T4 + assertUpdate("CREATE MATERIALIZED VIEW mv_njwu AS " + + "SELECT j.id, j.name, j.date FROM " + + " (SELECT t1.id, t3.name, t1.date FROM njwu_t1 t1 " + + " JOIN njwu_t2 t2 ON t1.key1 = t2.key1 AND t1.date = t2.date " + + " JOIN njwu_t3 t3 ON t2.key2 = t3.key2 AND t2.date = t3.date) j " + + "UNION ALL " + + "SELECT id, name, date FROM njwu_t4"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_njwu"); + + // Make T1 stale + assertUpdate("INSERT INTO njwu_t1 VALUES (5, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO njwu_t2 VALUES (300, 400, '2024-01-02')", 1); + assertUpdate("INSERT INTO njwu_t3 VALUES (400, 'Charlie', '2024-01-02')", 1); + + // Expected: + // - T1 gets stale predicate + // - T2, T3 see all data (in JOINs with stale table) + // - T4 gets FALSE (fresh table in UNION) + assertQuery("SELECT * FROM mv_njwu ORDER BY id", + "VALUES (1, 'Alice', '2024-01-01'), " + + "(4, 'Bob', '2024-01-01'), " + + "(5, 'Charlie', '2024-01-02')"); + + assertUpdate("DROP MATERIALIZED VIEW mv_njwu"); + assertUpdate("DROP TABLE njwu_t4"); + assertUpdate("DROP TABLE njwu_t3"); + assertUpdate("DROP TABLE njwu_t2"); + assertUpdate("DROP TABLE njwu_t1"); + } + + /** + * Test deeply nested joins: (A JOIN B) JOIN (C JOIN D) + * This pattern creates a balanced binary tree of joins where: + * - Each leaf (A, B, C, D) is a table + * - Two inner joins produce intermediate results + * - A final join combines the intermediate results + * + * When A becomes stale: + * - A sees stale predicate + * - B sees all data (must join with stale data from A) + * - C sees all data (joined in a subexpression that joins with stale side) + * - D sees all data (joined in a subexpression that joins with stale side) + * + * This tests that the UNION stitching logic correctly handles nested join structures. + */ + @Test + public void testDeeplyNestedJoins() + { + // Create 4 tables: orders (A), customers (B), products (C), categories (D) + assertUpdate("CREATE TABLE dnj_orders (order_id BIGINT, customer_id BIGINT, product_id BIGINT, order_date DATE) " + + "WITH (partitioning = ARRAY['order_date'])"); + assertUpdate("CREATE TABLE dnj_customers (customer_id BIGINT, customer_name VARCHAR, region VARCHAR, reg_date DATE) " + + "WITH (partitioning = ARRAY['reg_date'])"); + assertUpdate("CREATE TABLE dnj_products (product_id BIGINT, product_name VARCHAR, category_id BIGINT, product_date DATE) " + + "WITH (partitioning = ARRAY['product_date'])"); + assertUpdate("CREATE TABLE dnj_categories (category_id BIGINT, category_name VARCHAR, cat_date DATE) " + + "WITH (partitioning = ARRAY['cat_date'])"); + + // Initial data - all tables have data for '2024-01-01' + assertUpdate("INSERT INTO dnj_orders VALUES " + + "(1, 100, 1000, DATE '2024-01-01'), " + + "(2, 200, 2000, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_customers VALUES " + + "(100, 'Alice', 'US', DATE '2024-01-01'), " + + "(200, 'Bob', 'EU', DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_products VALUES " + + "(1000, 'Laptop', 10, DATE '2024-01-01'), " + + "(2000, 'Phone', 20, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO dnj_categories VALUES " + + "(10, 'Electronics', DATE '2024-01-01'), " + + "(20, 'Mobile', DATE '2024-01-01')", 2); + + // MV: (orders JOIN customers) JOIN (products JOIN categories) + // This creates a balanced tree structure: + // JOIN + // / \ + // JOIN JOIN + // / \ / \ + // orders cust prod cats + assertUpdate("CREATE MATERIALIZED VIEW mv_dnj AS " + + "SELECT oc.order_id, oc.customer_name, pc.product_name, pc.category_name, oc.order_date " + + "FROM " + + " (SELECT o.order_id, c.customer_name, o.product_id, o.order_date FROM dnj_orders o " + + " JOIN dnj_customers c ON o.customer_id = c.customer_id AND o.order_date = c.reg_date) oc " + + " JOIN " + + " (SELECT p.product_id, p.product_name, cat.category_name, p.product_date FROM dnj_products p " + + " JOIN dnj_categories cat ON p.category_id = cat.category_id AND p.product_date = cat.cat_date) pc " + + " ON oc.product_id = pc.product_id AND oc.order_date = pc.product_date"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + // Verify initial state - 2 rows + assertMaterializedViewQuery("SELECT * FROM mv_dnj ORDER BY order_id", + "VALUES (1, 'Alice', 'Laptop', 'Electronics', DATE '2024-01-01'), " + + "(2, 'Bob', 'Phone', 'Mobile', DATE '2024-01-01')"); + + // ============================================================ + // Scenario 1: Make only 1 table stale (orders) + // ============================================================ + // Pre-populate customers, products, categories for partition '2024-01-02' BEFORE refresh + // so that when we insert into orders AFTER refresh, only orders is stale + assertUpdate("INSERT INTO dnj_customers VALUES (300, 'Charlie', 'US', DATE '2024-01-02')", 1); + assertUpdate("INSERT INTO dnj_products VALUES (3000, 'Tablet', 10, DATE '2024-01-02')", 1); + assertUpdate("INSERT INTO dnj_categories VALUES (10, 'Electronics', DATE '2024-01-02')", 1); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + // Now insert into orders only - making only orders stale for '2024-01-02' + assertUpdate("INSERT INTO dnj_orders VALUES (3, 300, 3000, DATE '2024-01-02')", 1); + + // Verify MV is now partially materialized (stale) + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'mv_dnj'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + // Only orders (A) is stale for partition '2024-01-02' + // The join will use fresh data from customers, products, categories + assertMaterializedViewQuery("SELECT * FROM mv_dnj ORDER BY order_id", + "VALUES (1, 'Alice', 'Laptop', 'Electronics', DATE '2024-01-01'), " + + "(2, 'Bob', 'Phone', 'Mobile', DATE '2024-01-01'), " + + "(3, 'Charlie', 'Tablet', 'Electronics', DATE '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_dnj ORDER BY order_id", + true); + + // ============================================================ + // Scenario 2: Make 2 tables stale (orders and products) + // ============================================================ + // Pre-populate customers and categories for partition '2024-01-03' BEFORE refresh + assertUpdate("INSERT INTO dnj_customers VALUES (400, 'Diana', 'APAC', DATE '2024-01-03')", 1); + assertUpdate("INSERT INTO dnj_categories VALUES (20, 'Wearables', DATE '2024-01-03')", 1); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + // Now insert into orders and products only - making 2 tables stale + assertUpdate("INSERT INTO dnj_orders VALUES (4, 400, 4000, DATE '2024-01-03')", 1); + assertUpdate("INSERT INTO dnj_products VALUES (4000, 'Watch', 20, DATE '2024-01-03')", 1); + + // Verify MV is now partially materialized (stale) + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'mv_dnj'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + // orders (A) and products (C) are stale for partition '2024-01-03' + // customers (B) and categories (D) are fresh + assertMaterializedViewQuery("SELECT * FROM mv_dnj ORDER BY order_id", + "VALUES (1, 'Alice', 'Laptop', 'Electronics', DATE '2024-01-01'), " + + "(2, 'Bob', 'Phone', 'Mobile', DATE '2024-01-01'), " + + "(3, 'Charlie', 'Tablet', 'Electronics', DATE '2024-01-02'), " + + "(4, 'Diana', 'Watch', 'Wearables', DATE '2024-01-03')"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_dnj ORDER BY order_id", + true); + + // ============================================================ + // Scenario 3: Make 3 tables stale (orders, customers, products) + // ============================================================ + // Pre-populate only categories for partition '2024-01-04' BEFORE refresh + assertUpdate("INSERT INTO dnj_categories VALUES (10, 'Electronics', DATE '2024-01-04')", 1); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + // Now insert into orders, customers, and products - making 3 tables stale + assertUpdate("INSERT INTO dnj_orders VALUES (5, 500, 5000, DATE '2024-01-04')", 1); + assertUpdate("INSERT INTO dnj_customers VALUES (500, 'Eve', 'EMEA', DATE '2024-01-04')", 1); + assertUpdate("INSERT INTO dnj_products VALUES (5000, 'Headphones', 10, DATE '2024-01-04')", 1); + + // Verify MV is now partially materialized (stale) + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'mv_dnj'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + // orders (A), customers (B), and products (C) are stale for partition '2024-01-04' + // Only categories (D) is fresh + assertMaterializedViewQuery("SELECT * FROM mv_dnj ORDER BY order_id", + "VALUES (1, 'Alice', 'Laptop', 'Electronics', DATE '2024-01-01'), " + + "(2, 'Bob', 'Phone', 'Mobile', DATE '2024-01-01'), " + + "(3, 'Charlie', 'Tablet', 'Electronics', DATE '2024-01-02'), " + + "(4, 'Diana', 'Watch', 'Wearables', DATE '2024-01-03'), " + + "(5, 'Eve', 'Headphones', 'Electronics', DATE '2024-01-04')"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_dnj ORDER BY order_id", + true); + + // ============================================================ + // Scenario 4: Make all 4 tables stale + // ============================================================ + // Refresh first, then insert into all 4 tables for a new partition + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_dnj"); + + assertUpdate("INSERT INTO dnj_orders VALUES (6, 600, 6000, DATE '2024-01-05')", 1); + assertUpdate("INSERT INTO dnj_customers VALUES (600, 'Frank', 'LATAM', DATE '2024-01-05')", 1); + assertUpdate("INSERT INTO dnj_products VALUES (6000, 'Speaker', 30, DATE '2024-01-05')", 1); + assertUpdate("INSERT INTO dnj_categories VALUES (30, 'Audio', DATE '2024-01-05')", 1); + + // Verify MV is now partially materialized (stale) + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'mv_dnj'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + // All 4 tables are stale for partition '2024-01-05' + assertMaterializedViewQuery("SELECT * FROM mv_dnj ORDER BY order_id", + "VALUES (1, 'Alice', 'Laptop', 'Electronics', DATE '2024-01-01'), " + + "(2, 'Bob', 'Phone', 'Mobile', DATE '2024-01-01'), " + + "(3, 'Charlie', 'Tablet', 'Electronics', DATE '2024-01-02'), " + + "(4, 'Diana', 'Watch', 'Wearables', DATE '2024-01-03'), " + + "(5, 'Eve', 'Headphones', 'Electronics', DATE '2024-01-04'), " + + "(6, 'Frank', 'Speaker', 'Audio', DATE '2024-01-05')"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_dnj ORDER BY order_id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_dnj"); + assertUpdate("DROP TABLE dnj_categories"); + assertUpdate("DROP TABLE dnj_products"); + assertUpdate("DROP TABLE dnj_customers"); + assertUpdate("DROP TABLE dnj_orders"); + } + + /** + * Edge case: Aggregation should also act as a barrier. + * Pattern: (SELECT ... FROM T1 GROUP BY ...) UNION T2 + * - T1 inside aggregation → should see all data + * - T2 in UNION → should get FALSE predicate + */ + @Test + public void testAggregationWithUnion() + { + assertUpdate("CREATE TABLE awu_t1 (id BIGINT, value BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + assertUpdate("CREATE TABLE awu_t2 (id BIGINT, value BIGINT, date VARCHAR) " + + "WITH (partitioning = ARRAY['date'])"); + + // Initial data + assertUpdate("INSERT INTO awu_t1 VALUES (1, 10, '2024-01-01'), (1, 20, '2024-01-01')", 2); + assertUpdate("INSERT INTO awu_t2 VALUES (2, 100, '2024-01-01')", 1); + + // MV: (aggregated T1) UNION T2 + assertUpdate("CREATE MATERIALIZED VIEW mv_awu AS " + + "SELECT id, SUM(value) as total, date FROM awu_t1 GROUP BY id, date " + + "UNION ALL " + + "SELECT id, value as total, date FROM awu_t2"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_awu"); + + // Make T1 stale by adding more rows to aggregate + assertUpdate("INSERT INTO awu_t1 VALUES (1, 30, '2024-01-02')", 1); + + // Expected: + // - T1 sees all data for date='2024-01-02' to compute SUM correctly + // - T2 gets FALSE (fresh table in UNION) + assertQuery("SELECT * FROM mv_awu ORDER BY id, date", + "VALUES (1, 30, '2024-01-01'), " + + "(1, 30, '2024-01-02'), " + + "(2, 100, '2024-01-01')"); + + assertUpdate("DROP MATERIALIZED VIEW mv_awu"); + assertUpdate("DROP TABLE awu_t2"); + assertUpdate("DROP TABLE awu_t1"); + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) pattern with various staleness combinations. + * This tests that JOINs inside EXCEPT are handled correctly: + * - When a table on the left side of EXCEPT becomes stale, all tables on the left side + * must get complete data for the JOIN to work, and the right side needs all data too. + * - When a table on the right side becomes stale, similar logic applies. + */ + @Test + public void testJoinExceptJoinWithLeftSideStale() + { + // Pattern: (A JOIN B) EXCEPT (C JOIN D) where A becomes stale + assertUpdate("CREATE TABLE jexj_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexj_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data: create rows that will be in left but NOT in right (for EXCEPT to return) + // Left side (A JOIN B): produces (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') + assertUpdate("INSERT INTO jexj_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexj_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + // Right side (C JOIN D): produces (1, 'x', '2024-01-01') - will be excluded + assertUpdate("INSERT INTO jexj_c VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexj_d VALUES (100, 'x', '2024-01-01')", 1); + + // MV: (A JOIN B) EXCEPT (C JOIN D) -> should produce (2, 'y', '2024-01-01') + assertUpdate("CREATE MATERIALIZED VIEW mv_jexj AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexj_a a JOIN jexj_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexj_c c JOIN jexj_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexj"); + + // Verify initial state + assertQuery("SELECT * FROM mv_jexj ORDER BY id", "VALUES (2, 'y', '2024-01-01')"); + + // Make A stale by adding new partition + // New left side row: (3, 'z', '2024-01-02') + assertUpdate("INSERT INTO jexj_a VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexj_b VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - A gets stale predicate (dt='2024-01-02') + // - B, C, D must see ALL data (inside EXCEPT, predicate propagation disabled) + // - Result: (2, 'y', '2024-01-01') from storage + (3, 'z', '2024-01-02') from recompute + assertQuery("SELECT * FROM mv_jexj ORDER BY id", + "VALUES (2, 'y', '2024-01-01'), (3, 'z', '2024-01-02')"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jexj ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jexj"); + assertUpdate("DROP TABLE jexj_d"); + assertUpdate("DROP TABLE jexj_c"); + assertUpdate("DROP TABLE jexj_b"); + assertUpdate("DROP TABLE jexj_a"); + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where C (right side) becomes stale. + */ + @Test + public void testJoinExceptJoinWithRightSideStale() + { + assertUpdate("CREATE TABLE jexjr_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjr_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + // Left side: produces (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') + assertUpdate("INSERT INTO jexjr_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjr_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + // Right side: produces (1, 'x', '2024-01-01') + assertUpdate("INSERT INTO jexjr_c VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexjr_d VALUES (100, 'x', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jexjr AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexjr_a a JOIN jexjr_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexjr_c c JOIN jexjr_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexjr"); + + // Verify initial state: (2, 'y') is not in right side, so it's in EXCEPT result + assertQuery("SELECT * FROM mv_jexjr ORDER BY id", "VALUES (2, 'y', '2024-01-01')"); + + // Make C stale by adding new partition + // This adds (3, 'z', '2024-01-02') to right side, which should be excluded from left + assertUpdate("INSERT INTO jexjr_c VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjr_d VALUES (300, 'z', '2024-01-02')", 1); + // Also add matching data to left side + assertUpdate("INSERT INTO jexjr_a VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + assertUpdate("INSERT INTO jexjr_b VALUES (300, 'z', '2024-01-02'), (400, 'w', '2024-01-02')", 2); + + // Expected: + // - C gets stale predicate + // - A, B, D must see ALL data + // - For partition '2024-01-02': + // Left has: (3, 'z'), (4, 'w') + // Right has: (3, 'z') + // EXCEPT gives: (4, 'w') + // - Final: (2, 'y', '2024-01-01') from storage + (4, 'w', '2024-01-02') from recompute + assertQuery("SELECT * FROM mv_jexjr ORDER BY id", + "VALUES (2, 'y', '2024-01-01'), (4, 'w', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jexjr ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jexjr"); + assertUpdate("DROP TABLE jexjr_d"); + assertUpdate("DROP TABLE jexjr_c"); + assertUpdate("DROP TABLE jexjr_b"); + assertUpdate("DROP TABLE jexjr_a"); + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where both A and C become stale. + * This should create two union branches (one for each stale table). + */ + @Test + public void testJoinExceptJoinWithBothSidesStale() + { + assertUpdate("CREATE TABLE jexjb_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjb_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO jexjb_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjb_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjb_c VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexjb_d VALUES (100, 'x', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jexjb AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexjb_a a JOIN jexjb_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexjb_c c JOIN jexjb_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexjb"); + + assertQuery("SELECT * FROM mv_jexjb ORDER BY id", "VALUES (2, 'y', '2024-01-01')"); + + // Make BOTH A and C stale by adding to new partition + assertUpdate("INSERT INTO jexjb_a VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + assertUpdate("INSERT INTO jexjb_b VALUES (300, 'z', '2024-01-02'), (400, 'w', '2024-01-02')", 2); + assertUpdate("INSERT INTO jexjb_c VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjb_d VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - Creates two union branches (one for A stale, one for C stale) + // - For partition '2024-01-02': + // Left: (3, 'z'), (4, 'w') + // Right: (3, 'z') + // EXCEPT: (4, 'w') + // - Final: (2, 'y', '2024-01-01') from storage + (4, 'w', '2024-01-02') + assertQuery("SELECT * FROM mv_jexjb ORDER BY id", + "VALUES (2, 'y', '2024-01-01'), (4, 'w', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jexjb ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jexjb"); + assertUpdate("DROP TABLE jexjb_d"); + assertUpdate("DROP TABLE jexjb_c"); + assertUpdate("DROP TABLE jexjb_b"); + assertUpdate("DROP TABLE jexjb_a"); + } + + /** + * Test (A JOIN B) EXCEPT (C JOIN D) where A and B both become stale (same side of EXCEPT). + * This tests that tables on the same side of a JOIN within EXCEPT are handled correctly. + */ + @Test + public void testJoinExceptJoinWithSameSideJoinTablesStale() + { + assertUpdate("CREATE TABLE jexjs_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjs_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjs_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jexjs_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + assertUpdate("INSERT INTO jexjs_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjs_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jexjs_c VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO jexjs_d VALUES (100, 'x', '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jexjs AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jexjs_a a JOIN jexjs_b b ON a.key = b.key AND a.dt = b.dt " + + "EXCEPT " + + "SELECT c.id, d.value, c.dt " + + "FROM jexjs_c c JOIN jexjs_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jexjs"); + + assertQuery("SELECT * FROM mv_jexjs ORDER BY id", "VALUES (2, 'y', '2024-01-01')"); + + // Make BOTH A and B stale (same side of EXCEPT, joined together) + assertUpdate("INSERT INTO jexjs_a VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jexjs_b VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - Creates two union branches (one for A, one for B) + // - Both need complete data from C and D for EXCEPT + // - For partition '2024-01-02': + // Left: (3, 'z') + // Right: nothing (C and D have no data for this partition) + // EXCEPT: (3, 'z') + // - Final: (2, 'y', '2024-01-01') from storage + (3, 'z', '2024-01-02') + assertQuery("SELECT * FROM mv_jexjs ORDER BY id", + "VALUES (2, 'y', '2024-01-01'), (3, 'z', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jexjs ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jexjs"); + assertUpdate("DROP TABLE jexjs_d"); + assertUpdate("DROP TABLE jexjs_c"); + assertUpdate("DROP TABLE jexjs_b"); + assertUpdate("DROP TABLE jexjs_a"); + } + + /** + * Test (A JOIN B) INTERSECT (C JOIN D) where A (left side of INTERSECT) becomes stale. + * INTERSECT returns rows that exist in BOTH sides, so data must match. + */ + @Test + public void testJoinIntersectJoinWithLeftSideStale() + { + // Pattern: (A JOIN B) INTERSECT (C JOIN D) where A becomes stale + assertUpdate("CREATE TABLE jintl_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintl_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintl_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintl_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data: create matching rows on both sides for INTERSECT to return + // Left side (A JOIN B): produces (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') + assertUpdate("INSERT INTO jintl_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jintl_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + // Right side (C JOIN D): produces (1, 'x', '2024-01-01') - will be in INTERSECT result + assertUpdate("INSERT INTO jintl_c VALUES (1, 100, '2024-01-01')", 1); + assertUpdate("INSERT INTO jintl_d VALUES (100, 'x', '2024-01-01')", 1); + + // MV: (A JOIN B) INTERSECT (C JOIN D) -> should produce (1, 'x', '2024-01-01') + assertUpdate("CREATE MATERIALIZED VIEW mv_jintl AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jintl_a a JOIN jintl_b b ON a.key = b.key AND a.dt = b.dt " + + "INTERSECT " + + "SELECT c.id, d.value, c.dt " + + "FROM jintl_c c JOIN jintl_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jintl"); + + // Verify initial state + assertQuery("SELECT * FROM mv_jintl ORDER BY id", "VALUES (1, 'x', '2024-01-01')"); + + // Make A stale by adding new partition with matching data on both sides + assertUpdate("INSERT INTO jintl_a VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + assertUpdate("INSERT INTO jintl_b VALUES (300, 'z', '2024-01-02'), (400, 'w', '2024-01-02')", 2); + // Add matching row to right side - only (3, 'z') will be in INTERSECT + assertUpdate("INSERT INTO jintl_c VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jintl_d VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - A gets stale predicate (dt='2024-01-02') + // - Inside INTERSECT, predicate propagation is disabled between set operation branches + // - Result: (1, 'x', '2024-01-01') from storage + (3, 'z', '2024-01-02') from recompute + assertQuery("SELECT * FROM mv_jintl ORDER BY id", + "VALUES (1, 'x', '2024-01-01'), (3, 'z', '2024-01-02')"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jintl ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jintl"); + assertUpdate("DROP TABLE jintl_d"); + assertUpdate("DROP TABLE jintl_c"); + assertUpdate("DROP TABLE jintl_b"); + assertUpdate("DROP TABLE jintl_a"); + } + + /** + * Test (A JOIN B) INTERSECT (C JOIN D) where C (right side) becomes stale. + */ + @Test + public void testJoinIntersectJoinWithRightSideStale() + { + assertUpdate("CREATE TABLE jintr_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintr_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintr_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintr_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data + // Left side: produces (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') + assertUpdate("INSERT INTO jintr_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jintr_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + // Right side: produces (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') + assertUpdate("INSERT INTO jintr_c VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jintr_d VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jintr AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jintr_a a JOIN jintr_b b ON a.key = b.key AND a.dt = b.dt " + + "INTERSECT " + + "SELECT c.id, d.value, c.dt " + + "FROM jintr_c c JOIN jintr_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jintr"); + + // Verify initial state: both rows match on both sides + assertQuery("SELECT * FROM mv_jintr ORDER BY id", "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01')"); + + // Make C stale (right side of INTERSECT) + assertUpdate("INSERT INTO jintr_c VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + assertUpdate("INSERT INTO jintr_d VALUES (300, 'z', '2024-01-02'), (400, 'w', '2024-01-02')", 2); + // Add matching data to left side - only (3, 'z') will be in INTERSECT + assertUpdate("INSERT INTO jintr_a VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jintr_b VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - C gets stale predicate + // - For partition '2024-01-02': + // Left has: (3, 'z') + // Right has: (3, 'z'), (4, 'w') + // INTERSECT gives: (3, 'z') + // - Final: (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') from storage + (3, 'z', '2024-01-02') + assertQuery("SELECT * FROM mv_jintr ORDER BY id", + "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01'), (3, 'z', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jintr ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jintr"); + assertUpdate("DROP TABLE jintr_d"); + assertUpdate("DROP TABLE jintr_c"); + assertUpdate("DROP TABLE jintr_b"); + assertUpdate("DROP TABLE jintr_a"); + } + + /** + * Test (A JOIN B) INTERSECT (C JOIN D) where both A and C become stale. + * This should create two union branches (one for each stale table). + */ + @Test + public void testJoinIntersectJoinWithBothSidesStale() + { + assertUpdate("CREATE TABLE jintb_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintb_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintb_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jintb_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data with full match on both sides + assertUpdate("INSERT INTO jintb_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jintb_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jintb_c VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jintb_d VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jintb AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jintb_a a JOIN jintb_b b ON a.key = b.key AND a.dt = b.dt " + + "INTERSECT " + + "SELECT c.id, d.value, c.dt " + + "FROM jintb_c c JOIN jintb_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jintb"); + + assertQuery("SELECT * FROM mv_jintb ORDER BY id", "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01')"); + + // Make BOTH A and C stale by adding to new partition with matching data + assertUpdate("INSERT INTO jintb_a VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + assertUpdate("INSERT INTO jintb_b VALUES (300, 'z', '2024-01-02'), (400, 'w', '2024-01-02')", 2); + assertUpdate("INSERT INTO jintb_c VALUES (3, 300, '2024-01-02'), (5, 500, '2024-01-02')", 2); + assertUpdate("INSERT INTO jintb_d VALUES (300, 'z', '2024-01-02'), (500, 'v', '2024-01-02')", 2); + + // Expected: + // - Creates two union branches (one for A stale, one for C stale) + // - For partition '2024-01-02': + // Left: (3, 'z'), (4, 'w') + // Right: (3, 'z'), (5, 'v') + // INTERSECT: (3, 'z') + // - Final: (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') from storage + (3, 'z', '2024-01-02') + assertQuery("SELECT * FROM mv_jintb ORDER BY id", + "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01'), (3, 'z', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jintb ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jintb"); + assertUpdate("DROP TABLE jintb_d"); + assertUpdate("DROP TABLE jintb_c"); + assertUpdate("DROP TABLE jintb_b"); + assertUpdate("DROP TABLE jintb_a"); + } + + /** + * Test (A JOIN B) INTERSECT (C JOIN D) where A and B both become stale (same side of INTERSECT). + * This tests that tables on the same side of a JOIN within INTERSECT are handled correctly. + */ + @Test + public void testJoinIntersectJoinWithSameSideJoinTablesStale() + { + assertUpdate("CREATE TABLE jints_a (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jints_b (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jints_c (id BIGINT, key BIGINT, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE jints_d (key BIGINT, value VARCHAR, dt VARCHAR) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data with full match + assertUpdate("INSERT INTO jints_a VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jints_b VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + assertUpdate("INSERT INTO jints_c VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO jints_d VALUES (100, 'x', '2024-01-01'), (200, 'y', '2024-01-01')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_jints AS " + + "SELECT a.id, b.value, a.dt " + + "FROM jints_a a JOIN jints_b b ON a.key = b.key AND a.dt = b.dt " + + "INTERSECT " + + "SELECT c.id, d.value, c.dt " + + "FROM jints_c c JOIN jints_d d ON c.key = d.key AND c.dt = d.dt"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_jints"); + + assertQuery("SELECT * FROM mv_jints ORDER BY id", "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01')"); + + // Make BOTH A and B stale (same side of INTERSECT, joined together) + // Also add matching data to C and D for INTERSECT to work + assertUpdate("INSERT INTO jints_a VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jints_b VALUES (300, 'z', '2024-01-02')", 1); + assertUpdate("INSERT INTO jints_c VALUES (3, 300, '2024-01-02')", 1); + assertUpdate("INSERT INTO jints_d VALUES (300, 'z', '2024-01-02')", 1); + + // Expected: + // - Creates two union branches (one for A, one for B) + // - For partition '2024-01-02': + // Left: (3, 'z') + // Right: (3, 'z') + // INTERSECT: (3, 'z') + // - Final: (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01') from storage + (3, 'z', '2024-01-02') + assertQuery("SELECT * FROM mv_jints ORDER BY id", + "VALUES (1, 'x', '2024-01-01'), (2, 'y', '2024-01-01'), (3, 'z', '2024-01-02')"); + + assertMaterializedViewResultsMatch(getSession(), "SELECT * FROM mv_jints ORDER BY id", true); + + assertUpdate("DROP MATERIALIZED VIEW mv_jints"); + assertUpdate("DROP TABLE jints_d"); + assertUpdate("DROP TABLE jints_c"); + assertUpdate("DROP TABLE jints_b"); + assertUpdate("DROP TABLE jints_a"); + } + + @Test + public void testExceptWithoutJoinRightSideStale() + { + // Create two tables with the same schema - partition column 'dt' maps through to MV + assertUpdate("CREATE TABLE except_nojoin_left (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE except_nojoin_right (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data: + // Left: (1, 100), (2, 200), (3, 300) for 2024-01-01 + // Right: (2, 200) for 2024-01-01 + // EXCEPT result: (1, 100), (3, 300) - rows in left but not in right + assertUpdate("INSERT INTO except_nojoin_left VALUES " + + "(1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01'), (3, 300, DATE '2024-01-01')", 3); + assertUpdate("INSERT INTO except_nojoin_right VALUES (2, 200, DATE '2024-01-01')", 1); + + // Create MV with pure EXCEPT (no JOINs) + assertUpdate("CREATE MATERIALIZED VIEW mv_except_nojoin " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM except_nojoin_left " + + "EXCEPT " + + "SELECT id, value, dt FROM except_nojoin_right"); + + assertRefreshAndFullyMaterialized("mv_except_nojoin", 2); + + assertQuery("SELECT id, value FROM mv_except_nojoin ORDER BY id", + "VALUES (1, 100), (3, 300)"); + + // Make RIGHT side stale by inserting a new partition + // This tests that predicate propagation works from right to left via PassthroughColumnEquivalences + // Right side: add (5, 500) for 2024-01-02 + assertUpdate("INSERT INTO except_nojoin_right VALUES (5, 500, DATE '2024-01-02')", 1); + + // Also add data to left side for the same partition + // Left: add (5, 500), (6, 600) for 2024-01-02 + // EXCEPT result for 2024-01-02: (6, 600) - since (5, 500) is in both + assertUpdate("INSERT INTO except_nojoin_left VALUES " + + "(5, 500, DATE '2024-01-02'), (6, 600, DATE '2024-01-02')", 2); + + // Query should produce correct results via stitching: + // - Fresh data from storage: (1, 100), (3, 300) for 2024-01-01 + // - Recomputed stale data: (6, 600) for 2024-01-02 + // The key test is that the right-side stale predicate (dt='2024-01-02') + // propagates to the left side via PassthroughColumnEquivalences, not EqualityInference + assertQuery("SELECT id, value FROM mv_except_nojoin ORDER BY id", + "VALUES (1, 100), (3, 300), (6, 600)"); + + // Verify stitching produces same result as full recompute + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM mv_except_nojoin ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_except_nojoin"); + assertUpdate("DROP TABLE except_nojoin_right"); + assertUpdate("DROP TABLE except_nojoin_left"); + } + + @Test + public void testExceptWithoutJoinBothSidesStale() + { + assertUpdate("CREATE TABLE except_both_left (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE except_both_right (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data for 2024-01-01 + assertUpdate("INSERT INTO except_both_left VALUES " + + "(1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO except_both_right VALUES (2, 200, DATE '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_except_both " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM except_both_left " + + "EXCEPT " + + "SELECT id, value, dt FROM except_both_right"); + + assertRefreshAndFullyMaterialized("mv_except_both", 1); + assertQuery("SELECT id, value FROM mv_except_both ORDER BY id", "VALUES (1, 100)"); + + // Make LEFT side stale: add partition 2024-01-02 + assertUpdate("INSERT INTO except_both_left VALUES (3, 300, DATE '2024-01-02')", 1); + + // Make RIGHT side stale: add partition 2024-01-03 + // Also add corresponding left data + assertUpdate("INSERT INTO except_both_right VALUES (4, 400, DATE '2024-01-03')", 1); + assertUpdate("INSERT INTO except_both_left VALUES " + + "(4, 400, DATE '2024-01-03'), (5, 500, DATE '2024-01-03')", 2); + + // Expected results: + // - 2024-01-01: (1, 100) from storage (fresh) + // - 2024-01-02: (3, 300) from left delta (left stale, right has nothing) + // - 2024-01-03: (5, 500) from right stale propagation + // (4, 400) is in both sides so excluded by EXCEPT + assertQuery("SELECT id, value FROM mv_except_both ORDER BY id", + "VALUES (1, 100), (3, 300), (5, 500)"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM mv_except_both ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_except_both"); + assertUpdate("DROP TABLE except_both_right"); + assertUpdate("DROP TABLE except_both_left"); + } + + @Test + public void testExceptBothSidesStaleSamePartition() + { + assertUpdate("CREATE TABLE except_same_left (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + assertUpdate("CREATE TABLE except_same_right (id INTEGER, value INTEGER, dt DATE) " + + "WITH (partitioning = ARRAY['dt'])"); + + // Initial data for dt='2024-01-01' + // Left: (1, 100), (2, 200) + // Right: (3, 300) -- doesn't overlap with Left + // EXCEPT result: (1, 100), (2, 200) + assertUpdate("INSERT INTO except_same_left VALUES " + + "(1, 100, DATE '2024-01-01'), (2, 200, DATE '2024-01-01')", 2); + assertUpdate("INSERT INTO except_same_right VALUES (3, 300, DATE '2024-01-01')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_except_same " + + "WITH (partitioning = ARRAY['dt']) AS " + + "SELECT id, value, dt FROM except_same_left " + + "EXCEPT " + + "SELECT id, value, dt FROM except_same_right"); + + assertRefreshAndFullyMaterialized("mv_except_same", 2); + assertQuery("SELECT id, value FROM mv_except_same ORDER BY id", "VALUES (1, 100), (2, 200)"); + + // Now add data to BOTH L and R in the SAME partition dt='2024-01-01': + // - L gets new row (10, 1000) + // - R gets (2, 200) which should SUBTRACT the existing L row (2, 200) + // + // After inserts: + // Left: (1, 100), (2, 200), (10, 1000) + // Right: (3, 300), (2, 200) + // EXCEPT result: (1, 100), (10, 1000) -- (2, 200) is now in R + assertUpdate("INSERT INTO except_same_left VALUES (10, 1000, DATE '2024-01-01')", 1); + assertUpdate("INSERT INTO except_same_right VALUES (2, 200, DATE '2024-01-01')", 1); + + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = 'mv_except_same'", + "SELECT 'PARTIALLY_MATERIALIZED'"); + + // Expected: (1, 100), (10, 1000) + // + // How the differential rewrite computes this correctly: + // - L's stale partition: dt='2024-01-01', R's stale partition: dt='2024-01-01' (same!) + // - deltaLeft: ∆L - R' = {(1,100),(2,200),(10,1000)} - {(3,300),(2,200)} = {(1,100),(10,1000)} + // Note: ∆L includes ALL rows from stale partitions (not just new rows) + // - deltaRight: L[unchanged] filtered to R's stale - R' + // L[unchanged] excludes dt='2024-01-01' (since L is stale there) = {} + // This is CORRECT - using unchanged prevents duplicates when L and R share stale partitions + // - deltaResult = {(1,100),(10,1000)} ∪ {} = {(1,100),(10,1000)} + // - freshPlan = {} (MV storage has no rows outside stale partitions) + // - Final: {} ∪ {(1,100),(10,1000)} = {(1,100),(10,1000)} ✓ + Session useStitchingSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_stale_read_behavior", "USE_STITCHING") + .build(); + assertQuery(useStitchingSession, + "SELECT id, value FROM mv_except_same ORDER BY id", + "VALUES (1, 100), (10, 1000)"); + + assertMaterializedViewResultsMatch(getSession(), + "SELECT id, value FROM mv_except_same ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_except_same"); + assertUpdate("DROP TABLE except_same_right"); + assertUpdate("DROP TABLE except_same_left"); + } + + @Test + public void testExceptJoinTableScan() + { + // Pattern: (A EXCEPT B) JOIN C + // Tests that EXCEPT's unchanged variant correctly accounts for anti-monotonicity + // in the right input when used as the left child of a JOIN. + assertUpdate("CREATE TABLE exjt_a (id BIGINT, key BIGINT) " + + "WITH (partitioning = ARRAY['id', 'key'])"); + assertUpdate("CREATE TABLE exjt_b (id BIGINT, key BIGINT) " + + "WITH (partitioning = ARRAY['id', 'key'])"); + assertUpdate("CREATE TABLE exjt_c (key BIGINT, value VARCHAR) " + + "WITH (partitioning = ARRAY['key', 'value'])"); + + // Left side (A EXCEPT B): produces (1, 100), (3, 300) + assertUpdate("INSERT INTO exjt_a VALUES (1, 100), (2, 200), (3, 300), (4, 400)", 4); + assertUpdate("INSERT INTO exjt_b VALUES (2, 200), (4, 400), (5, 500)", 3); + // Right side C: (300, 'z') + assertUpdate("INSERT INTO exjt_c VALUES (300, 'z')", 1); + + // MV: (A EXCEPT B) JOIN C -> should produce (3, 'z') + assertUpdate("CREATE MATERIALIZED VIEW mv_exjt AS " + + "WITH nt(id, key) AS (SELECT * FROM exjt_a EXCEPT SELECT * FROM exjt_b) " + + "SELECT a.id, b.value " + + "FROM nt a JOIN exjt_c b ON a.key = b.key"); + getQueryRunner().execute("REFRESH MATERIALIZED VIEW mv_exjt"); + + // Verify initial state + assertQuery("SELECT * FROM mv_exjt ORDER BY id", "VALUES (3, 'z')"); + + // Now make changes: + // A gains (5, 500), (6, 600), (7, 700) + // B gains (1, 100), (6, 600) -- note: (1, 100) was in A EXCEPT B before, now removed + // C gains (100, 'x'), (700, 'h') + assertUpdate("INSERT INTO exjt_a VALUES (5, 500), (6, 600), (7, 700)", 3); + assertUpdate("INSERT INTO exjt_b VALUES (1, 100), (6, 600)", 2); + assertUpdate("INSERT INTO exjt_c VALUES (100, 'x'), (700, 'h')", 2); + + // After updates: + // A EXCEPT B = {(3, 300), (7, 700)} + // - (1, 100) removed because B now has it + // - (5, 500) in both A and B + // - (6, 600) in both A and B + // - (7, 700) only in A + // (A EXCEPT B) JOIN C = {(3, 'z'), (7, 'h')} + // - (3, 300) joins C's (300, 'z') + // - (7, 700) joins C's (700, 'h') + // - (1, 100) should NOT appear since it's no longer in A EXCEPT B + assertQuery("SELECT * FROM mv_exjt ORDER BY id", "VALUES (3, 'z'), (7, 'h')"); + assertMaterializedViewResultsMatch(getSession(), + "SELECT * FROM mv_exjt ORDER BY id", + true); + + assertUpdate("DROP MATERIALIZED VIEW mv_exjt"); + assertUpdate("DROP TABLE exjt_c"); + assertUpdate("DROP TABLE exjt_b"); + assertUpdate("DROP TABLE exjt_a"); + } + + private void assertMaterializedViewQuery(@Language("SQL") String actual, @Language("SQL") String expected) + { + assertQuery(actual, expected); + assertMaterializedViewResultsMatch(actual); + } + + private void assertRefreshAndFullyMaterialized(String viewName, long expectedRows) + { + assertUpdate("REFRESH MATERIALIZED VIEW " + viewName, expectedRows); + assertQuery( + "SELECT freshness_state FROM information_schema.materialized_views " + + "WHERE table_schema = 'test_schema' AND table_name = '" + viewName + "'", + "SELECT 'FULLY_MATERIALIZED'"); + } + + private void assertMaterializedViewResultsMatch(String query) + { + assertMaterializedViewResultsMatch(query, true); + } + + private void assertMaterializedViewResultsMatch(Session session, String query) + { + assertMaterializedViewResultsMatch(session, query, true); + } + + private void assertMaterializedViewResultsMatch(String query, boolean assertOrdered) + { + // Query with USE_STITCHING mode (default - uses storage + stitching) + Session withStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_stale_read_behavior", "USE_STITCHING") + .build(); + + // Query with USE_VIEW_QUERY mode (forces recompute from base tables) + Session skipStorageSession = Session.builder(getQueryRunner().getDefaultSession()) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + + // Verify that both approaches produce identical results + if (assertOrdered) { + assertQueryWithSameQueryRunner(skipStorageSession, query, withStorageSession, query); + } + else { + // For unordered comparison, materialize both results and compare as sets + MaterializedResult withStorageResult = computeActual(withStorageSession, query); + MaterializedResult skipStorageResult = computeActual(skipStorageSession, query); + assertEqualsIgnoreOrder(skipStorageResult.getMaterializedRows(), withStorageResult.getMaterializedRows()); + } + } + + private void assertMaterializedViewResultsMatch(Session session, String query, boolean assertOrdered) + { + // Query with USE_STITCHING mode (default - uses storage + stitching) + Session withStorageSession = Session.builder(session) + .setSystemProperty("materialized_view_stale_read_behavior", "USE_STITCHING") + .build(); + + // Query with USE_VIEW_QUERY mode (forces recompute from base tables) + Session skipStorageSession = Session.builder(session) + .setSystemProperty("materialized_view_force_stale", "true") + .setSystemProperty("materialized_view_stale_read_behavior", "USE_VIEW_QUERY") + .build(); + + // Verify that both approaches produce identical results + if (assertOrdered) { + assertQueryWithSameQueryRunner(skipStorageSession, query, withStorageSession, query); + } + else { + // For unordered comparison, materialize both results and compare as sets + MaterializedResult withStorageResult = computeActual(withStorageSession, query); + MaterializedResult skipStorageResult = computeActual(skipStorageSession, query); + assertEqualsIgnoreOrder(skipStorageResult.getMaterializedRows(), withStorageResult.getMaterializedRows()); + } + } + + @Test + public void testSecurityInvokerWithRowFilterBlocksStitching() + { + assertUpdate("CREATE TABLE mv_security_base (id BIGINT, value BIGINT, ds VARCHAR) WITH (partitioning = ARRAY['ds'])"); + assertUpdate("INSERT INTO mv_security_base VALUES (1, 100, '2024-01-01'), (2, 200, '2024-01-01')", 2); + assertUpdate("INSERT INTO mv_security_base VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_security_test SECURITY INVOKER " + + "WITH (partitioning = ARRAY['ds']) AS SELECT id, value, ds FROM mv_security_base"); + assertUpdate("REFRESH MATERIALIZED VIEW mv_security_test", 4); + + // Insert new data to make MV partially stale + assertUpdate("INSERT INTO mv_security_base VALUES (5, 500, '2024-01-03'), (6, 600, '2024-01-03')", 2); + + try { + // Add row filter: restricted_user cannot see ds='2024-01-01' + getQueryRunner().getAccessControl().rowFilter( + new QualifiedObjectName("iceberg", "test_schema", "mv_security_base"), + "restricted_user", + new ViewExpression("restricted_user", Optional.of("iceberg"), Optional.of("test_schema"), "ds <> '2024-01-01'")); + + Session restrictedStitchingSession = Session.builder(getQueryRunner().getDefaultSession()) + .setIdentity(new Identity("restricted_user", Optional.empty())) + .setSystemProperty("materialized_view_stale_read_behavior", "USE_STITCHING") + .build(); + + // With the security fix: + // - Stitching is blocked because row filter exists on base table with INVOKER security + // - Falls back to view query which applies the row filter + // - restricted_user should NOT see ds='2024-01-01' rows + // Without the fix, stitching would bypass row filters for fresh partitions + assertQuery(restrictedStitchingSession, + "SELECT id, value, ds FROM mv_security_test ORDER BY id", + "VALUES (3, 300, '2024-01-02'), (4, 400, '2024-01-02'), (5, 500, '2024-01-03'), (6, 600, '2024-01-03')"); + } + finally { + getQueryRunner().getAccessControl().reset(); + assertUpdate("DROP MATERIALIZED VIEW mv_security_test"); + assertUpdate("DROP TABLE mv_security_base"); + } + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergRestMaterializedViews.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergRestMaterializedViews.java new file mode 100644 index 0000000000000..fb261e01fa232 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergRestMaterializedViews.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; +import org.assertj.core.util.Files; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.iceberg.CatalogType.REST; +import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.getRestServer; +import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.restConnectorProperties; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; + +@Test(singleThreaded = true) +public class TestIcebergRestMaterializedViews + extends TestIcebergMaterializedViewsBase +{ + private TestingHttpServer restServer; + private String serverUri; + + @BeforeClass + @Override + public void init() + throws Exception + { + warehouseLocation = Files.newTemporaryFolder(); + + restServer = getRestServer(warehouseLocation.getAbsolutePath()); + restServer.start(); + + serverUri = restServer.getBaseUrl().toString(); + super.init(); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + throws Exception + { + if (restServer != null) { + restServer.stop(); + } + if (warehouseLocation != null) { + deleteRecursively(warehouseLocation.toPath(), ALLOW_INSECURE); + } + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder() + .setCatalogType(REST) + .setExtraConnectorProperties(restConnectorProperties(serverUri)) + .setDataDirectory(Optional.of(warehouseLocation.toPath())) + .setSchemaName("test_schema") + .setCreateTpchTables(false) + .setExtraProperties(ImmutableMap.of("experimental.legacy-materialized-views", "false")) + .build().getQueryRunner(); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTableName.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTableName.java index 7cabaeea0e8d3..fef4bdc0d6e60 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTableName.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergTableName.java @@ -79,4 +79,74 @@ private static void assertFrom(String inputName, String tableName, IcebergTableT { assertFrom(inputName, tableName, icebergTableType, Optional.empty(), Optional.empty()); } + + @Test + public void testBranchParsing() + { + // Basic branch parsing + assertFromWithBranch("orders.branch_audit_branch", "orders", IcebergTableType.DATA, Optional.empty(), Optional.of("audit_branch")); + assertFromWithBranch("customers.branch_dev", "customers", IcebergTableType.DATA, Optional.empty(), Optional.of("dev")); + assertFromWithBranch("table.branch_feature_123", "table", IcebergTableType.DATA, Optional.empty(), Optional.of("feature_123")); + // Branch with underscores and hyphens + assertFromWithBranch("orders.branch_audit_branch_v2", "orders", IcebergTableType.DATA, Optional.empty(), Optional.of("audit_branch_v2")); + assertFromWithBranch("orders.branch_test-branch", "orders", IcebergTableType.DATA, Optional.empty(), Optional.of("test-branch")); + // Branch with table types (allowed combinations) + assertFromWithBranch("orders.branch_audit$history", "orders", IcebergTableType.HISTORY, Optional.empty(), Optional.of("audit")); + assertFromWithBranch("orders.branch_audit$snapshots", "orders", IcebergTableType.SNAPSHOTS, Optional.empty(), Optional.of("audit")); + assertFromWithBranch("orders.branch_audit$partitions", "orders", IcebergTableType.PARTITIONS, Optional.empty(), Optional.of("audit")); + assertFromWithBranch("orders.branch_audit$manifests", "orders", IcebergTableType.MANIFESTS, Optional.empty(), Optional.of("audit")); + assertFromWithBranch("orders.branch_audit$files", "orders", IcebergTableType.FILES, Optional.empty(), Optional.of("audit")); + assertFromWithBranch("orders.branch_audit$changelog", "orders", IcebergTableType.CHANGELOG, Optional.empty(), Optional.of("audit")); + // Branch with snapshot version should be rejected (branches and snapshots are mutually exclusive) + assertInvalid("orders.branch_audit@123", "Invalid Iceberg table name (cannot use @ version with branch): orders.branch_audit@123"); + assertInvalid("orders.branch_audit$data@123", "Invalid Iceberg table name (cannot use @ version with branch): orders.branch_audit$data@123"); + assertInvalid("orders.branch_audit$partitions@456", "Invalid Iceberg table name (cannot use @ version with branch): orders.branch_audit$partitions@456"); + // Verify no branch is parsed for regular tables + assertFromWithBranch("orders", "orders", IcebergTableType.DATA, Optional.empty(), Optional.empty()); + assertFromWithBranch("orders@123", "orders", IcebergTableType.DATA, Optional.of(123L), Optional.empty()); + assertFromWithBranch("orders$history", "orders", IcebergTableType.HISTORY, Optional.empty(), Optional.empty()); + } + + @Test + public void testBranchJsonRoundTrip() + { + // Test JSON serialization/deserialization preserves branchName + IcebergTableName original = IcebergTableName.from("orders.branch_audit_branch"); + assertEquals(original.getTableName(), "orders"); + assertEquals(original.getTableType(), IcebergTableType.DATA); + assertEquals(original.getBranchName(), Optional.of("audit_branch")); + assertEquals(original.getSnapshotId(), Optional.empty()); + // Create a new instance with same values (simulating JSON round-trip) + IcebergTableName roundTrip = new IcebergTableName( + original.getTableName(), + original.getTableType(), + original.getSnapshotId(), + original.getBranchName(), + original.getChangelogEndSnapshot()); + assertEquals(roundTrip.getTableName(), "orders"); + assertEquals(roundTrip.getTableType(), IcebergTableType.DATA); + assertEquals(roundTrip.getBranchName(), Optional.of("audit_branch")); + assertEquals(roundTrip.getSnapshotId(), Optional.empty()); + // Test with branch and table type + IcebergTableName withType = IcebergTableName.from("orders.branch_audit$history"); + assertEquals(withType.getTableName(), "orders"); + assertEquals(withType.getTableType(), IcebergTableType.HISTORY); + assertEquals(withType.getBranchName(), Optional.of("audit")); + IcebergTableName withTypeRoundTrip = new IcebergTableName( + withType.getTableName(), + withType.getTableType(), + withType.getSnapshotId(), + withType.getBranchName(), + withType.getChangelogEndSnapshot()); + assertEquals(withTypeRoundTrip.getBranchName(), Optional.of("audit")); + } + + private static void assertFromWithBranch(String inputName, String tableName, IcebergTableType icebergTableType, Optional snapshotId, Optional branchName) + { + IcebergTableName name = IcebergTableName.from(inputName); + assertEquals(name.getTableName(), tableName); + assertEquals(name.getTableType(), icebergTableType); + assertEquals(name.getSnapshotId(), snapshotId); + assertEquals(name.getBranchName(), branchName); + } } diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java new file mode 100644 index 0000000000000..fb28aee470d42 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/TestIcebergV3.java @@ -0,0 +1,477 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg; + +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.io.CloseableIterable; +import org.testng.annotations.Test; + +import java.io.File; +import java.nio.file.Path; +import java.util.Map; +import java.util.OptionalInt; + +import static com.facebook.presto.iceberg.CatalogType.HADOOP; +import static com.facebook.presto.iceberg.FileFormat.PARQUET; +import static com.facebook.presto.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; +import static com.facebook.presto.iceberg.IcebergQueryRunner.getIcebergDataDirectoryPath; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; + +public class TestIcebergV3 + extends AbstractTestQueryFramework +{ + private static final String TEST_SCHEMA = "tpch"; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder() + .setCatalogType(HADOOP) + .setFormat(PARQUET) + .setNodeCount(OptionalInt.of(1)) + .setCreateTpchTables(false) + .setAddJmxPlugin(false) + .build().getQueryRunner(); + } + + private void dropTable(String tableName) + { + assertQuerySucceeds("DROP TABLE IF EXISTS " + tableName); + } + + @Test + public void testCreateV3Table() + { + String tableName = "test_create_v3_table"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + assertQuery("SELECT * FROM " + tableName, "SELECT * WHERE false"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testCreateUnsupportedFormatVersion() + { + String tableName = "test_create_v4_table"; + // Ensure clean state in case a previous run created the table + dropTable(tableName); + + assertQueryFails( + "CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '4')", + ".*Iceberg table format version 4 is not supported.*"); + } + + @Test + public void testUpgradeV2ToV3() + { + String tableName = "test_upgrade_v2_to_v3"; + try { + // Create v2 table + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '2')"); + Table table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 2); + + // Upgrade to v3 + BaseTable baseTable = (BaseTable) table; + TableOperations operations = baseTable.operations(); + TableMetadata currentMetadata = operations.current(); + operations.commit(currentMetadata, currentMetadata.upgradeToFormatVersion(3)); + + // Verify the upgrade + table = loadTable(tableName); + assertEquals(((BaseTable) table).operations().current().formatVersion(), 3); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testInsertIntoV3Table() + { + String tableName = "test_insert_v3_table"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + assertQuery("SELECT * FROM " + tableName, "VALUES (1, 'one'), (2, 'two')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'three')", 1); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testDeleteOnV3TableNotSupported() + { + String tableName = "test_v3_delete"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.delete.mode\" = 'merge-on-read')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)", 3); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)"); + assertThatThrownBy(() -> getQueryRunner().execute("DELETE FROM " + tableName + " WHERE id = 1")) + .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testTruncateV3Table() + { + String tableName = "test_v3_truncate"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0), (3, 'Charlie', 300.0)", 3); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + + assertUpdate("DELETE FROM " + tableName, 3); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); + + assertUpdate("INSERT INTO " + tableName + " VALUES (4, 'Dave', 400.0)", 1); + assertQuery("SELECT * FROM " + tableName, "VALUES (4, 'Dave', 400.0)"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testMetadataDeleteOnV3PartitionedTable() + { + String tableName = "test_v3_metadata_delete"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE, part VARCHAR)" + + " WITH (\"format-version\" = '3', partitioning = ARRAY['part'])"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 100.0, 'A'), (2, 'Bob', 200.0, 'A')," + + " (3, 'Charlie', 300.0, 'B'), (4, 'Dave', 400.0, 'C')", 4); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 4"); + + assertUpdate("DELETE FROM " + tableName + " WHERE part = 'A'", 2); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 2"); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (3, 'Charlie', 300.0, 'B'), (4, 'Dave', 400.0, 'C')"); + + assertUpdate("DELETE FROM " + tableName + " WHERE part = 'B'", 1); + assertQuery("SELECT * FROM " + tableName, "VALUES (4, 'Dave', 400.0, 'C')"); + + assertUpdate("DELETE FROM " + tableName + " WHERE part = 'C'", 1); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 0"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testUpdateOnV3TableNotSupported() + { + String tableName = "test_v3_update"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, status VARCHAR, score DOUBLE) WITH (\"format-version\" = '3', \"write.update.mode\" = 'merge-on-read')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'Alice', 'active', 85.5), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)", + 3); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'Alice', 'active', 85.5), (2, 'Bob', 'active', 92.0), (3, 'Charlie', 'inactive', 78.3)"); + assertThatThrownBy(() -> getQueryRunner() + .execute("UPDATE " + tableName + " SET status = 'updated', score = 95.0 WHERE id = 1")) + .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testMergeOnV3TableNotSupported() + { + String tableName = "test_v3_merge_target"; + String sourceTable = "test_v3_merge_source"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, value DOUBLE) WITH (\"format-version\" = '3', \"write.update.mode\" = 'merge-on-read')"); + assertUpdate("CREATE TABLE " + sourceTable + " (id INTEGER, name VARCHAR, value DOUBLE)"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)", 2); + assertUpdate("INSERT INTO " + sourceTable + " VALUES (1, 'Alice Updated', 150.0), (3, 'Charlie', 300.0)", + 2); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", "VALUES (1, 'Alice', 100.0), (2, 'Bob', 200.0)"); + assertQuery("SELECT * FROM " + sourceTable + " ORDER BY id", + "VALUES (1, 'Alice Updated', 150.0), (3, 'Charlie', 300.0)"); + assertThatThrownBy(() -> getQueryRunner().execute( + "MERGE INTO " + tableName + " t USING " + sourceTable + " s ON t.id = s.id " + + "WHEN MATCHED THEN UPDATE SET name = s.name, value = s.value " + + "WHEN NOT MATCHED THEN INSERT (id, name, value) VALUES (s.id, s.name, s.value)")) + .hasMessageContaining("Iceberg table updates for format version 3 are not supported yet"); + } + finally { + dropTable(tableName); + dropTable(sourceTable); + } + } + + @Test + public void testOptimizeOnV3Table() + { + String tableName = "test_v3_optimize"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, category VARCHAR, value DOUBLE) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'A', 100.0)", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (2, 'B', 200.0)", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (3, 'A', 150.0)", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES (4, 'C', 300.0)", 1); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'A', 100.0), (2, 'B', 200.0), (3, 'A', 150.0), (4, 'C', 300.0)"); + + assertQuerySucceeds(format("CALL system.rewrite_data_files('%s', '%s')", TEST_SCHEMA, tableName)); + + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'A', 100.0), (2, 'B', 200.0), (3, 'A', 150.0), (4, 'C', 300.0)"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testPuffinDeletionVectorsNotSupported() + throws Exception + { + String tableName = "test_puffin_deletion_vectors_not_supported"; + try { + assertUpdate("CREATE TABLE " + tableName + " (id integer, value varchar) WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'one'), (2, 'two')", 2); + + Table table = loadTable(tableName); + + // Attach a PUFFIN delete vector to an existing data file in the v3 table + try (CloseableIterable tasks = table.newScan().planFiles()) { + FileScanTask task = tasks.iterator().next(); + + DeleteFile puffinDeleteFile = FileMetadata.deleteFileBuilder(task.spec()) + .ofPositionDeletes() + .withPath(task.file().path().toString() + ".puffin") + .withFileSizeInBytes(16) + .withFormat(FileFormat.PUFFIN) + .withRecordCount(1) + .withContentOffset(0) + .withContentSizeInBytes(16) + .withReferencedDataFile(task.file().path().toString()) + .build(); + + table.newRowDelta() + .addDeletes(puffinDeleteFile) + .commit(); + } + + assertQueryFails("SELECT * FROM " + tableName, "Iceberg deletion vectors.*PUFFIN.*not supported"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3SupportedOperations() + { + String tableName = "test_v3_supported"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, name VARCHAR, created_date DATE, amount DECIMAL(10,2)) WITH (\"format-version\" = '3', partitioning = ARRAY['created_date'])"); + + assertUpdate("INSERT INTO " + tableName + " VALUES " + + "(1, 'Transaction A', DATE '2024-01-01', 100.50), " + + "(2, 'Transaction B', DATE '2024-01-02', 250.75), " + + "(3, 'Transaction C', DATE '2024-01-01', 175.00)", 3); + + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES " + + "(1, 'Transaction A', DATE '2024-01-01', 100.50), " + + "(2, 'Transaction B', DATE '2024-01-02', 250.75), " + + "(3, 'Transaction C', DATE '2024-01-01', 175.00)"); + + assertQuery( + "SELECT created_date, count(*), sum(amount) FROM " + tableName + + " GROUP BY created_date ORDER BY created_date", + "VALUES " + + "(DATE '2024-01-01', 2, 275.50), " + + "(DATE '2024-01-02', 1, 250.75)"); + + assertQuery("SELECT * FROM " + tableName + + " WHERE created_date = DATE '2024-01-01' ORDER BY id", + "VALUES " + + "(1, 'Transaction A', DATE '2024-01-01', 100.50), " + + "(3, 'Transaction C', DATE '2024-01-01', 175.00)"); + + assertUpdate("INSERT INTO " + tableName + " VALUES (4, 'Transaction D', DATE '2024-01-03', 300.00)", 1); + + assertQuery("SELECT count(*) as total_count FROM " + tableName, "SELECT 4"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testSelectFromV3TableAfterInsert() + { + String tableName = "test_select_v3_table"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id integer, name varchar, price decimal(10,2))" + + " WITH (\"format-version\" = '3')"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'apple', 1.50), (2, 'banana', 0.75)," + + " (3, 'cherry', 2.00)", 3); + assertQuery("SELECT * FROM " + tableName + " ORDER BY id", + "VALUES (1, 'apple', 1.50), (2, 'banana', 0.75)," + + " (3, 'cherry', 2.00)"); + assertQuery("SELECT count(*) FROM " + tableName, "SELECT 3"); + assertQuery("SELECT sum(price) FROM " + tableName, "SELECT 4.25"); + assertQuery("SELECT name FROM " + tableName + + " WHERE price > 1.00 ORDER BY name", + "VALUES ('apple'), ('cherry')"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3TableWithPartitioning() + { + String tableName = "test_v3_partitioned_table"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id integer, category varchar, value integer)" + + " WITH (\"format-version\" = '3', partitioning = ARRAY['category'])"); + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'A', 100), (2, 'B', 200), (3, 'A', 150)", 3); + assertQuery("SELECT * FROM " + tableName + + " WHERE category = 'A' ORDER BY id", + "VALUES (1, 'A', 100), (3, 'A', 150)"); + assertQuery("SELECT category, sum(value) FROM " + tableName + + " GROUP BY category ORDER BY category", + "VALUES ('A', 250), ('B', 200)"); + } + finally { + dropTable(tableName); + } + } + + @Test + public void testV3TableEncryptionNotSupported() + { + String tableName = "test_v3_encrypted"; + try { + assertUpdate("CREATE TABLE " + tableName + + " (id INTEGER, data VARCHAR)" + + " WITH (\"format-version\" = '3')"); + // Insert data so the table has a snapshot + // (validation requires a non-null snapshot) + assertUpdate("INSERT INTO " + tableName + + " VALUES (1, 'unencrypted')", 1); + + // Set encryption property via the Iceberg API + Table table = loadTable(tableName); + table.updateProperties() + .set("encryption.key-id", "test-key-id") + .commit(); + + // Both SELECT and INSERT should fail because the validation + // rejects encryption + assertThatThrownBy(() -> getQueryRunner().execute( + "SELECT * FROM " + tableName)) + .hasMessageContaining( + "Iceberg table encryption is not supported"); + + assertThatThrownBy(() -> getQueryRunner().execute( + "INSERT INTO " + tableName + " VALUES (2, 'more')")) + .hasMessageContaining( + "Iceberg table encryption is not supported"); + } + finally { + // Use Iceberg API to drop table directly, bypassing Presto's + // validateTableForPresto + dropTableViaIceberg(tableName); + } + } + + private Table loadTable(String tableName) + { + Catalog catalog = CatalogUtil.loadCatalog( + HadoopCatalog.class.getName(), ICEBERG_CATALOG, + getProperties(), new Configuration()); + return catalog.loadTable(TableIdentifier.of(TEST_SCHEMA, tableName)); + } + + private Map getProperties() + { + File metastoreDir = getCatalogDirectory(); + return ImmutableMap.of("warehouse", metastoreDir.toString()); + } + + private File getCatalogDirectory() + { + Path dataDirectory = getDistributedQueryRunner() + .getCoordinator().getDataDirectory(); + Path catalogDirectory = getIcebergDataDirectoryPath( + dataDirectory, HADOOP.name(), + new IcebergConfig().getFileFormat(), false); + return catalogDirectory.toFile(); + } + + private void dropTableViaIceberg(String tableName) + { + Catalog catalog = CatalogUtil.loadCatalog( + HadoopCatalog.class.getName(), ICEBERG_CATALOG, + getProperties(), new Configuration()); + catalog.dropTable( + TableIdentifier.of(TEST_SCHEMA, tableName), true); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergMaterializedViewsHive.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergMaterializedViewsHive.java new file mode 100644 index 0000000000000..e8b32c8201968 --- /dev/null +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestIcebergMaterializedViewsHive.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.iceberg.hive; + +import com.facebook.presto.iceberg.TestIcebergMaterializedViewsBase; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; +import org.assertj.core.util.Files; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.iceberg.CatalogType.HIVE; +import static com.facebook.presto.iceberg.IcebergQueryRunner.builder; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; + +@Test(singleThreaded = true) +public class TestIcebergMaterializedViewsHive + extends TestIcebergMaterializedViewsBase +{ + @BeforeClass + @Override + public void init() + throws Exception + { + warehouseLocation = Files.newTemporaryFolder(); + super.init(); + getQueryRunner().execute("CREATE SCHEMA IF NOT EXISTS test_schema"); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + throws Exception + { + deleteRecursively(warehouseLocation.toPath(), ALLOW_INSECURE); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return builder() + .setCatalogType(HIVE) + .setDataDirectory(Optional.of(warehouseLocation.toPath())) + .setSchemaName("test_schema") + .setCreateTpchTables(false) + .setExtraProperties(ImmutableMap.of("experimental.legacy-materialized-views", "false")) + .build().getQueryRunner(); + } +} diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java index 4fe2febcddba9..0bb878c24fa31 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/hive/TestRenameTableOnFragileFileSystem.java @@ -202,7 +202,7 @@ public String formatRowExpression(ConnectorSession session, RowExpression expres private static final String newTablePermissionFilePath = String.format("%s/%s/%s/%s/%s", catalogDirectory, newSchemaName, newTableName, ".prestoPermissions", "testFile"); IcebergTableHandle icebergTableHandle = new IcebergTableHandle(originSchemaName, - new IcebergTableName(originTableName, IcebergTableType.DATA, Optional.empty(), Optional.empty()), + new IcebergTableName(originTableName, IcebergTableType.DATA, Optional.empty(), Optional.empty(), Optional.empty()), false, Optional.empty(), Optional.empty(), diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergDistributedRest.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergDistributedRest.java index 12ce9afd56b9f..53e076fcdef2c 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergDistributedRest.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/rest/TestIcebergDistributedRest.java @@ -87,6 +87,10 @@ protected QueryRunner createQueryRunner() Map connectorProperties = ImmutableMap.builder() .putAll(restConnectorProperties(serverUri)) .put("iceberg.rest.session.type", SessionType.USER.name()) + // Enable OAuth2 authentication to trigger token exchange flow + // The credential is required to initialize the OAuth2Manager + .put("iceberg.rest.auth.type", "OAUTH2") + .put("iceberg.rest.auth.oauth2.credential", "client:secret") .build(); return IcebergQueryRunner.builder() diff --git a/presto-iceberg/src/test/java/org/apache/iceberg/rest/IcebergRestCatalogServlet.java b/presto-iceberg/src/test/java/org/apache/iceberg/rest/IcebergRestCatalogServlet.java index a25bb089000b7..8b48eaf7232dc 100644 --- a/presto-iceberg/src/test/java/org/apache/iceberg/rest/IcebergRestCatalogServlet.java +++ b/presto-iceberg/src/test/java/org/apache/iceberg/rest/IcebergRestCatalogServlet.java @@ -28,6 +28,7 @@ import org.apache.iceberg.rest.HTTPRequest.HTTPMethod; import org.apache.iceberg.rest.RESTCatalogAdapter.Route; import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.util.Pair; import java.io.IOException; @@ -53,6 +54,11 @@ public class IcebergRestCatalogServlet { private static final Logger LOG = Logger.get(IcebergRestCatalogServlet.class); + private static final String SUBJECT_TOKEN = "subject_token"; + private static final String GRANT_TYPE = "grant_type"; + private static final String TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"; + private static final String TOKEN_EXCHANGE_PREFIX = "token-exchange-token:sub="; + private final RESTCatalogAdapter restCatalogAdapter; private final Map responseHeaders = ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); @@ -105,11 +111,32 @@ protected void execute(ServletRequestContext context, HttpServletResponse respon } if (context.error().isPresent()) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - RESTObjectMapper.mapper().writeValue(response.getWriter(), context.error().get()); + ErrorResponse error = context.error().get(); + response.setStatus(error.code()); + RESTObjectMapper.mapper().writeValue(response.getWriter(), error); return; } + // Handle token exchange requests specially to preserve user identity + if (context.route() == Route.TOKENS && context.body() instanceof Map) { + @SuppressWarnings("unchecked") + Map tokenRequest = (Map) context.body(); + String grantType = tokenRequest.get(GRANT_TYPE); + String subjectToken = tokenRequest.get(SUBJECT_TOKEN); + + if (TOKEN_EXCHANGE_GRANT_TYPE.equals(grantType) && subjectToken != null) { + // Return the subject token prefixed so that authorization check can extract the original JWT + String responseToken = TOKEN_EXCHANGE_PREFIX + subjectToken; + OAuthTokenResponse oauthResponse = OAuthTokenResponse.builder() + .withToken(responseToken) + .withTokenType("Bearer") + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .build(); + RESTObjectMapper.mapper().writeValue(response.getWriter(), oauthResponse); + return; + } + } + try { HTTPRequest request = restCatalogAdapter.buildRequest( context.method(), diff --git a/presto-internal-communication/pom.xml b/presto-internal-communication/pom.xml new file mode 100644 index 0000000000000..182e4ca56a512 --- /dev/null +++ b/presto-internal-communication/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.297-SNAPSHOT + + + presto-internal-communication + presto-internal-communication + Presto - Internal Communication and Authentication + + + ${project.parent.basedir} + true + + + + + com.facebook.airlift + http-client + + + + com.facebook.airlift + configuration + + + + com.facebook.airlift + node + + + + com.facebook.airlift + units + + + + com.facebook.airlift + log + + + + com.facebook.airlift + http-server + + + + com.facebook.airlift + jaxrs + + + + com.google.inject + guice + + + + com.google.guava + guava + + + + io.jsonwebtoken + jjwt-api + + + + jakarta.ws.rs + jakarta.ws.rs-api + + + + jakarta.inject + jakarta.inject-api + + + + jakarta.validation + jakarta.validation-api + + + + jakarta.servlet + jakarta.servlet-api + + + + jakarta.annotation + jakarta.annotation-api + + + + com.facebook.airlift.drift + drift-transport-netty + + + + com.fasterxml.jackson.core + jackson-annotations + + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/BasicPrincipal.java b/presto-internal-communication/src/main/java/com/facebook/presto/security/BasicPrincipal.java similarity index 100% rename from presto-main-base/src/main/java/com/facebook/presto/security/BasicPrincipal.java rename to presto-internal-communication/src/main/java/com/facebook/presto/security/BasicPrincipal.java diff --git a/presto-internal-communication/src/main/java/com/facebook/presto/server/CommonInternalCommunicationModule.java b/presto-internal-communication/src/main/java/com/facebook/presto/server/CommonInternalCommunicationModule.java new file mode 100644 index 0000000000000..b0c3ca45e1b59 --- /dev/null +++ b/presto-internal-communication/src/main/java/com/facebook/presto/server/CommonInternalCommunicationModule.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.server; + +import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; +import com.facebook.airlift.http.client.HttpClientConfig; +import com.facebook.airlift.http.client.spnego.KerberosConfig; +import com.facebook.presto.server.security.InternalAuthenticationFilter; +import com.google.inject.Binder; +import com.google.inject.Module; + +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Locale; + +import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; +import static com.facebook.airlift.http.server.KerberosConfig.HTTP_SERVER_AUTHENTICATION_KRB5_KEYTAB; +import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static com.facebook.presto.server.InternalCommunicationConfig.INTERNAL_COMMUNICATION_KERBEROS_ENABLED; +import static com.google.common.base.Verify.verify; + +public class CommonInternalCommunicationModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + InternalCommunicationConfig internalCommunicationConfig = buildConfigObject(InternalCommunicationConfig.class); + configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, config -> { + config.setKeyStorePath(internalCommunicationConfig.getKeyStorePath()); + config.setKeyStorePassword(internalCommunicationConfig.getKeyStorePassword()); + config.setTrustStorePath(internalCommunicationConfig.getTrustStorePath()); + config.setTrustStorePassword(internalCommunicationConfig.getTrustStorePassword()); + if (internalCommunicationConfig.getIncludedCipherSuites().isPresent()) { + config.setHttpsIncludedCipherSuites(internalCommunicationConfig.getIncludedCipherSuites().get()); + } + if (internalCommunicationConfig.getExcludeCipherSuites().isPresent()) { + config.setHttpsExcludedCipherSuites(internalCommunicationConfig.getExcludeCipherSuites().get()); + } + }); + + install(installModuleIf(InternalCommunicationConfig.class, InternalCommunicationConfig::isKerberosEnabled, kerberosInternalCommunicationModule())); + binder.bind(InternalAuthenticationManager.class); + httpClientBinder(binder).bindGlobalFilter(InternalAuthenticationManager.class); + jaxrsBinder(binder).bind(InternalAuthenticationFilter.class); + } + + private Module kerberosInternalCommunicationModule() + { + return binder -> { + InternalCommunicationConfig clientKerberosConfig = buildConfigObject(InternalCommunicationConfig.class); + com.facebook.airlift.http.server.KerberosConfig serverKerberosConfig = buildConfigObject(com.facebook.airlift.http.server.KerberosConfig.class); + verify(serverKerberosConfig.getKeytab() != null, "%s must be set when %s is true", HTTP_SERVER_AUTHENTICATION_KRB5_KEYTAB, INTERNAL_COMMUNICATION_KERBEROS_ENABLED); + + configBinder(binder).bindConfigGlobalDefaults(KerberosConfig.class, kerberosConfig -> { + kerberosConfig.setConfig(serverKerberosConfig.getKerberosConfig()); + kerberosConfig.setKeytab(serverKerberosConfig.getKeytab()); + kerberosConfig.setUseCanonicalHostname(clientKerberosConfig.isKerberosUseCanonicalHostname()); + }); + + String kerberosPrincipal = serverKerberosConfig.getServiceName() + "/" + getLocalCanonicalHostName(); + configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, httpClientConfig -> { + httpClientConfig.setAuthenticationEnabled(true); + httpClientConfig.setKerberosPrincipal(kerberosPrincipal); + httpClientConfig.setKerberosRemoteServiceName(serverKerberosConfig.getServiceName()); + }); + }; + } + + private static String getLocalCanonicalHostName() + { + try { + return InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(Locale.US); + } + catch (UnknownHostException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/InternalAuthenticationManager.java b/presto-internal-communication/src/main/java/com/facebook/presto/server/InternalAuthenticationManager.java similarity index 100% rename from presto-main/src/main/java/com/facebook/presto/server/InternalAuthenticationManager.java rename to presto-internal-communication/src/main/java/com/facebook/presto/server/InternalAuthenticationManager.java diff --git a/presto-main-base/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java b/presto-internal-communication/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java similarity index 100% rename from presto-main-base/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java rename to presto-internal-communication/src/main/java/com/facebook/presto/server/InternalCommunicationConfig.java diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/InternalAuthenticationFilter.java b/presto-internal-communication/src/main/java/com/facebook/presto/server/security/InternalAuthenticationFilter.java similarity index 100% rename from presto-main/src/main/java/com/facebook/presto/server/security/InternalAuthenticationFilter.java rename to presto-internal-communication/src/main/java/com/facebook/presto/server/security/InternalAuthenticationFilter.java diff --git a/presto-main-base/src/main/java/com/facebook/presto/server/security/RoleType.java b/presto-internal-communication/src/main/java/com/facebook/presto/server/security/RoleType.java similarity index 100% rename from presto-main-base/src/main/java/com/facebook/presto/server/security/RoleType.java rename to presto-internal-communication/src/main/java/com/facebook/presto/server/security/RoleType.java diff --git a/presto-main-base/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java b/presto-internal-communication/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java similarity index 100% rename from presto-main-base/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java rename to presto-internal-communication/src/test/java/com/facebook/presto/server/TestInternalCommunicationConfig.java diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 690eabca968ee..bbb879642d86c 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -308,6 +308,11 @@ com.facebook.presto:presto-ui + + io.netty:* true diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 71cbd0302217a..1ce4d068cb292 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -53,12 +53,6 @@ jakarta.validation-api - - com.google.errorprone - error_prone_annotations - true - - com.fasterxml.jackson.core jackson-databind diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java index 813944f3a2463..8f6b75fd2dde6 100644 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java +++ b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/JmxConnectorFactory.java @@ -14,7 +14,7 @@ package com.facebook.presto.connector.jmx; import com.facebook.airlift.bootstrap.Bootstrap; -import com.facebook.presto.connector.jmx.util.RebindSafeMBeanServer; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.connector.Connector; diff --git a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java b/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java deleted file mode 100644 index 73dc0340a4343..0000000000000 --- a/presto-jmx/src/main/java/com/facebook/presto/connector/jmx/util/RebindSafeMBeanServer.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.connector.jmx.util; - -import com.facebook.airlift.log.Logger; -import com.google.errorprone.annotations.ThreadSafe; - -import javax.management.Attribute; -import javax.management.AttributeList; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceAlreadyExistsException; -import javax.management.InstanceNotFoundException; -import javax.management.IntrospectionException; -import javax.management.InvalidAttributeValueException; -import javax.management.ListenerNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanInfo; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.NotCompliantMBeanException; -import javax.management.NotificationFilter; -import javax.management.NotificationListener; -import javax.management.ObjectInstance; -import javax.management.ObjectName; -import javax.management.OperationsException; -import javax.management.QueryExp; -import javax.management.ReflectionException; -import javax.management.loading.ClassLoaderRepository; - -import java.io.ObjectInputStream; -import java.util.Set; - -// TODO: move this to airlift or jmxutils - -/** - * MBeanServer wrapper that a ignores calls to registerMBean when there is already - * a MBean registered with the specified object name. - */ -@ThreadSafe -public class RebindSafeMBeanServer - implements MBeanServer -{ - private static final Logger log = Logger.get(RebindSafeMBeanServer.class); - - private final MBeanServer mbeanServer; - - public RebindSafeMBeanServer(MBeanServer mbeanServer) - { - this.mbeanServer = mbeanServer; - } - - /** - * Delegates to the wrapped mbean server, but if a mbean is already registered - * with the specified name, the existing instance is returned. - */ - @Override - public ObjectInstance registerMBean(Object object, ObjectName name) - throws MBeanRegistrationException, NotCompliantMBeanException - { - while (true) { - try { - // try to register the mbean - return mbeanServer.registerMBean(object, name); - } - catch (InstanceAlreadyExistsException ignored) { - } - - try { - // a mbean is already installed, try to return the already registered instance - ObjectInstance objectInstance = mbeanServer.getObjectInstance(name); - log.debug("%s already bound to %s", name, objectInstance); - return objectInstance; - } - catch (InstanceNotFoundException ignored) { - // the mbean was removed before we could get the reference - // start the whole process over again - } - } - } - - @Override - public void unregisterMBean(ObjectName name) - throws InstanceNotFoundException, MBeanRegistrationException - { - mbeanServer.unregisterMBean(name); - } - - @Override - public ObjectInstance getObjectInstance(ObjectName name) - throws InstanceNotFoundException - { - return mbeanServer.getObjectInstance(name); - } - - @Override - public Set queryMBeans(ObjectName name, QueryExp query) - { - return mbeanServer.queryMBeans(name, query); - } - - @Override - public Set queryNames(ObjectName name, QueryExp query) - { - return mbeanServer.queryNames(name, query); - } - - @Override - public boolean isRegistered(ObjectName name) - { - return mbeanServer.isRegistered(name); - } - - @Override - public Integer getMBeanCount() - { - return mbeanServer.getMBeanCount(); - } - - @Override - public Object getAttribute(ObjectName name, String attribute) - throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttribute(name, attribute); - } - - @Override - public AttributeList getAttributes(ObjectName name, String[] attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttributes(name, attributes); - } - - @Override - public void setAttribute(ObjectName name, Attribute attribute) - throws InstanceNotFoundException, AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException - { - mbeanServer.setAttribute(name, attribute); - } - - @Override - public AttributeList setAttributes(ObjectName name, AttributeList attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.setAttributes(name, attributes); - } - - @Override - public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) - throws InstanceNotFoundException, MBeanException, ReflectionException - { - return mbeanServer.invoke(name, operationName, params, signature); - } - - @Override - public String getDefaultDomain() - { - return mbeanServer.getDefaultDomain(); - } - - @Override - public String[] getDomains() - { - return mbeanServer.getDomains(); - } - - @Override - public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public MBeanInfo getMBeanInfo(ObjectName name) - throws InstanceNotFoundException, IntrospectionException, ReflectionException - { - return mbeanServer.getMBeanInfo(name); - } - - @Override - public boolean isInstanceOf(ObjectName name, String className) - throws InstanceNotFoundException - { - return mbeanServer.isInstanceOf(name, className); - } - - @Override - public Object instantiate(String className) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className); - } - - @Override - public Object instantiate(String className, ObjectName loaderName) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName); - } - - @Override - public Object instantiate(String className, Object[] params, String[] signature) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className, params, signature); - } - - @Override - public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName, params, signature); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(ObjectName name, byte[] data) - throws OperationsException - { - return mbeanServer.deserialize(name, data); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(String className, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, data); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, loaderName, data); - } - - @Override - public ClassLoader getClassLoaderFor(ObjectName mbeanName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoaderFor(mbeanName); - } - - @Override - public ClassLoader getClassLoader(ObjectName loaderName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoader(loaderName); - } - - @Override - public ClassLoaderRepository getClassLoaderRepository() - { - return mbeanServer.getClassLoaderRepository(); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name, params, signature); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName, params, signature); - } -} diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index ef6527352e2df..ce209f0d8b1fc 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -72,6 +72,19 @@ org.apache.kafka kafka-clients + + + org.lz4 + lz4-java + + + + + + + at.yawk.lz4 + lz4-java + runtime diff --git a/presto-lance/pom.xml b/presto-lance/pom.xml new file mode 100644 index 0000000000000..33918c2d04f38 --- /dev/null +++ b/presto-lance/pom.xml @@ -0,0 +1,216 @@ + + + 4.0.0 + + com.facebook.presto + presto-root + 0.297-SNAPSHOT + + + presto-lance + Presto - LanceDB Connector + presto-plugin + + + ${project.parent.basedir} + 18.3.0 + + + + + com.facebook.airlift + bootstrap + + + + com.facebook.airlift + configuration + + + + com.facebook.airlift + json + + + + com.facebook.airlift + log + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + + com.facebook.presto + presto-spi + provided + + + + com.facebook.presto + presto-common + provided + + + + io.airlift + slice + provided + + + + com.facebook.airlift + units + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + + + javax.inject + javax.inject + + + + javax.validation + validation-api + 2.0.1.Final + + + + com.facebook.presto + presto-common-arrow + + + + org.lance + lance-core + 2.0.1 + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.lance + lance-namespace-apache-client + + + + + + org.apache.arrow + arrow-memory-core + ${arrow.version} + + + org.slf4j + slf4j-api + + + + + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + + + + + com.facebook.presto + presto-testng-services + test + + + org.testng + testng + test + + + com.facebook.presto + presto-tests + test + + + com.facebook.presto + presto-main + test + + + com.facebook.presto + presto-main-base + test + + + + + + + org.basepom.maven + duplicate-finder-maven-plugin + + + LICENSE-EDL-1.0.txt + LICENSE-EPL-1.0.txt + arrow-git.properties + about.html + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.arrow:arrow-memory-unsafe + com.fasterxml.jackson.core:jackson-core + + + + + + + diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceArrowToPageScanner.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceArrowToPageScanner.java new file mode 100644 index 0000000000000..22d25a3adedc6 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceArrowToPageScanner.java @@ -0,0 +1,120 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.facebook.presto.common.Page; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.spi.PrestoException; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.lance.ipc.LanceScanner; + +import java.io.IOException; +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class LanceArrowToPageScanner + implements AutoCloseable +{ + private final ScannerFactory scannerFactory; + private final ArrowReader arrowReader; + private final List columns; + private final ArrowBlockBuilder arrowBlockBuilder; + private long lastBatchBytes; + + public LanceArrowToPageScanner( + BufferAllocator allocator, + List columns, + ScannerFactory scannerFactory, + ArrowBlockBuilder arrowBlockBuilder) + { + this.columns = requireNonNull(columns, "columns is null"); + this.scannerFactory = requireNonNull(scannerFactory, "scannerFactory is null"); + this.arrowBlockBuilder = requireNonNull(arrowBlockBuilder, "arrowBlockBuilder is null"); + List columnNames = columns.stream() + .map(LanceColumnHandle::getColumnName) + .collect(toImmutableList()); + LanceScanner scanner = scannerFactory.open(allocator, columnNames); + this.arrowReader = scanner.scanBatches(); + } + + public boolean read() + { + try { + boolean hasNext = arrowReader.loadNextBatch(); + if (hasNext) { + VectorSchemaRoot root = arrowReader.getVectorSchemaRoot(); + lastBatchBytes = 0; + for (FieldVector vector : root.getFieldVectors()) { + for (ArrowBuf buf : vector.getFieldBuffers()) { + if (buf != null) { + lastBatchBytes += buf.capacity(); + } + } + } + } + return hasNext; + } + catch (IOException e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, "Failed to read Arrow batch", e); + } + } + + public long getLastBatchBytes() + { + return lastBatchBytes; + } + + public Page convert() + { + VectorSchemaRoot root; + try { + root = arrowReader.getVectorSchemaRoot(); + } + catch (IOException e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, "Failed to get VectorSchemaRoot", e); + } + + int rowCount = root.getRowCount(); + Block[] blocks = new Block[columns.size()]; + + for (int col = 0; col < columns.size(); col++) { + LanceColumnHandle column = columns.get(col); + FieldVector vector = root.getVector(column.getColumnName()); + Type type = column.getColumnType(); + blocks[col] = arrowBlockBuilder.buildBlockFromFieldVector(vector, type, null); + } + + return new Page(rowCount, blocks); + } + + @Override + public void close() + { + try { + arrowReader.close(); + } + catch (IOException e) { + // ignore + } + scannerFactory.close(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceBasePageSource.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceBasePageSource.java new file mode 100644 index 0000000000000..35f729e26c69c --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceBasePageSource.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.facebook.presto.common.Page; +import com.facebook.presto.spi.ConnectorPageSource; +import org.apache.arrow.memory.BufferAllocator; + +import java.io.IOException; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public abstract class LanceBasePageSource + implements ConnectorPageSource +{ + protected final LanceTableHandle tableHandle; + protected final LanceArrowToPageScanner arrowToPageScanner; + protected final BufferAllocator bufferAllocator; + protected long readBytes; + protected boolean finished; + + public LanceBasePageSource( + LanceTableHandle tableHandle, + List columns, + ScannerFactory scannerFactory, + ArrowBlockBuilder arrowBlockBuilder, + BufferAllocator parentAllocator) + { + this.tableHandle = requireNonNull(tableHandle, "tableHandle is null"); + requireNonNull(columns, "columns is null"); + requireNonNull(scannerFactory, "scannerFactory is null"); + requireNonNull(arrowBlockBuilder, "arrowBlockBuilder is null"); + requireNonNull(parentAllocator, "parentAllocator is null"); + + this.bufferAllocator = parentAllocator + .newChildAllocator(tableHandle.getTableName(), 0, Long.MAX_VALUE); + + try { + this.arrowToPageScanner = new LanceArrowToPageScanner( + bufferAllocator, + columns, + scannerFactory, + arrowBlockBuilder); + } + catch (RuntimeException e) { + bufferAllocator.close(); + throw e; + } + + this.finished = false; + } + + @Override + public long getCompletedBytes() + { + return readBytes; + } + + @Override + public long getCompletedPositions() + { + return 0; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public boolean isFinished() + { + return finished; + } + + @Override + public Page getNextPage() + { + if (finished) { + return null; + } + if (!arrowToPageScanner.read()) { + finished = true; + return null; + } + readBytes += arrowToPageScanner.getLastBatchBytes(); + return arrowToPageScanner.convert(); + } + + @Override + public long getSystemMemoryUsage() + { + return 0; + } + + @Override + public void close() + throws IOException + { + try { + arrowToPageScanner.close(); + } + finally { + bufferAllocator.close(); + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceColumnHandle.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceColumnHandle.java new file mode 100644 index 0000000000000..9ae1281e52048 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceColumnHandle.java @@ -0,0 +1,218 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.BooleanType; +import com.facebook.presto.common.type.DateType; +import com.facebook.presto.common.type.DoubleType; +import com.facebook.presto.common.type.IntegerType; +import com.facebook.presto.common.type.RealType; +import com.facebook.presto.common.type.RowType; +import com.facebook.presto.common.type.SmallintType; +import com.facebook.presto.common.type.TimestampType; +import com.facebook.presto.common.type.TinyintType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarbinaryType; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class LanceColumnHandle + implements ColumnHandle +{ + private final String columnName; + private final Type columnType; + private final boolean nullable; + + @JsonCreator + public LanceColumnHandle( + @JsonProperty("columnName") String columnName, + @JsonProperty("columnType") Type columnType, + @JsonProperty("nullable") boolean nullable) + { + this.columnName = requireNonNull(columnName, "columnName is null"); + this.columnType = requireNonNull(columnType, "columnType is null"); + this.nullable = nullable; + } + + public LanceColumnHandle(String columnName, Type columnType) + { + this(columnName, columnType, true); + } + + @JsonProperty + public String getColumnName() + { + return columnName; + } + + @JsonProperty + public Type getColumnType() + { + return columnType; + } + + @JsonProperty + public boolean isNullable() + { + return nullable; + } + + public ColumnMetadata getColumnMetadata() + { + return ColumnMetadata.builder() + .setName(columnName) + .setType(columnType) + .setNullable(nullable) + .build(); + } + + public static Type toPrestoType(Field field) + { + ArrowType type = field.getType(); + + if (type instanceof ArrowType.FixedSizeList || type instanceof ArrowType.List) { + Type elementType = RealType.REAL; + if (field.getChildren() != null && !field.getChildren().isEmpty()) { + elementType = toPrestoType(field.getChildren().get(0)); + } + return new ArrayType(elementType); + } + + if (type instanceof ArrowType.Bool) { + return BooleanType.BOOLEAN; + } + else if (type instanceof ArrowType.Int) { + ArrowType.Int intType = (ArrowType.Int) type; + switch (intType.getBitWidth()) { + case 8: + return TinyintType.TINYINT; + case 16: + return SmallintType.SMALLINT; + case 32: + return IntegerType.INTEGER; + case 64: + return BigintType.BIGINT; + } + } + else if (type instanceof ArrowType.FloatingPoint) { + ArrowType.FloatingPoint fpType = (ArrowType.FloatingPoint) type; + if (fpType.getPrecision() == FloatingPointPrecision.SINGLE) { + return RealType.REAL; + } + return DoubleType.DOUBLE; + } + else if (type instanceof ArrowType.Utf8 || type instanceof ArrowType.LargeUtf8) { + return VarcharType.VARCHAR; + } + else if (type instanceof ArrowType.Binary || type instanceof ArrowType.LargeBinary) { + return VarbinaryType.VARBINARY; + } + else if (type instanceof ArrowType.Date) { + return DateType.DATE; + } + else if (type instanceof ArrowType.Timestamp) { + return TimestampType.TIMESTAMP; + } + throw new UnsupportedOperationException("Unsupported Arrow type: " + type); + } + + public static ArrowType toArrowType(Type prestoType) + { + if (prestoType.equals(BooleanType.BOOLEAN)) { + return ArrowType.Bool.INSTANCE; + } + else if (prestoType.equals(TinyintType.TINYINT)) { + return new ArrowType.Int(8, true); + } + else if (prestoType.equals(SmallintType.SMALLINT)) { + return new ArrowType.Int(16, true); + } + else if (prestoType.equals(IntegerType.INTEGER)) { + return new ArrowType.Int(32, true); + } + else if (prestoType.equals(BigintType.BIGINT)) { + return new ArrowType.Int(64, true); + } + else if (prestoType.equals(RealType.REAL)) { + return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); + } + else if (prestoType.equals(DoubleType.DOUBLE)) { + return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + } + else if (prestoType instanceof VarcharType) { + return ArrowType.Utf8.INSTANCE; + } + else if (prestoType instanceof VarbinaryType) { + return ArrowType.Binary.INSTANCE; + } + else if (prestoType instanceof DateType) { + return new ArrowType.Date(DateUnit.DAY); + } + else if (prestoType instanceof TimestampType) { + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); + } + else if (prestoType instanceof ArrayType) { + return ArrowType.List.INSTANCE; + } + else if (prestoType instanceof RowType) { + return ArrowType.Struct.INSTANCE; + } + throw new UnsupportedOperationException("Unsupported Presto type: " + prestoType); + } + + @Override + public int hashCode() + { + return Objects.hash(columnName, columnType); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + LanceColumnHandle other = (LanceColumnHandle) obj; + return Objects.equals(this.columnName, other.columnName) && + Objects.equals(this.columnType, other.columnType); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("columnName", columnName) + .add("columnType", columnType) + .add("nullable", nullable) + .toString(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceCommitTaskData.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceCommitTaskData.java new file mode 100644 index 0000000000000..2c44e1b5534a9 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceCommitTaskData.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import static java.util.Objects.requireNonNull; + +public class LanceCommitTaskData +{ + private final String fragmentsJson; + private final long writtenBytes; + private final long rowCount; + + @JsonCreator + public LanceCommitTaskData( + @JsonProperty("fragmentsJson") String fragmentsJson, + @JsonProperty("writtenBytes") long writtenBytes, + @JsonProperty("rowCount") long rowCount) + { + this.fragmentsJson = requireNonNull(fragmentsJson, "fragmentsJson is null"); + this.writtenBytes = writtenBytes; + this.rowCount = rowCount; + } + + @JsonProperty + public String getFragmentsJson() + { + return fragmentsJson; + } + + @JsonProperty + public long getWrittenBytes() + { + return writtenBytes; + } + + @JsonProperty + public long getRowCount() + { + return rowCount; + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceConfig.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConfig.java new file mode 100644 index 0000000000000..d126e061821ac --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConfig.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.configuration.ConfigDescription; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +public class LanceConfig +{ + private String impl = "dir"; + private String rootUrl = ""; + private boolean singleLevelNs = true; + private int readBatchSize = 8192; + private int maxRowsPerFile = 1_000_000; + private int maxRowsPerGroup = 100_000; + private int writeBatchSize = 10_000; + private long indexCacheSizeBytes = 134_217_728; // 128MB + private long metadataCacheSizeBytes = 134_217_728; // 128MB + + @NotNull + public String getImpl() + { + return impl; + } + + @Config("lance.impl") + @ConfigDescription("Namespace implementation: 'dir' or full class name") + public LanceConfig setImpl(String impl) + { + this.impl = impl; + return this; + } + + @NotNull + public String getRootUrl() + { + return rootUrl; + } + + @Config("lance.root-url") + @ConfigDescription("Lance root storage path") + public LanceConfig setRootUrl(String rootUrl) + { + this.rootUrl = rootUrl; + return this; + } + + public boolean isSingleLevelNs() + { + return singleLevelNs; + } + + @Config("lance.single-level-ns") + @ConfigDescription("Access 1st level namespace with virtual 'default' schema") + public LanceConfig setSingleLevelNs(boolean singleLevelNs) + { + this.singleLevelNs = singleLevelNs; + return this; + } + + @Min(1) + public int getReadBatchSize() + { + return readBatchSize; + } + + @Config("lance.read-batch-size") + @ConfigDescription("Number of rows per batch during reads") + public LanceConfig setReadBatchSize(int readBatchSize) + { + this.readBatchSize = readBatchSize; + return this; + } + + @Min(1) + public int getMaxRowsPerFile() + { + return maxRowsPerFile; + } + + @Config("lance.max-rows-per-file") + @ConfigDescription("Maximum number of rows per Lance file") + public LanceConfig setMaxRowsPerFile(int maxRowsPerFile) + { + this.maxRowsPerFile = maxRowsPerFile; + return this; + } + + @Min(1) + public int getMaxRowsPerGroup() + { + return maxRowsPerGroup; + } + + @Config("lance.max-rows-per-group") + @ConfigDescription("Maximum number of rows per row group") + public LanceConfig setMaxRowsPerGroup(int maxRowsPerGroup) + { + this.maxRowsPerGroup = maxRowsPerGroup; + return this; + } + + @Min(1) + public int getWriteBatchSize() + { + return writeBatchSize; + } + + @Config("lance.write-batch-size") + @ConfigDescription("Number of rows to batch before writing to Arrow") + public LanceConfig setWriteBatchSize(int writeBatchSize) + { + this.writeBatchSize = writeBatchSize; + return this; + } + + @Min(0) + public long getIndexCacheSizeBytes() + { + return indexCacheSizeBytes; + } + + @Config("lance.index-cache-size-bytes") + @ConfigDescription("Size in bytes for Lance index cache per worker") + public LanceConfig setIndexCacheSizeBytes(long indexCacheSizeBytes) + { + this.indexCacheSizeBytes = indexCacheSizeBytes; + return this; + } + + @Min(0) + public long getMetadataCacheSizeBytes() + { + return metadataCacheSizeBytes; + } + + @Config("lance.metadata-cache-size-bytes") + @ConfigDescription("Size in bytes for Lance metadata cache per worker") + public LanceConfig setMetadataCacheSizeBytes(long metadataCacheSizeBytes) + { + this.metadataCacheSizeBytes = metadataCacheSizeBytes; + return this; + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnector.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnector.java new file mode 100644 index 0000000000000..241819422fc13 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnector.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.bootstrap.LifeCycleManager; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.transaction.IsolationLevel; + +import javax.inject.Inject; + +import static java.util.Objects.requireNonNull; + +public class LanceConnector + implements Connector +{ + private static final Logger log = Logger.get(LanceConnector.class); + + private final LifeCycleManager lifeCycleManager; + private final LanceMetadata metadata; + private final LanceNamespaceHolder namespaceHolder; + private final ConnectorSplitManager splitManager; + private final ConnectorPageSourceProvider pageSourceProvider; + private final ConnectorPageSinkProvider pageSinkProvider; + + @Inject + public LanceConnector( + LifeCycleManager lifeCycleManager, + LanceMetadata metadata, + LanceNamespaceHolder namespaceHolder, + ConnectorSplitManager splitManager, + ConnectorPageSourceProvider pageSourceProvider, + ConnectorPageSinkProvider pageSinkProvider) + { + this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.namespaceHolder = requireNonNull(namespaceHolder, "namespaceHolder is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + return LanceTransactionHandle.INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) + { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return pageSourceProvider; + } + + @Override + public ConnectorPageSinkProvider getPageSinkProvider() + { + return pageSinkProvider; + } + + @Override + public void shutdown() + { + try { + lifeCycleManager.stop(); + } + catch (Exception e) { + log.error(e, "Error shutting down connector"); + } + finally { + namespaceHolder.shutdown(); + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnectorFactory.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnectorFactory.java new file mode 100644 index 0000000000000..baaaacdda6836 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceConnectorFactory.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.classloader.ThreadContextClassLoader; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.google.inject.Injector; + +import java.util.Map; + +public class LanceConnectorFactory + implements ConnectorFactory +{ + @Override + public String getName() + { + return "lance"; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new LanceHandleResolver(); + } + + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) + { + ClassLoader classLoader = LanceConnectorFactory.class.getClassLoader(); + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + Bootstrap app = new Bootstrap( + new JsonModule(), + new LanceModule(), + binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager())); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(LanceConnector.class); + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceErrorCode.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceErrorCode.java new file mode 100644 index 0000000000000..1dcac762bdc46 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceErrorCode.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.ErrorType; +import com.facebook.presto.spi.ErrorCodeSupplier; + +import static com.facebook.presto.common.ErrorType.EXTERNAL; + +public enum LanceErrorCode + implements ErrorCodeSupplier +{ + LANCE_ERROR(0, EXTERNAL), + LANCE_TABLE_NOT_FOUND(1, EXTERNAL), + LANCE_FILESYSTEM_ERROR(2, EXTERNAL), + LANCE_TYPE_NOT_SUPPORTED(3, EXTERNAL); + + private final ErrorCode errorCode; + + LanceErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0520_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentData.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentData.java new file mode 100644 index 0000000000000..0f9854976e9f9 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentData.java @@ -0,0 +1,316 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.lance.FragmentMetadata; +import org.lance.fragment.DataFile; +import org.lance.fragment.DeletionFile; +import org.lance.fragment.DeletionFileType; +import org.lance.fragment.RowIdMeta; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +/** + * JSON-serializable representation of Lance FragmentMetadata. + * Replaces Java ObjectOutputStream serialization for cross-node commit data. + */ +public class LanceFragmentData +{ + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final int id; + private final List files; + private final long physicalRows; + private final LanceDeletionFile deletionFile; + private final String rowIdMetadata; + + @JsonCreator + public LanceFragmentData( + @JsonProperty("id") int id, + @JsonProperty("files") List files, + @JsonProperty("physicalRows") long physicalRows, + @JsonProperty("deletionFile") LanceDeletionFile deletionFile, + @JsonProperty("rowIdMetadata") String rowIdMetadata) + { + this.id = id; + this.files = requireNonNull(files, "files is null"); + this.physicalRows = physicalRows; + this.deletionFile = deletionFile; + this.rowIdMetadata = rowIdMetadata; + } + + public static LanceFragmentData fromFragmentMetadata(FragmentMetadata fragment) + { + List files = fragment.getFiles().stream() + .map(LanceDataFile::fromDataFile) + .collect(toImmutableList()); + + LanceDeletionFile deletionFile = null; + if (fragment.getDeletionFile() != null) { + deletionFile = LanceDeletionFile.fromDeletionFile(fragment.getDeletionFile()); + } + + String rowIdMetadata = null; + if (fragment.getRowIdMeta() != null) { + rowIdMetadata = fragment.getRowIdMeta().getMetadata(); + } + + return new LanceFragmentData( + fragment.getId(), + files, + fragment.getPhysicalRows(), + deletionFile, + rowIdMetadata); + } + + public FragmentMetadata toFragmentMetadata() + { + List dataFiles = files.stream() + .map(LanceDataFile::toDataFile) + .collect(toImmutableList()); + + DeletionFile delFile = deletionFile != null ? deletionFile.toDeletionFile() : null; + RowIdMeta rowIdMeta = rowIdMetadata != null ? new RowIdMeta(rowIdMetadata) : null; + + return new FragmentMetadata(id, dataFiles, physicalRows, delFile, rowIdMeta); + } + + @JsonProperty + public int getId() + { + return id; + } + + @JsonProperty + public List getFiles() + { + return files; + } + + @JsonProperty + public long getPhysicalRows() + { + return physicalRows; + } + + @JsonProperty + public LanceDeletionFile getDeletionFile() + { + return deletionFile; + } + + @JsonProperty + public String getRowIdMetadata() + { + return rowIdMetadata; + } + + public static String serializeFragments(List fragments) + { + try { + List data = fragments.stream() + .map(LanceFragmentData::fromFragmentMetadata) + .collect(toImmutableList()); + return MAPPER.writeValueAsString(data); + } + catch (JsonProcessingException e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, "Failed to serialize fragment metadata", e); + } + } + + public static List deserializeFragments(String json) + { + try { + List data = MAPPER.readValue(json, new TypeReference>() {}); + return data.stream() + .map(LanceFragmentData::toFragmentMetadata) + .collect(toImmutableList()); + } + catch (JsonProcessingException e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, "Failed to deserialize fragment metadata", e); + } + } + + public static class LanceDataFile + { + private final String path; + private final int[] fields; + private final int[] columnIndices; + private final int fileMajorVersion; + private final int fileMinorVersion; + private final Long fileSizeBytes; + private final Integer baseId; + + @JsonCreator + public LanceDataFile( + @JsonProperty("path") String path, + @JsonProperty("fields") int[] fields, + @JsonProperty("columnIndices") int[] columnIndices, + @JsonProperty("fileMajorVersion") int fileMajorVersion, + @JsonProperty("fileMinorVersion") int fileMinorVersion, + @JsonProperty("fileSizeBytes") Long fileSizeBytes, + @JsonProperty("baseId") Integer baseId) + { + this.path = requireNonNull(path, "path is null"); + this.fields = requireNonNull(fields, "fields is null"); + this.columnIndices = requireNonNull(columnIndices, "columnIndices is null"); + this.fileMajorVersion = fileMajorVersion; + this.fileMinorVersion = fileMinorVersion; + this.fileSizeBytes = fileSizeBytes; + this.baseId = baseId; + } + + public static LanceDataFile fromDataFile(DataFile dataFile) + { + return new LanceDataFile( + dataFile.getPath(), + dataFile.getFields(), + dataFile.getColumnIndices(), + dataFile.getFileMajorVersion(), + dataFile.getFileMinorVersion(), + dataFile.getFileSizeBytes(), + dataFile.getBaseId().orElse(null)); + } + + public DataFile toDataFile() + { + return new DataFile(path, fields, columnIndices, fileMajorVersion, fileMinorVersion, fileSizeBytes, baseId); + } + + @JsonProperty + public String getPath() + { + return path; + } + + @JsonProperty + public int[] getFields() + { + return fields; + } + + @JsonProperty + public int[] getColumnIndices() + { + return columnIndices; + } + + @JsonProperty + public int getFileMajorVersion() + { + return fileMajorVersion; + } + + @JsonProperty + public int getFileMinorVersion() + { + return fileMinorVersion; + } + + @JsonProperty + public Long getFileSizeBytes() + { + return fileSizeBytes; + } + + @JsonProperty + public Integer getBaseId() + { + return baseId; + } + } + + public static class LanceDeletionFile + { + private final long id; + private final long readVersion; + private final Long numDeletedRows; + private final String fileType; + private final Integer baseId; + + @JsonCreator + public LanceDeletionFile( + @JsonProperty("id") long id, + @JsonProperty("readVersion") long readVersion, + @JsonProperty("numDeletedRows") Long numDeletedRows, + @JsonProperty("fileType") String fileType, + @JsonProperty("baseId") Integer baseId) + { + this.id = id; + this.readVersion = readVersion; + this.numDeletedRows = numDeletedRows; + this.fileType = fileType; + this.baseId = baseId; + } + + public static LanceDeletionFile fromDeletionFile(DeletionFile deletionFile) + { + return new LanceDeletionFile( + deletionFile.getId(), + deletionFile.getReadVersion(), + deletionFile.getNumDeletedRows(), + deletionFile.getFileType() != null ? deletionFile.getFileType().name() : null, + deletionFile.getBaseId().orElse(null)); + } + + public DeletionFile toDeletionFile() + { + return new DeletionFile( + id, + readVersion, + numDeletedRows, + fileType != null ? DeletionFileType.valueOf(fileType) : null, + baseId); + } + + @JsonProperty + public long getId() + { + return id; + } + + @JsonProperty + public long getReadVersion() + { + return readVersion; + } + + @JsonProperty + public Long getNumDeletedRows() + { + return numDeletedRows; + } + + @JsonProperty + public String getFileType() + { + return fileType; + } + + @JsonProperty + public Integer getBaseId() + { + return baseId; + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentPageSource.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentPageSource.java new file mode 100644 index 0000000000000..5207f087ed879 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceFragmentPageSource.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.log.Logger; +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.google.common.collect.ImmutableList; +import org.apache.arrow.memory.BufferAllocator; +import org.lance.Dataset; +import org.lance.ReadOptions; +import org.lance.ipc.LanceScanner; +import org.lance.ipc.ScanOptions; + +import java.util.List; + +public class LanceFragmentPageSource + extends LanceBasePageSource +{ + private static final Logger log = Logger.get(LanceFragmentPageSource.class); + + public LanceFragmentPageSource( + LanceTableHandle tableHandle, + List columns, + List fragments, + String tablePath, + int readBatchSize, + ReadOptions readOptions, + ArrowBlockBuilder arrowBlockBuilder, + BufferAllocator parentAllocator) + { + super(tableHandle, columns, new FragmentScannerFactory(fragments, tablePath, readBatchSize, readOptions), arrowBlockBuilder, parentAllocator); + } + + private static class FragmentScannerFactory + implements ScannerFactory + { + private final List fragmentIds; + private final String tablePath; + private final int readBatchSize; + private final ReadOptions readOptions; + private Dataset dataset; + private LanceScanner scanner; + + FragmentScannerFactory(List fragmentIds, String tablePath, int readBatchSize, ReadOptions readOptions) + { + this.fragmentIds = ImmutableList.copyOf(fragmentIds); + this.tablePath = tablePath; + this.readBatchSize = readBatchSize; + this.readOptions = readOptions; + } + + @Override + public LanceScanner open(BufferAllocator allocator, List columns) + { + ScanOptions.Builder optionsBuilder = new ScanOptions.Builder(); + if (!columns.isEmpty()) { + optionsBuilder.columns(columns); + } + optionsBuilder.batchSize(readBatchSize); + optionsBuilder.fragmentIds(fragmentIds); + + this.dataset = Dataset.open(tablePath, readOptions); + this.scanner = dataset.newScan(optionsBuilder.build()); + return scanner; + } + + @Override + public void close() + { + try { + if (scanner != null) { + scanner.close(); + } + } + catch (Exception e) { + log.warn(e, "Error closing lance scanner"); + } + try { + if (dataset != null) { + dataset.close(); + } + } + catch (Exception e) { + log.warn(e, "Error closing lance dataset"); + } + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceHandleResolver.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceHandleResolver.java new file mode 100644 index 0000000000000..ea303aed93100 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceHandleResolver.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorOutputTableHandle; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +public class LanceHandleResolver + implements ConnectorHandleResolver +{ + @Override + public Class getTableHandleClass() + { + return LanceTableHandle.class; + } + + @Override + public Class getTableLayoutHandleClass() + { + return LanceTableLayoutHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return LanceColumnHandle.class; + } + + @Override + public Class getSplitClass() + { + return LanceSplit.class; + } + + @Override + public Class getOutputTableHandleClass() + { + return LanceWritableTableHandle.class; + } + + @Override + public Class getInsertTableHandleClass() + { + return LanceWritableTableHandle.class; + } + + @Override + public Class getTransactionHandleClass() + { + return LanceTransactionHandle.class; + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceMetadata.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceMetadata.java new file mode 100644 index 0000000000000..c18a0bc85f128 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceMetadata.java @@ -0,0 +1,280 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorNewTableLayout; +import com.facebook.presto.spi.ConnectorOutputTableHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayout; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.ConnectorTableLayoutResult; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.SchemaTablePrefix; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorOutputMetadata; +import com.facebook.presto.spi.statistics.ComputedStatistics; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slice; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; + +import javax.inject.Inject; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; + +public class LanceMetadata + implements ConnectorMetadata +{ + public static final String LANCE_DEFAULT_SCHEMA = "default"; + + private final LanceNamespaceHolder namespaceHolder; + private final JsonCodec commitTaskDataCodec; + + @Inject + public LanceMetadata( + LanceNamespaceHolder namespaceHolder, + JsonCodec commitTaskDataCodec) + { + this.namespaceHolder = requireNonNull(namespaceHolder, "namespaceHolder is null"); + this.commitTaskDataCodec = requireNonNull(commitTaskDataCodec, "commitTaskDataCodec is null"); + } + + @Override + public boolean schemaExists(ConnectorSession session, String schemaName) + { + return LANCE_DEFAULT_SCHEMA.equals(schemaName); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + return ImmutableList.of(LANCE_DEFAULT_SCHEMA); + } + + @Override + public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) + { + if (!schemaExists(session, tableName.getSchemaName())) { + return null; + } + if (!namespaceHolder.tableExists(tableName.getTableName())) { + return null; + } + return new LanceTableHandle(tableName.getSchemaName(), tableName.getTableName()); + } + + @Override + public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) + { + return new ConnectorTableLayout(handle); + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle table) + { + LanceTableHandle lanceTable = (LanceTableHandle) table; + if (!namespaceHolder.tableExists(lanceTable.getTableName())) { + return null; + } + Schema arrowSchema = namespaceHolder.describeTable(lanceTable.getTableName()); + SchemaTableName schemaTableName = new SchemaTableName(lanceTable.getSchemaName(), lanceTable.getTableName()); + + ImmutableList.Builder columnsMetadata = ImmutableList.builder(); + for (Field field : arrowSchema.getFields()) { + columnsMetadata.add(ColumnMetadata.builder() + .setName(field.getName()) + .setType(LanceColumnHandle.toPrestoType(field)) + .setNullable(field.isNullable()) + .build()); + } + + return new ConnectorTableMetadata(schemaTableName, columnsMetadata.build()); + } + + @Override + public List listTables(ConnectorSession session, Optional schemaName) + { + String schema = schemaName.orElse(LANCE_DEFAULT_SCHEMA); + return namespaceHolder.listTables().stream() + .map(tableName -> new SchemaTableName(schema, tableName)) + .collect(toImmutableList()); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) + { + LanceTableHandle lanceTable = (LanceTableHandle) tableHandle; + if (!namespaceHolder.tableExists(lanceTable.getTableName())) { + return ImmutableMap.of(); + } + Schema arrowSchema = namespaceHolder.describeTable(lanceTable.getTableName()); + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + for (Field field : arrowSchema.getFields()) { + LanceColumnHandle columnHandle = new LanceColumnHandle( + field.getName(), + LanceColumnHandle.toPrestoType(field), + field.isNullable()); + columnHandles.put(field.getName(), columnHandle); + } + return columnHandles.build(); + } + + @Override + public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) + { + return ((LanceColumnHandle) columnHandle).getColumnMetadata(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + List tables = prefix.getTableName() != null + ? singletonList(prefix.toSchemaTableName()) + : listTables(session, Optional.ofNullable(prefix.getSchemaName())); + + ImmutableMap.Builder> columns = ImmutableMap.builder(); + for (SchemaTableName tableName : tables) { + ConnectorTableHandle tableHandle = getTableHandle(session, tableName); + if (tableHandle != null) { + ConnectorTableMetadata tableMetadata = getTableMetadata(session, tableHandle); + if (tableMetadata != null) { + columns.put(tableName, tableMetadata.getColumns()); + } + } + } + return columns.build(); + } + + @Override + public ConnectorTableLayoutResult getTableLayoutForConstraint( + ConnectorSession session, + ConnectorTableHandle table, + Constraint constraint, + Optional> desiredColumns) + { + LanceTableHandle lanceTable = (LanceTableHandle) table; + ConnectorTableLayout layout = new ConnectorTableLayout( + new LanceTableLayoutHandle(lanceTable, constraint.getSummary())); + return new ConnectorTableLayoutResult(layout, constraint.getSummary()); + } + + @Override + public ConnectorOutputTableHandle beginCreateTable( + ConnectorSession session, + ConnectorTableMetadata tableMetadata, + Optional layout) + { + Schema arrowSchema = LancePageToArrowConverter.toArrowSchema(tableMetadata.getColumns()); + + namespaceHolder.createTable( + tableMetadata.getTable().getTableName(), + arrowSchema); + + List columns = tableMetadata.getColumns().stream() + .map(col -> new LanceColumnHandle(col.getName(), col.getType(), col.isNullable())) + .collect(toImmutableList()); + + return new LanceWritableTableHandle( + tableMetadata.getTable().getSchemaName(), + tableMetadata.getTable().getTableName(), + arrowSchema.toJson(), + columns); + } + + @Override + public Optional finishCreateTable( + ConnectorSession session, + ConnectorOutputTableHandle tableHandle, + Collection fragments, + Collection computedStatistics) + { + LanceWritableTableHandle handle = (LanceWritableTableHandle) tableHandle; + + if (!fragments.isEmpty()) { + List allFragments = collectFragments(fragments); + namespaceHolder.commitAppend(handle.getTableName(), allFragments); + } + return Optional.empty(); + } + + @Override + public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) + { + LanceTableHandle lanceTable = (LanceTableHandle) tableHandle; + Schema arrowSchema = namespaceHolder.describeTable(lanceTable.getTableName()); + + List columns = arrowSchema.getFields().stream() + .map(field -> new LanceColumnHandle( + field.getName(), + LanceColumnHandle.toPrestoType(field), + field.isNullable())) + .collect(toImmutableList()); + + return new LanceWritableTableHandle( + lanceTable.getSchemaName(), + lanceTable.getTableName(), + arrowSchema.toJson(), + columns); + } + + @Override + public Optional finishInsert( + ConnectorSession session, + ConnectorInsertTableHandle tableHandle, + Collection fragments, + Collection computedStatistics) + { + LanceWritableTableHandle handle = (LanceWritableTableHandle) tableHandle; + + if (!fragments.isEmpty()) { + List allFragments = collectFragments(fragments); + namespaceHolder.commitAppend(handle.getTableName(), allFragments); + } + return Optional.empty(); + } + + @Override + public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) + { + LanceTableHandle lanceTable = (LanceTableHandle) tableHandle; + namespaceHolder.dropTable(lanceTable.getTableName()); + } + + private List collectFragments(Collection fragments) + { + ImmutableList.Builder allFragments = ImmutableList.builder(); + for (Slice slice : fragments) { + LanceCommitTaskData commitData = commitTaskDataCodec.fromJson(slice.getBytes()); + allFragments.addAll(LanceFragmentData.deserializeFragments(commitData.getFragmentsJson())); + } + return allFragments.build(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceModule.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceModule.java new file mode 100644 index 0000000000000..2dcb16cab944c --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceModule.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; + +public class LanceModule + implements Module +{ + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(LanceConfig.class); + binder.bind(LanceNamespaceHolder.class).in(Scopes.SINGLETON); + binder.bind(LanceConnector.class).in(Scopes.SINGLETON); + binder.bind(LanceMetadata.class).in(Scopes.SINGLETON); + binder.bind(ArrowBlockBuilder.class).in(Scopes.SINGLETON); + binder.bind(ConnectorSplitManager.class).to(LanceSplitManager.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSourceProvider.class).to(LancePageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(ConnectorPageSinkProvider.class).to(LancePageSinkProvider.class).in(Scopes.SINGLETON); + jsonCodecBinder(binder).bindJsonCodec(LanceCommitTaskData.class); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceNamespaceHolder.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceNamespaceHolder.java new file mode 100644 index 0000000000000..ece5fc5b307b5 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceNamespaceHolder.java @@ -0,0 +1,212 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.log.Logger; +import com.google.common.io.MoreFiles; +import com.google.common.io.RecursiveDeleteOption; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.pojo.Schema; +import org.lance.Dataset; +import org.lance.Fragment; +import org.lance.FragmentMetadata; +import org.lance.FragmentOperation; +import org.lance.ReadOptions; +import org.lance.WriteParams; + +import javax.inject.Inject; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +/** + * Holds the Lance namespace configuration and provides table management operations. + * For the "dir" implementation, directly manages a directory-based table store. + * All tables live under a single "default" schema mapped to the root directory. + */ +public class LanceNamespaceHolder +{ + private static final Logger log = Logger.get(LanceNamespaceHolder.class); + public static final String DEFAULT_SCHEMA = "default"; + public static final String TABLE_PATH_SUFFIX = ".lance"; + + private final BufferAllocator allocator; + private final String root; + private final boolean singleLevelNs; + private final long indexCacheSizeBytes; + private final long metadataCacheSizeBytes; + + @Inject + public LanceNamespaceHolder(LanceConfig config) + { + this.root = requireNonNull(config.getRootUrl(), "root is null"); + this.singleLevelNs = config.isSingleLevelNs(); + this.indexCacheSizeBytes = config.getIndexCacheSizeBytes(); + this.metadataCacheSizeBytes = config.getMetadataCacheSizeBytes(); + this.allocator = new RootAllocator(Long.MAX_VALUE); + log.debug("LanceNamespaceHolder initialized: root=%s, singleLevelNs=%s", root, singleLevelNs); + } + + public void shutdown() + { + try { + allocator.close(); + } + catch (Exception e) { + log.warn(e, "Error closing Arrow allocator"); + } + } + + public BufferAllocator getAllocator() + { + return allocator; + } + + public String getRoot() + { + return root; + } + + public boolean isSingleLevelNs() + { + return singleLevelNs; + } + + /** + * Build ReadOptions with configured cache sizes. + */ + public ReadOptions buildReadOptions() + { + return new ReadOptions.Builder() + .setIndexCacheSizeBytes(indexCacheSizeBytes) + .setMetadataCacheSizeBytes(metadataCacheSizeBytes) + .build(); + } + + /** + * Get the filesystem path for a table. + */ + public String getTablePath(String tableName) + { + return Paths.get(root, tableName + TABLE_PATH_SUFFIX).toUri().toString(); + } + + /** + * Check if a table exists on the filesystem. + */ + public boolean tableExists(String tableName) + { + try { + Path path = Paths.get(root, tableName + TABLE_PATH_SUFFIX); + return Files.isDirectory(path); + } + catch (Exception e) { + return false; + } + } + + /** + * Get the Arrow schema for a table. + */ + public Schema describeTable(String tableName) + { + String tablePath = getTablePath(tableName); + try (Dataset dataset = Dataset.open(tablePath, buildReadOptions())) { + return dataset.getSchema(); + } + } + + /** + * List all tables in a schema. + */ + public List listTables() + { + Path rootPath = Paths.get(root); + if (!Files.isDirectory(rootPath)) { + return Collections.emptyList(); + } + List tables = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(rootPath, "*" + TABLE_PATH_SUFFIX)) { + for (Path entry : stream) { + if (Files.isDirectory(entry)) { + String fileName = entry.getFileName().toString(); + tables.add(fileName.substring(0, fileName.length() - TABLE_PATH_SUFFIX.length())); + } + } + } + catch (IOException e) { + log.warn(e, "Failed to list tables in %s", root); + } + return tables; + } + + /** + * Create an empty table with the given schema. + */ + public void createTable(String tableName, Schema arrowSchema) + { + String tablePath = getTablePath(tableName); + WriteParams params = new WriteParams.Builder().build(); + Dataset.create(allocator, tablePath, arrowSchema, params).close(); + } + + /** + * Drop a table. + */ + public void dropTable(String tableName) + { + Path tablePath = Paths.get(root, tableName + TABLE_PATH_SUFFIX); + if (Files.exists(tablePath)) { + try { + MoreFiles.deleteRecursively(tablePath, RecursiveDeleteOption.ALLOW_INSECURE); + } + catch (IOException e) { + throw new RuntimeException("Failed to delete table " + tableName, e); + } + } + } + + /** + * Commit fragments to a table (append operation). + */ + public void commitAppend(String tableName, List fragments) + { + String tablePath = getTablePath(tableName); + try (Dataset dataset = Dataset.open(tablePath, buildReadOptions())) { + FragmentOperation.Append appendOp = new FragmentOperation.Append(fragments); + Dataset.commit(allocator, tablePath, appendOp, Optional.of(dataset.version()), Collections.emptyMap()).close(); + } + } + + /** + * Get fragments for a table. + */ + public List getFragments(String tableName) + { + String tablePath = getTablePath(tableName); + try (Dataset dataset = Dataset.open(tablePath, buildReadOptions())) { + return dataset.getFragments(); + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSink.java b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSink.java new file mode 100644 index 0000000000000..3b3388d784120 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSink.java @@ -0,0 +1,170 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.Page; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.lance.Fragment; +import org.lance.FragmentMetadata; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class LancePageSink + implements ConnectorPageSink +{ + private static final Logger log = Logger.get(LancePageSink.class); + + private final String datasetUri; + private final Schema arrowSchema; + private final List columnTypes; + private final JsonCodec jsonCodec; + private final BufferAllocator allocator; + + private final List bufferedPages = new ArrayList<>(); + private long writtenBytes; + private long rowCount; + private boolean finished; + + public LancePageSink( + String datasetUri, + Schema arrowSchema, + List columns, + JsonCodec jsonCodec, + BufferAllocator parentAllocator) + { + this.datasetUri = requireNonNull(datasetUri, "datasetUri is null"); + this.arrowSchema = requireNonNull(arrowSchema, "arrowSchema is null"); + this.columnTypes = columns.stream() + .map(LanceColumnHandle::getColumnType) + .collect(toImmutableList()); + this.jsonCodec = requireNonNull(jsonCodec, "jsonCodec is null"); + this.allocator = requireNonNull(parentAllocator, "parentAllocator is null") + .newChildAllocator("page-sink", 0, Long.MAX_VALUE); + } + + @Override + public CompletableFuture appendPage(Page page) + { + bufferedPages.add(page); + rowCount += page.getPositionCount(); + writtenBytes += page.getSizeInBytes(); + return NOT_BLOCKED; + } + + @Override + public CompletableFuture> finish() + { + if (finished) { + throw new IllegalStateException("PageSink already finished"); + } + finished = true; + + try { + String fragmentsJson; + if (bufferedPages.isEmpty()) { + fragmentsJson = "[]"; + } + else { + fragmentsJson = writeFragments(); + } + + LanceCommitTaskData commitData = new LanceCommitTaskData( + fragmentsJson, writtenBytes, rowCount); + + Slice slice = Slices.wrappedBuffer(jsonCodec.toJsonBytes(commitData)); + return completedFuture(ImmutableList.of(slice)); + } + catch (PrestoException e) { + throw e; + } + catch (Exception e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, + "Failed to write Lance fragments: " + e.getMessage(), e); + } + finally { + cleanup(); + } + } + + private String writeFragments() + { + try (VectorSchemaRoot root = VectorSchemaRoot.create(arrowSchema, allocator)) { + long totalRowsLong = bufferedPages.stream() + .mapToLong(Page::getPositionCount) + .sum(); + if (totalRowsLong > Integer.MAX_VALUE) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, + "Total row count exceeds maximum: " + totalRowsLong); + } + int totalRows = (int) totalRowsLong; + + root.allocateNew(); + + int currentOffset = 0; + for (Page page : bufferedPages) { + int pageRows = page.getPositionCount(); + for (int channel = 0; channel < page.getChannelCount(); channel++) { + LancePageToArrowConverter.writeBlockToVectorAtOffset( + page.getBlock(channel), + root.getVector(channel), + columnTypes.get(channel), + pageRows, + currentOffset); + } + currentOffset += pageRows; + } + root.setRowCount(totalRows); + + List fragments = Fragment.create( + datasetUri, allocator, root, + new org.lance.WriteParams.Builder().build()); + + return LanceFragmentData.serializeFragments(fragments); + } + } + + @Override + public void abort() + { + cleanup(); + } + + private void cleanup() + { + bufferedPages.clear(); + try { + allocator.close(); + } + catch (Exception e) { + log.warn(e, "Failed to close allocator"); + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSinkProvider.java b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSinkProvider.java new file mode 100644 index 0000000000000..c836e2cfea629 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSinkProvider.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorOutputTableHandle; +import com.facebook.presto.spi.ConnectorPageSink; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.PageSinkContext; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import org.apache.arrow.vector.types.pojo.Schema; + +import javax.inject.Inject; + +import java.io.IOException; + +import static java.util.Objects.requireNonNull; + +public class LancePageSinkProvider + implements ConnectorPageSinkProvider +{ + private final LanceNamespaceHolder namespaceHolder; + private final JsonCodec jsonCodec; + + @Inject + public LancePageSinkProvider( + LanceNamespaceHolder namespaceHolder, + JsonCodec jsonCodec) + { + this.namespaceHolder = requireNonNull(namespaceHolder, "namespaceHolder is null"); + this.jsonCodec = requireNonNull(jsonCodec, "jsonCodec is null"); + } + + @Override + public ConnectorPageSink createPageSink( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorOutputTableHandle outputTableHandle, + PageSinkContext pageSinkContext) + { + LanceWritableTableHandle handle = (LanceWritableTableHandle) outputTableHandle; + return createPageSink(handle); + } + + @Override + public ConnectorPageSink createPageSink( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorInsertTableHandle insertTableHandle, + PageSinkContext pageSinkContext) + { + LanceWritableTableHandle handle = (LanceWritableTableHandle) insertTableHandle; + return createPageSink(handle); + } + + private ConnectorPageSink createPageSink(LanceWritableTableHandle handle) + { + Schema arrowSchema; + try { + arrowSchema = Schema.fromJSON(handle.getSchemaJson()); + } + catch (IOException e) { + throw new PrestoException(LanceErrorCode.LANCE_ERROR, + "Failed to parse Arrow schema", e); + } + + String tablePath = namespaceHolder.getTablePath(handle.getTableName()); + + return new LancePageSink( + tablePath, + arrowSchema, + handle.getInputColumns(), + jsonCodec, + namespaceHolder.getAllocator()); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSourceProvider.java b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSourceProvider.java new file mode 100644 index 0000000000000..2f6cc65ea3a71 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageSourceProvider.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.SplitContext; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +import javax.inject.Inject; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class LancePageSourceProvider + implements ConnectorPageSourceProvider +{ + private final LanceNamespaceHolder namespaceHolder; + private final LanceConfig config; + private final ArrowBlockBuilder arrowBlockBuilder; + + @Inject + public LancePageSourceProvider(LanceNamespaceHolder namespaceHolder, LanceConfig config, ArrowBlockBuilder arrowBlockBuilder) + { + this.namespaceHolder = requireNonNull(namespaceHolder, "namespaceHolder is null"); + this.config = requireNonNull(config, "config is null"); + this.arrowBlockBuilder = requireNonNull(arrowBlockBuilder, "arrowBlockBuilder is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableLayoutHandle layout, + List columns, + SplitContext splitContext, + RuntimeStats runtimeStats) + { + LanceSplit lanceSplit = (LanceSplit) split; + LanceTableLayoutHandle layoutHandle = (LanceTableLayoutHandle) layout; + LanceTableHandle tableHandle = layoutHandle.getTable(); + + List lanceColumns = columns.stream() + .map(LanceColumnHandle.class::cast) + .collect(toImmutableList()); + + String tablePath = namespaceHolder.getTablePath(tableHandle.getTableName()); + + return new LanceFragmentPageSource( + tableHandle, + lanceColumns, + lanceSplit.getFragments(), + tablePath, + config.getReadBatchSize(), + namespaceHolder.buildReadOptions(), + arrowBlockBuilder, + namespaceHolder.getAllocator()); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LancePageToArrowConverter.java b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageToArrowConverter.java new file mode 100644 index 0000000000000..17d2dfd05bf52 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LancePageToArrowConverter.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.BooleanType; +import com.facebook.presto.common.type.DateType; +import com.facebook.presto.common.type.DoubleType; +import com.facebook.presto.common.type.IntegerType; +import com.facebook.presto.common.type.RealType; +import com.facebook.presto.common.type.SmallintType; +import com.facebook.presto.common.type.TimestampType; +import com.facebook.presto.common.type.TinyintType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarbinaryType; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableList; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.util.List; + +import static java.lang.Float.intBitsToFloat; + +public final class LancePageToArrowConverter +{ + private LancePageToArrowConverter() {} + + public static Schema toArrowSchema(List columns) + { + ImmutableList.Builder fields = ImmutableList.builder(); + for (ColumnMetadata column : columns) { + ArrowType arrowType = LanceColumnHandle.toArrowType(column.getType()); + fields.add(new Field(column.getName(), new FieldType(column.isNullable(), arrowType, null), null)); + } + return new Schema(fields.build()); + } + + public static void writeBlockToVector(Block block, FieldVector vector, Type type, int rowCount) + { + writeBlockToVectorAtOffset(block, vector, type, rowCount, 0); + } + + public static void writeBlockToVectorAtOffset(Block block, FieldVector vector, Type type, int rowCount, int offset) + { + for (int i = 0; i < rowCount; i++) { + if (block.isNull(i)) { + // Arrow vectors handle nulls automatically with null bitmap + continue; + } + int targetIndex = offset + i; + if (type instanceof BooleanType) { + ((BitVector) vector).setSafe(targetIndex, type.getBoolean(block, i) ? 1 : 0); + } + else if (type instanceof TinyintType) { + ((TinyIntVector) vector).setSafe(targetIndex, (byte) type.getLong(block, i)); + } + else if (type instanceof SmallintType) { + ((SmallIntVector) vector).setSafe(targetIndex, (short) type.getLong(block, i)); + } + else if (type instanceof IntegerType) { + ((IntVector) vector).setSafe(targetIndex, (int) type.getLong(block, i)); + } + else if (type instanceof BigintType) { + ((BigIntVector) vector).setSafe(targetIndex, type.getLong(block, i)); + } + else if (type instanceof RealType) { + ((Float4Vector) vector).setSafe(targetIndex, intBitsToFloat((int) type.getLong(block, i))); + } + else if (type instanceof DoubleType) { + ((Float8Vector) vector).setSafe(targetIndex, type.getDouble(block, i)); + } + else if (type instanceof VarcharType) { + byte[] bytes = type.getSlice(block, i).getBytes(); + ((VarCharVector) vector).setSafe(targetIndex, bytes); + } + else if (type instanceof VarbinaryType) { + byte[] bytes = type.getSlice(block, i).getBytes(); + ((VarBinaryVector) vector).setSafe(targetIndex, bytes); + } + else if (type instanceof DateType) { + ((DateDayVector) vector).setSafe(targetIndex, (int) type.getLong(block, i)); + } + else if (type instanceof TimestampType) { + ((TimeStampMicroVector) vector).setSafe(targetIndex, type.getLong(block, i)); + } + else { + throw new PrestoException(LanceErrorCode.LANCE_TYPE_NOT_SUPPORTED, + "Unsupported type for Arrow conversion: " + type); + } + } + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LancePlugin.java b/presto-lance/src/main/java/com/facebook/presto/lance/LancePlugin.java new file mode 100644 index 0000000000000..882c3d88f271d --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LancePlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.google.common.collect.ImmutableList; + +public class LancePlugin + implements Plugin +{ + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new LanceConnectorFactory()); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplit.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplit.java new file mode 100644 index 0000000000000..fa6472d215e67 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplit.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.HostAddress; +import com.facebook.presto.spi.NodeProvider; +import com.facebook.presto.spi.schedule.NodeSelectionStrategy; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Objects; + +import static com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE; +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class LanceSplit + implements ConnectorSplit +{ + private final List fragments; + + @JsonCreator + public LanceSplit( + @JsonProperty("fragments") List fragments) + { + this.fragments = ImmutableList.copyOf(requireNonNull(fragments, "fragments is null")); + } + + @JsonProperty + public List getFragments() + { + return fragments; + } + + @Override + public NodeSelectionStrategy getNodeSelectionStrategy() + { + return NO_PREFERENCE; + } + + @Override + public List getPreferredNodes(NodeProvider nodeProvider) + { + return ImmutableList.of(); + } + + @Override + public Object getInfo() + { + return ImmutableMap.builder() + .put("fragments", fragments) + .build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LanceSplit that = (LanceSplit) o; + return Objects.equals(fragments, that.fragments); + } + + @Override + public int hashCode() + { + return Objects.hash(fragments); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("fragments", fragments) + .toString(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplitManager.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplitManager.java new file mode 100644 index 0000000000000..47f9d6e018f4d --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceSplitManager.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.FixedSplitSource; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.google.common.collect.ImmutableList; +import org.lance.Fragment; + +import javax.inject.Inject; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class LanceSplitManager + implements ConnectorSplitManager +{ + private final LanceNamespaceHolder namespaceHolder; + + @Inject + public LanceSplitManager(LanceNamespaceHolder namespaceHolder) + { + this.namespaceHolder = requireNonNull(namespaceHolder, "namespaceHolder is null"); + } + + @Override + public ConnectorSplitSource getSplits( + ConnectorTransactionHandle transactionHandle, + ConnectorSession session, + ConnectorTableLayoutHandle layout, + SplitSchedulingContext splitSchedulingContext) + { + LanceTableLayoutHandle layoutHandle = (LanceTableLayoutHandle) layout; + LanceTableHandle tableHandle = layoutHandle.getTable(); + + List fragments = namespaceHolder.getFragments( + tableHandle.getTableName()); + + List splits = fragments.stream() + .map(fragment -> (ConnectorSplit) new LanceSplit( + ImmutableList.of(fragment.getId()))) + .collect(toImmutableList()); + + return new FixedSplitSource(splits); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableHandle.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableHandle.java new file mode 100644 index 0000000000000..8806ba8ee387d --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableHandle.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.ConnectorTableHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class LanceTableHandle + implements ConnectorTableHandle +{ + private final String schemaName; + private final String tableName; + + @JsonCreator + public LanceTableHandle( + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName) + { + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + this.tableName = requireNonNull(tableName, "tableName is null"); + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @Override + public int hashCode() + { + return Objects.hash(schemaName, tableName); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + LanceTableHandle other = (LanceTableHandle) obj; + return Objects.equals(this.schemaName, other.schemaName) && + Objects.equals(this.tableName, other.tableName); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("schemaName", schemaName) + .add("tableName", tableName) + .toString(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableLayoutHandle.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableLayoutHandle.java new file mode 100644 index 0000000000000..9f5968711380b --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTableLayoutHandle.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class LanceTableLayoutHandle + implements ConnectorTableLayoutHandle +{ + private final LanceTableHandle table; + private final TupleDomain tupleDomain; + + @JsonCreator + public LanceTableLayoutHandle( + @JsonProperty("table") LanceTableHandle table, + @JsonProperty("tupleDomain") TupleDomain domain) + { + this.table = requireNonNull(table, "table is null"); + this.tupleDomain = requireNonNull(domain, "tupleDomain is null"); + } + + @JsonProperty + public LanceTableHandle getTable() + { + return table; + } + + @JsonProperty + public TupleDomain getTupleDomain() + { + return tupleDomain; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LanceTableLayoutHandle that = (LanceTableLayoutHandle) o; + return Objects.equals(table, that.table) && + Objects.equals(tupleDomain, that.tupleDomain); + } + + @Override + public int hashCode() + { + return Objects.hash(table, tupleDomain); + } + + @Override + public String toString() + { + return table.toString(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceTransactionHandle.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTransactionHandle.java new file mode 100644 index 0000000000000..1dee7096afe80 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceTransactionHandle.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +public enum LanceTransactionHandle + implements ConnectorTransactionHandle +{ + INSTANCE +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/LanceWritableTableHandle.java b/presto-lance/src/main/java/com/facebook/presto/lance/LanceWritableTableHandle.java new file mode 100644 index 0000000000000..28e8c26505759 --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/LanceWritableTableHandle.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorOutputTableHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class LanceWritableTableHandle + implements ConnectorInsertTableHandle, ConnectorOutputTableHandle +{ + private final String schemaName; + private final String tableName; + private final String schemaJson; + private final List inputColumns; + + @JsonCreator + public LanceWritableTableHandle( + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("schemaJson") String schemaJson, + @JsonProperty("inputColumns") List inputColumns) + { + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + this.tableName = requireNonNull(tableName, "tableName is null"); + this.schemaJson = requireNonNull(schemaJson, "schemaJson is null"); + this.inputColumns = ImmutableList.copyOf(requireNonNull(inputColumns, "inputColumns is null")); + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getSchemaJson() + { + return schemaJson; + } + + @JsonProperty + public List getInputColumns() + { + return inputColumns; + } + + @Override + public int hashCode() + { + return Objects.hash(schemaName, tableName, schemaJson, inputColumns); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + LanceWritableTableHandle other = (LanceWritableTableHandle) obj; + return Objects.equals(this.schemaName, other.schemaName) && + Objects.equals(this.tableName, other.tableName) && + Objects.equals(this.schemaJson, other.schemaJson) && + Objects.equals(this.inputColumns, other.inputColumns); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("schemaName", schemaName) + .add("tableName", tableName) + .toString(); + } +} diff --git a/presto-lance/src/main/java/com/facebook/presto/lance/ScannerFactory.java b/presto-lance/src/main/java/com/facebook/presto/lance/ScannerFactory.java new file mode 100644 index 0000000000000..6861b1c15752c --- /dev/null +++ b/presto-lance/src/main/java/com/facebook/presto/lance/ScannerFactory.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import org.apache.arrow.memory.BufferAllocator; +import org.lance.ipc.LanceScanner; + +import java.util.List; + +public interface ScannerFactory +{ + LanceScanner open(BufferAllocator allocator, List columns); + + void close(); +} diff --git a/presto-lance/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin b/presto-lance/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin new file mode 100644 index 0000000000000..05805c911d148 --- /dev/null +++ b/presto-lance/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin @@ -0,0 +1 @@ +com.facebook.presto.lance.LancePlugin diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/LanceQueryRunner.java b/presto-lance/src/test/java/com/facebook/presto/lance/LanceQueryRunner.java new file mode 100644 index 0000000000000..47cb45fb5989a --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/LanceQueryRunner.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.Session; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +public class LanceQueryRunner +{ + private static final Logger log = Logger.get(LanceQueryRunner.class); + private static final String DEFAULT_SOURCE = "test"; + private static final String DEFAULT_CATALOG = "lance"; + private static final String DEFAULT_SCHEMA = "default"; + + private LanceQueryRunner() + { + } + + public static DistributedQueryRunner createLanceQueryRunner(Map connectorProperties) + throws Exception + { + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(createSession()) + .setExtraProperties(ImmutableMap.of("http-server.http.port", "8080")) + .build(); + try { + queryRunner.installPlugin(new LancePlugin()); + connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); + + // Use a temp directory for lance root + Path tempDir = Files.createTempDirectory("lance-test"); + connectorProperties.putIfAbsent("lance.root-url", tempDir.toString()); + + queryRunner.createCatalog(DEFAULT_CATALOG, "lance", connectorProperties); + return queryRunner; + } + catch (Exception e) { + queryRunner.close(); + throw e; + } + } + + public static Session createSession() + { + return testSessionBuilder() + .setSource(DEFAULT_SOURCE) + .setCatalog(DEFAULT_CATALOG) + .setSchema(DEFAULT_SCHEMA) + .build(); + } + + public static void main(String[] args) + throws Exception + { + DistributedQueryRunner queryRunner = createLanceQueryRunner(ImmutableMap.of()); + log.info(format("Presto server started: %s", queryRunner.getCoordinator().getBaseUrl())); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceColumnHandle.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceColumnHandle.java new file mode 100644 index 0000000000000..072d1f2f9b94d --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceColumnHandle.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.airlift.json.JsonObjectMapperProvider; +import com.facebook.airlift.json.ObjectMapperProvider; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.type.TypeDeserializer; +import com.google.common.collect.ImmutableMap; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.testng.annotations.Test; + +import java.util.Collections; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.RealType.REAL; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; +import static org.testng.Assert.assertEquals; + +public class TestLanceColumnHandle +{ + @Test + public void testJsonRoundTrip() + { + LanceColumnHandle handle = new LanceColumnHandle("col1", BIGINT, true); + JsonCodec codec = getJsonCodec(); + String json = codec.toJson(handle); + LanceColumnHandle copy = codec.fromJson(json); + assertEquals(copy, handle); + assertEquals(copy.getColumnName(), "col1"); + assertEquals(copy.getColumnType(), BIGINT); + assertEquals(copy.isNullable(), true); + } + + @Test + public void testArrowToPrestoType() + { + assertEquals(LanceColumnHandle.toPrestoType(field("a", ArrowType.Bool.INSTANCE)), BOOLEAN); + assertEquals(LanceColumnHandle.toPrestoType(field("b", new ArrowType.Int(32, true))), INTEGER); + assertEquals(LanceColumnHandle.toPrestoType(field("c", new ArrowType.Int(64, true))), BIGINT); + assertEquals(LanceColumnHandle.toPrestoType(field("d", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE))), REAL); + assertEquals(LanceColumnHandle.toPrestoType(field("e", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))), DOUBLE); + assertEquals(LanceColumnHandle.toPrestoType(field("f", ArrowType.Utf8.INSTANCE)), VARCHAR); + } + + @Test + public void testPrestoToArrowType() + { + assertEquals(LanceColumnHandle.toArrowType(BOOLEAN), ArrowType.Bool.INSTANCE); + assertEquals(LanceColumnHandle.toArrowType(INTEGER), new ArrowType.Int(32, true)); + assertEquals(LanceColumnHandle.toArrowType(BIGINT), new ArrowType.Int(64, true)); + assertEquals(LanceColumnHandle.toArrowType(REAL), new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)); + assertEquals(LanceColumnHandle.toArrowType(DOUBLE), new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)); + assertEquals(LanceColumnHandle.toArrowType(VARCHAR), ArrowType.Utf8.INSTANCE); + } + + private static Field field(String name, ArrowType type) + { + return new Field(name, new FieldType(true, type, null), Collections.emptyList()); + } + + private JsonCodec getJsonCodec() + { + ObjectMapperProvider objectMapperProvider = new JsonObjectMapperProvider(); + objectMapperProvider.setJsonDeserializers(ImmutableMap.of(Type.class, new TypeDeserializer(createTestFunctionAndTypeManager()))); + return new JsonCodecFactory(objectMapperProvider).jsonCodec(LanceColumnHandle.class); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceCommitTaskData.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceCommitTaskData.java new file mode 100644 index 0000000000000..632ef473eb997 --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceCommitTaskData.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import org.testng.annotations.Test; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static org.testng.Assert.assertEquals; + +public class TestLanceCommitTaskData +{ + @Test + public void testJsonRoundTrip() + { + LanceCommitTaskData data = new LanceCommitTaskData( + "[{\"id\":1}]", + 1024L, + 100L); + JsonCodec codec = jsonCodec(LanceCommitTaskData.class); + String json = codec.toJson(data); + LanceCommitTaskData copy = codec.fromJson(json); + assertEquals(copy.getFragmentsJson(), "[{\"id\":1}]"); + assertEquals(copy.getWrittenBytes(), 1024L); + assertEquals(copy.getRowCount(), 100L); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceConfig.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceConfig.java new file mode 100644 index 0000000000000..4685b10b92a53 --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceConfig.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestLanceConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(LanceConfig.class) + .setImpl("dir") + .setRootUrl("") + .setSingleLevelNs(true) + .setReadBatchSize(8192) + .setMaxRowsPerFile(1_000_000) + .setMaxRowsPerGroup(100_000) + .setWriteBatchSize(10_000) + .setIndexCacheSizeBytes(134_217_728L) + .setMetadataCacheSizeBytes(134_217_728L)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("lance.impl", "rest") + .put("lance.root-url", "/data/lance") + .put("lance.single-level-ns", "false") + .put("lance.read-batch-size", "4096") + .put("lance.max-rows-per-file", "500000") + .put("lance.max-rows-per-group", "50000") + .put("lance.write-batch-size", "5000") + .put("lance.index-cache-size-bytes", "268435456") + .put("lance.metadata-cache-size-bytes", "536870912") + .build(); + + LanceConfig expected = new LanceConfig() + .setImpl("rest") + .setRootUrl("/data/lance") + .setSingleLevelNs(false) + .setReadBatchSize(4096) + .setMaxRowsPerFile(500_000) + .setMaxRowsPerGroup(50_000) + .setWriteBatchSize(5_000) + .setIndexCacheSizeBytes(268_435_456L) + .setMetadataCacheSizeBytes(536_870_912L); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceFragmentPageSource.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceFragmentPageSource.java new file mode 100644 index 0000000000000..e2c3fea4f2ba9 --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceFragmentPageSource.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.plugin.arrow.ArrowBlockBuilder; +import com.facebook.presto.common.Page; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.spi.ColumnHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; +import org.lance.Fragment; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestLanceFragmentPageSource +{ + private LanceNamespaceHolder namespaceHolder; + private LanceTableHandle tableHandle; + private String tablePath; + private List fragments; + private ArrowBlockBuilder arrowBlockBuilder; + + @BeforeMethod + public void setUp() + throws Exception + { + URL dbUrl = Resources.getResource(TestLanceFragmentPageSource.class, "/example_db"); + assertNotNull(dbUrl, "example_db resource not found"); + String rootPath = Paths.get(dbUrl.toURI()).toString(); + LanceConfig config = new LanceConfig() + .setRootUrl(rootPath) + .setSingleLevelNs(true); + namespaceHolder = new LanceNamespaceHolder(config); + arrowBlockBuilder = new ArrowBlockBuilder(createTestFunctionAndTypeManager()); + tableHandle = new LanceTableHandle("default", "test_table1"); + tablePath = namespaceHolder.getTablePath("test_table1"); + fragments = namespaceHolder.getFragments("test_table1"); + } + + @Test + public void testFragmentScan() + throws Exception + { + List columns = getColumns(); + + try (LanceFragmentPageSource pageSource = new LanceFragmentPageSource( + tableHandle, + columns, + ImmutableList.of(fragments.get(0).getId()), + tablePath, + 8192, + namespaceHolder.buildReadOptions(), + arrowBlockBuilder, + namespaceHolder.getAllocator())) { + Page page = pageSource.getNextPage(); + assertNotNull(page); + assertEquals(page.getChannelCount(), 4); + assertEquals(page.getPositionCount(), 2); + + // Verify first column (x) has expected values + Block xBlock = page.getBlock(0); + assertEquals(BIGINT.getLong(xBlock, 0), 0L); + + Page nextPage = pageSource.getNextPage(); + assertNull(nextPage); + assertTrue(pageSource.isFinished()); + } + } + + @Test + public void testColumnProjection() + throws Exception + { + Map columnHandleMap = getColumnHandles(); + LanceColumnHandle colB = (LanceColumnHandle) columnHandleMap.get("b"); + LanceColumnHandle colX = (LanceColumnHandle) columnHandleMap.get("x"); + List projectedColumns = ImmutableList.of(colB, colX); + + try (LanceFragmentPageSource pageSource = new LanceFragmentPageSource( + tableHandle, + projectedColumns, + ImmutableList.of(fragments.get(0).getId()), + tablePath, + 8192, + namespaceHolder.buildReadOptions(), + arrowBlockBuilder, + namespaceHolder.getAllocator())) { + Page page = pageSource.getNextPage(); + assertNotNull(page); + assertEquals(page.getChannelCount(), 2); + assertEquals(page.getPositionCount(), 2); + + Block bBlock = page.getBlock(0); + assertEquals(BIGINT.getLong(bBlock, 0), 0L); + assertEquals(BIGINT.getLong(bBlock, 1), 3L); + + Block xBlock = page.getBlock(1); + assertEquals(BIGINT.getLong(xBlock, 0), 0L); + assertEquals(BIGINT.getLong(xBlock, 1), 1L); + } + } + + @Test + public void testPartialColumnProjection() + throws Exception + { + Map columnHandleMap = getColumnHandles(); + LanceColumnHandle colC = (LanceColumnHandle) columnHandleMap.get("c"); + LanceColumnHandle colX = (LanceColumnHandle) columnHandleMap.get("x"); + List projectedColumns = ImmutableList.of(colC, colX); + + try (LanceFragmentPageSource pageSource = new LanceFragmentPageSource( + tableHandle, + projectedColumns, + ImmutableList.of(fragments.get(0).getId()), + tablePath, + 8192, + namespaceHolder.buildReadOptions(), + arrowBlockBuilder, + namespaceHolder.getAllocator())) { + Page page = pageSource.getNextPage(); + assertNotNull(page); + assertEquals(page.getChannelCount(), 2); + assertEquals(page.getPositionCount(), 2); + + Block cBlock = page.getBlock(0); + assertEquals(BIGINT.getLong(cBlock, 0), 0L); + assertEquals(BIGINT.getLong(cBlock, 1), -1L); + + Block xBlock = page.getBlock(1); + assertEquals(BIGINT.getLong(xBlock, 0), 0L); + assertEquals(BIGINT.getLong(xBlock, 1), 1L); + } + } + + private List getColumns() + { + return getColumnHandles().values().stream() + .map(LanceColumnHandle.class::cast) + .collect(toImmutableList()); + } + + private Map getColumnHandles() + { + LanceMetadata metadata = new LanceMetadata(namespaceHolder, jsonCodec(LanceCommitTaskData.class)); + return metadata.getColumnHandles(null, tableHandle); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceMetadata.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceMetadata.java new file mode 100644 index 0000000000000..cbb93d7d18d7a --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceMetadata.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.SchemaTableName; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Resources; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.net.URL; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestLanceMetadata +{ + private LanceMetadata metadata; + + @BeforeMethod + public void setUp() + throws Exception + { + URL dbUrl = Resources.getResource(TestLanceMetadata.class, "/example_db"); + assertNotNull(dbUrl, "example_db resource not found"); + String rootPath = Paths.get(dbUrl.toURI()).toString(); + LanceConfig config = new LanceConfig() + .setRootUrl(rootPath) + .setSingleLevelNs(true); + LanceNamespaceHolder namespaceHolder = new LanceNamespaceHolder(config); + JsonCodec commitTaskDataCodec = jsonCodec(LanceCommitTaskData.class); + metadata = new LanceMetadata(namespaceHolder, commitTaskDataCodec); + } + + @Test + public void testListSchemaNames() + { + List schemas = metadata.listSchemaNames(null); + assertEquals(schemas.size(), 1); + assertEquals(schemas.get(0), "default"); + } + + @Test + public void testGetTableHandle() + { + ConnectorTableHandle handle = metadata.getTableHandle(null, new SchemaTableName("default", "test_table1")); + assertNotNull(handle); + assertEquals(handle, new LanceTableHandle("default", "test_table1")); + + ConnectorTableHandle handle2 = metadata.getTableHandle(null, new SchemaTableName("default", "test_table2")); + assertNotNull(handle2); + assertEquals(handle2, new LanceTableHandle("default", "test_table2")); + + // non-existent schema + assertNull(metadata.getTableHandle(null, new SchemaTableName("other_schema", "test_table1"))); + + // non-existent table + assertNull(metadata.getTableHandle(null, new SchemaTableName("default", "nonexistent"))); + } + + @Test + public void testGetColumnHandles() + { + LanceTableHandle tableHandle = new LanceTableHandle("default", "test_table1"); + Map columns = metadata.getColumnHandles(null, tableHandle); + assertNotNull(columns); + assertEquals(columns.size(), 4); + assertTrue(columns.containsKey("x")); + assertTrue(columns.containsKey("y")); + assertTrue(columns.containsKey("b")); + assertTrue(columns.containsKey("c")); + } + + @Test + public void testGetTableMetadata() + { + LanceTableHandle tableHandle = new LanceTableHandle("default", "test_table1"); + ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(null, tableHandle); + assertNotNull(tableMetadata); + assertEquals(tableMetadata.getTable(), new SchemaTableName("default", "test_table1")); + assertEquals(tableMetadata.getColumns().size(), 4); + + // Verify column names + Set columnNames = tableMetadata.getColumns().stream() + .map(col -> col.getName()) + .collect(Collectors.toSet()); + assertEquals(columnNames, ImmutableSet.of("x", "y", "b", "c")); + } + + @Test + public void testListTables() + { + // all tables in default schema + List tables = metadata.listTables(null, Optional.of("default")); + Set tableSet = ImmutableSet.copyOf(tables); + assertEquals(tableSet, ImmutableSet.of( + new SchemaTableName("default", "test_table1"), + new SchemaTableName("default", "test_table2"), + new SchemaTableName("default", "test_table3"), + new SchemaTableName("default", "test_table4"))); + + // no schema filter + List allTables = metadata.listTables(null, Optional.empty()); + assertEquals(ImmutableSet.copyOf(allTables), tableSet); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLancePlugin.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLancePlugin.java new file mode 100644 index 0000000000000..047f7355fc16e --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLancePlugin.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.testing.TestingConnectorContext; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.MoreCollectors; +import org.testng.annotations.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +public class TestLancePlugin +{ + @Test + public void testCreateConnector() + throws Exception + { + ConnectorFactory factory = StreamSupport + .stream(new LancePlugin().getConnectorFactories().spliterator(), false) + .collect(MoreCollectors.onlyElement()); + assertNotNull(factory); + assertEquals(factory.getName(), "lance"); + Path tempDir = Files.createTempDirectory("lance-test"); + try { + factory.create( + "test", + ImmutableMap.of("lance.root-url", tempDir.toString()), + new TestingConnectorContext()) + .shutdown(); + } + finally { + deleteRecursively(tempDir); + } + } + + private static void deleteRecursively(Path path) + throws Exception + { + if (Files.isDirectory(path)) { + try (Stream entries = Files.list(path)) { + for (Path entry : (Iterable) entries::iterator) { + deleteRecursively(entry); + } + } + } + Files.deleteIfExists(path); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceSplit.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceSplit.java new file mode 100644 index 0000000000000..b83314e18b2fd --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceSplit.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static org.testng.Assert.assertEquals; + +public class TestLanceSplit +{ + @Test + public void testJsonRoundTrip() + { + LanceSplit split = new LanceSplit(ImmutableList.of(0, 1, 2)); + JsonCodec codec = jsonCodec(LanceSplit.class); + String json = codec.toJson(split); + LanceSplit copy = codec.fromJson(json); + assertEquals(copy.getFragments(), ImmutableList.of(0, 1, 2)); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceTableHandle.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceTableHandle.java new file mode 100644 index 0000000000000..ea59aff551009 --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceTableHandle.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.facebook.airlift.json.JsonCodec; +import org.testng.annotations.Test; + +import static com.facebook.airlift.json.JsonCodec.jsonCodec; +import static org.testng.Assert.assertEquals; + +public class TestLanceTableHandle +{ + private final LanceTableHandle tableHandle = new LanceTableHandle("default", "test_table"); + + @Test + public void testJsonRoundTrip() + { + JsonCodec codec = jsonCodec(LanceTableHandle.class); + String json = codec.toJson(tableHandle); + LanceTableHandle copy = codec.fromJson(json); + assertEquals(copy, tableHandle); + } +} diff --git a/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceWritableTableHandle.java b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceWritableTableHandle.java new file mode 100644 index 0000000000000..32996db8acb64 --- /dev/null +++ b/presto-lance/src/test/java/com/facebook/presto/lance/TestLanceWritableTableHandle.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.lance; + +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import java.util.List; + +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertEquals; + +public class TestLanceWritableTableHandle +{ + @Test + public void testProperties() + { + List columns = ImmutableList.of( + new LanceColumnHandle("id", INTEGER, false), + new LanceColumnHandle("name", VARCHAR, true)); + LanceWritableTableHandle handle = new LanceWritableTableHandle( + "default", "test_table", "{}", columns); + + assertEquals(handle.getSchemaName(), "default"); + assertEquals(handle.getTableName(), "test_table"); + assertEquals(handle.getSchemaJson(), "{}"); + assertEquals(handle.getInputColumns().size(), 2); + assertEquals(handle.getInputColumns().get(0).getColumnName(), "id"); + } +} diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_latest.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_latest.manifest new file mode 100644 index 0000000000000..ad3bfea3fa163 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_latest.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/0-4daea2b4-b38b-4542-af0c-5a839ceab54a.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/0-4daea2b4-b38b-4542-af0c-5a839ceab54a.txn new file mode 100644 index 0000000000000..a4de7d66c40bd Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/0-4daea2b4-b38b-4542-af0c-5a839ceab54a.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/1-99519b7f-c80f-4961-bacc-d556df5ae798.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/1-99519b7f-c80f-4961-bacc-d556df5ae798.txn new file mode 100644 index 0000000000000..e5d1c6c78de54 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/1-99519b7f-c80f-4961-bacc-d556df5ae798.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/2-b9f7655d-01e1-4fa7-8ca2-ddc646564fb8.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/2-b9f7655d-01e1-4fa7-8ca2-ddc646564fb8.txn new file mode 100644 index 0000000000000..0eeae4812848a Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/2-b9f7655d-01e1-4fa7-8ca2-ddc646564fb8.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/3-90bc5dd5-204d-42ba-b39a-65f2abce1602.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/3-90bc5dd5-204d-42ba-b39a-65f2abce1602.txn new file mode 100644 index 0000000000000..26e81b4bf6228 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/3-90bc5dd5-204d-42ba-b39a-65f2abce1602.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/4-dffa23f0-c357-4935-a9c1-e286099b5533.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/4-dffa23f0-c357-4935-a9c1-e286099b5533.txn new file mode 100644 index 0000000000000..9f4d38bed499a --- /dev/null +++ b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/4-dffa23f0-c357-4935-a9c1-e286099b5533.txn @@ -0,0 +1,3 @@ +$dffa23f0-c357-4935-a9c1-e286099b5533ê8 +x ÿÿÿÿÿÿÿÿÿ*int6408 +y ÿÿÿÿÿÿÿÿÿ*int6408 \ No newline at end of file diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/5-8bfb238d-4a29-4582-ab7d-8c53e2253e47.txn b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/5-8bfb238d-4a29-4582-ab7d-8c53e2253e47.txn new file mode 100644 index 0000000000000..c5ebfccd035d4 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_transactions/5-8bfb238d-4a29-4582-ab7d-8c53e2253e47.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/1.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/1.manifest new file mode 100644 index 0000000000000..9afc401b67a58 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/1.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/2.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/2.manifest new file mode 100644 index 0000000000000..02964ac016274 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/2.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/3.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/3.manifest new file mode 100644 index 0000000000000..d58ef994212c6 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/3.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/4.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/4.manifest new file mode 100644 index 0000000000000..d93ec5f20c80c Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/4.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/5.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/5.manifest new file mode 100644 index 0000000000000..2e829ca96b715 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/5.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/6.manifest b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/6.manifest new file mode 100644 index 0000000000000..ad3bfea3fa163 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/_versions/6.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/083d1c7c-b0d2-4ff3-b7ff-4237ea586491.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/083d1c7c-b0d2-4ff3-b7ff-4237ea586491.lance new file mode 100644 index 0000000000000..d20bd1d1d80d5 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/083d1c7c-b0d2-4ff3-b7ff-4237ea586491.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/25c37abd-c753-419b-b420-4847ce2de5a1.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/25c37abd-c753-419b-b420-4847ce2de5a1.lance new file mode 100644 index 0000000000000..8c320c2d27578 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/25c37abd-c753-419b-b420-4847ce2de5a1.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/2c8a0da6-1ace-4b1c-baf0-ed48b04996dc.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/2c8a0da6-1ace-4b1c-baf0-ed48b04996dc.lance new file mode 100644 index 0000000000000..089439d826b46 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/2c8a0da6-1ace-4b1c-baf0-ed48b04996dc.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/ac0bf34e-0e0d-4e3b-ae7e-ab247cae5f77.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/ac0bf34e-0e0d-4e3b-ae7e-ab247cae5f77.lance new file mode 100644 index 0000000000000..123baf9574aac Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/ac0bf34e-0e0d-4e3b-ae7e-ab247cae5f77.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/c888f970-b7b3-4efb-9293-d7c6dc4996d2.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/c888f970-b7b3-4efb-9293-d7c6dc4996d2.lance new file mode 100644 index 0000000000000..92b5c5bfab42b Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/c888f970-b7b3-4efb-9293-d7c6dc4996d2.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table1.lance/data/cbe16da7-b812-43a1-87f1-521470dfed32.lance b/presto-lance/src/test/resources/example_db/test_table1.lance/data/cbe16da7-b812-43a1-87f1-521470dfed32.lance new file mode 100644 index 0000000000000..6aa336b8e0f58 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table1.lance/data/cbe16da7-b812-43a1-87f1-521470dfed32.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_deletions/0-1-8958018423523767581.arrow b/presto-lance/src/test/resources/example_db/test_table2.lance/_deletions/0-1-8958018423523767581.arrow new file mode 100644 index 0000000000000..0ff9e8401fbd2 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_deletions/0-1-8958018423523767581.arrow differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_latest.manifest b/presto-lance/src/test/resources/example_db/test_table2.lance/_latest.manifest new file mode 100644 index 0000000000000..c278c42a37dab Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_latest.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn b/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn new file mode 100644 index 0000000000000..39f2c93c022d2 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn b/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn new file mode 100644 index 0000000000000..b7b085fcf5b10 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/1.manifest b/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/1.manifest new file mode 100644 index 0000000000000..8fb093d607e9c Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/1.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/2.manifest b/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/2.manifest new file mode 100644 index 0000000000000..c278c42a37dab Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/_versions/2.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table2.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance b/presto-lance/src/test/resources/example_db/test_table2.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance new file mode 100644 index 0000000000000..ae5ace0c3b39e Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table2.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_deletions/0-1-8958018423523767581.arrow b/presto-lance/src/test/resources/example_db/test_table3.lance/_deletions/0-1-8958018423523767581.arrow new file mode 100644 index 0000000000000..0ff9e8401fbd2 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_deletions/0-1-8958018423523767581.arrow differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_latest.manifest b/presto-lance/src/test/resources/example_db/test_table3.lance/_latest.manifest new file mode 100644 index 0000000000000..e7e6573b28441 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_latest.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn new file mode 100644 index 0000000000000..39f2c93c022d2 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/0-304ab2ef-f7bc-47b8-aeb6-9110ec67bf98.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn new file mode 100644 index 0000000000000..b7b085fcf5b10 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/1-1baf3405-66ab-4668-9578-5c333acd0440.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/2-8e340735-2a60-438b-9cf0-ec662fb25f1a.txn b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/2-8e340735-2a60-438b-9cf0-ec662fb25f1a.txn new file mode 100644 index 0000000000000..5021d6474bf5a Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_transactions/2-8e340735-2a60-438b-9cf0-ec662fb25f1a.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/1.manifest b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/1.manifest new file mode 100644 index 0000000000000..8fb093d607e9c Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/1.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/2.manifest b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/2.manifest new file mode 100644 index 0000000000000..c278c42a37dab Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/2.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/3.manifest b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/3.manifest new file mode 100644 index 0000000000000..e7e6573b28441 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/_versions/3.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance b/presto-lance/src/test/resources/example_db/test_table3.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance new file mode 100644 index 0000000000000..ae5ace0c3b39e Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/data/016c15dc-2c94-4382-b7a4-2c7def9c3897.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table3.lance/data/e6574672-b3cb-4bc7-92a8-db8754dac368.lance b/presto-lance/src/test/resources/example_db/test_table3.lance/data/e6574672-b3cb-4bc7-92a8-db8754dac368.lance new file mode 100644 index 0000000000000..004a874b0b838 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table3.lance/data/e6574672-b3cb-4bc7-92a8-db8754dac368.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/d32dac97-985b-4628-b1b4-e4b64947e115/index.idx b/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/d32dac97-985b-4628-b1b4-e4b64947e115/index.idx new file mode 100644 index 0000000000000..9ba041c6d31da Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/d32dac97-985b-4628-b1b4-e4b64947e115/index.idx differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/f358a219-95e8-4956-be35-0835f2bed10f/index.idx b/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/f358a219-95e8-4956-be35-0835f2bed10f/index.idx new file mode 100644 index 0000000000000..9c86bf0a03277 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_indices/f358a219-95e8-4956-be35-0835f2bed10f/index.idx differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_latest.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_latest.manifest new file mode 100644 index 0000000000000..93b8060625d5a Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_latest.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/0-c4ece134-3d52-41a8-b2ec-0fb9fff76c35.txn b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/0-c4ece134-3d52-41a8-b2ec-0fb9fff76c35.txn new file mode 100644 index 0000000000000..15c6233536522 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/0-c4ece134-3d52-41a8-b2ec-0fb9fff76c35.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/1-cac38053-d1b8-4ff5-b34c-0e47b41c1b56.txn b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/1-cac38053-d1b8-4ff5-b34c-0e47b41c1b56.txn new file mode 100644 index 0000000000000..603a2023ec001 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/1-cac38053-d1b8-4ff5-b34c-0e47b41c1b56.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/2-f3ac6254-2471-4c8a-8183-e529af6d2603.txn b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/2-f3ac6254-2471-4c8a-8183-e529af6d2603.txn new file mode 100644 index 0000000000000..4359abf588ba9 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/2-f3ac6254-2471-4c8a-8183-e529af6d2603.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/3-e08f185e-5734-4533-bee5-325567f2221a.txn b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/3-e08f185e-5734-4533-bee5-325567f2221a.txn new file mode 100644 index 0000000000000..849aad9471f5b Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/3-e08f185e-5734-4533-bee5-325567f2221a.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/4-2536db77-3757-414b-a525-f8f3288e9d80.txn b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/4-2536db77-3757-414b-a525-f8f3288e9d80.txn new file mode 100644 index 0000000000000..32d0c7d28eae9 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_transactions/4-2536db77-3757-414b-a525-f8f3288e9d80.txn differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/1.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/1.manifest new file mode 100644 index 0000000000000..bacf074f4c002 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/1.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/2.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/2.manifest new file mode 100644 index 0000000000000..043802ce5067d Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/2.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/3.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/3.manifest new file mode 100644 index 0000000000000..4e3ebb1919966 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/3.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/4.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/4.manifest new file mode 100644 index 0000000000000..426f1ba6f4f65 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/4.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/5.manifest b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/5.manifest new file mode 100644 index 0000000000000..93b8060625d5a Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/_versions/5.manifest differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/03c1a82b-a745-4bfe-8413-9441e4ed216e.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/03c1a82b-a745-4bfe-8413-9441e4ed216e.lance new file mode 100644 index 0000000000000..6c7822d71f99d Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/03c1a82b-a745-4bfe-8413-9441e4ed216e.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/2f786e97-1d4c-43e5-bc32-6f7a444396f1.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/2f786e97-1d4c-43e5-bc32-6f7a444396f1.lance new file mode 100644 index 0000000000000..9fb75907db078 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/2f786e97-1d4c-43e5-bc32-6f7a444396f1.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/34199dea-ca38-460b-af71-a816b0f093a1.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/34199dea-ca38-460b-af71-a816b0f093a1.lance new file mode 100644 index 0000000000000..15bd2f6095741 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/34199dea-ca38-460b-af71-a816b0f093a1.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/37ff0067-df64-4ba7-8c50-2086eb2b8127.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/37ff0067-df64-4ba7-8c50-2086eb2b8127.lance new file mode 100644 index 0000000000000..bd7732dd741b0 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/37ff0067-df64-4ba7-8c50-2086eb2b8127.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/4062824b-36bd-42e6-9283-22e9f29dc5ed.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/4062824b-36bd-42e6-9283-22e9f29dc5ed.lance new file mode 100644 index 0000000000000..b6753118f6064 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/4062824b-36bd-42e6-9283-22e9f29dc5ed.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/4d41cc61-800b-46b0-a548-893a35201cf1.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/4d41cc61-800b-46b0-a548-893a35201cf1.lance new file mode 100644 index 0000000000000..c14245dff6966 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/4d41cc61-800b-46b0-a548-893a35201cf1.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/66c4453b-7e80-411d-8508-e7f6dfeb693e.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/66c4453b-7e80-411d-8508-e7f6dfeb693e.lance new file mode 100644 index 0000000000000..90007b84741d4 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/66c4453b-7e80-411d-8508-e7f6dfeb693e.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/7ac6d965-4d35-4e2b-825b-f4a4a8be9024.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/7ac6d965-4d35-4e2b-825b-f4a4a8be9024.lance new file mode 100644 index 0000000000000..71f59ba6c4705 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/7ac6d965-4d35-4e2b-825b-f4a4a8be9024.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/86d11ae4-4a8f-48bc-b1b8-3c850a67c871.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/86d11ae4-4a8f-48bc-b1b8-3c850a67c871.lance new file mode 100644 index 0000000000000..bb61fcdea1035 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/86d11ae4-4a8f-48bc-b1b8-3c850a67c871.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/cd32d611-941e-4aa9-88c4-72193c618255.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/cd32d611-941e-4aa9-88c4-72193c618255.lance new file mode 100644 index 0000000000000..c2fd193ee243d Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/cd32d611-941e-4aa9-88c4-72193c618255.lance differ diff --git a/presto-lance/src/test/resources/example_db/test_table4.lance/data/ec05a2ea-2387-45a0-a146-1208997c4f12.lance b/presto-lance/src/test/resources/example_db/test_table4.lance/data/ec05a2ea-2387-45a0-a146-1208997c4f12.lance new file mode 100644 index 0000000000000..356f66f3c2305 Binary files /dev/null and b/presto-lance/src/test/resources/example_db/test_table4.lance/data/ec05a2ea-2387-45a0-a146-1208997c4f12.lance differ diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index 2900e09ea3cb4..afd37499f1a95 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -84,6 +84,33 @@ presto-expressions + + com.facebook.presto + presto-internal-communication + + + com.facebook.airlift + http-server + + + com.facebook.airlift + http-client + + + com.facebook.airlift + jaxrs + + + io.jsonwebtoken + jjwt-api + + + com.facebook.airlift.drift + drift-transport-netty + + + + com.facebook.presto presto-main-tests @@ -562,6 +589,7 @@ com.facebook.airlift.drift:drift-transport-spi + com.facebook.airlift.drift:drift-transport-netty io.netty:netty-buffer diff --git a/presto-main-base/src/main/java/com/facebook/presto/Session.java b/presto-main-base/src/main/java/com/facebook/presto/Session.java index 0492f7d1bd300..63219c39082e9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/Session.java +++ b/presto-main-base/src/main/java/com/facebook/presto/Session.java @@ -43,6 +43,7 @@ import com.facebook.presto.sql.planner.optimizations.OptimizerResultCollector; import com.facebook.presto.transaction.TransactionManager; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -58,6 +59,7 @@ import java.util.stream.Collectors; import static com.facebook.presto.SystemSessionProperties.LEGACY_JSON_CAST; +import static com.facebook.presto.SystemSessionProperties.getTryFunctionCatchableErrors; import static com.facebook.presto.SystemSessionProperties.isCanonicalizedJsonExtract; import static com.facebook.presto.SystemSessionProperties.isFieldNameInJsonCastEnabled; import static com.facebook.presto.SystemSessionProperties.isLegacyMapSubscript; @@ -538,6 +540,7 @@ public SqlFunctionProperties getSqlFunctionProperties() .setExtraCredentials(identity.getExtraCredentials()) .setWarnOnCommonNanPatterns(warnOnCommonNanPatterns(this)) .setCanonicalizedJsonExtract(isCanonicalizedJsonExtract(this)) + .setTryCatchableErrorCodes(parseTryCatchableErrorCodes(getTryFunctionCatchableErrors(this))) .build(); } @@ -625,6 +628,19 @@ public static SessionBuilder builder(Session session) return new SessionBuilder(session); } + private static Set parseTryCatchableErrorCodes(String errorCodesString) + { + if (errorCodesString == null || errorCodesString.isEmpty()) { + return ImmutableSet.of(); + } + return Splitter.on(",") + .trimResults() + .omitEmptyStrings() + .splitToList(errorCodesString) + .stream() + .collect(ImmutableSet.toImmutableSet()); + } + public static class SessionBuilder { private QueryId queryId; diff --git a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java index 9726e14b7c40f..1d4bbdde14608 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -41,6 +41,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinNotNullInferenceStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.LeftJoinArrayContainsToInnerJoinStrategy; +import com.facebook.presto.sql.analyzer.FeaturesConfig.LocalExchangeParentPreferenceStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialMergePushdownStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartitioningPrecisionStrategy; @@ -172,6 +173,10 @@ public final class SystemSessionProperties public static final String LEGACY_TIMESTAMP = "legacy_timestamp"; public static final String ENABLE_INTERMEDIATE_AGGREGATIONS = "enable_intermediate_aggregations"; public static final String PUSH_AGGREGATION_THROUGH_JOIN = "push_aggregation_through_join"; + public static final String PUSH_SEMI_JOIN_THROUGH_UNION = "push_semi_join_through_union"; + public static final String SIMPLIFY_COALESCE_OVER_JOIN_KEYS = "simplify_coalesce_over_join_keys"; + public static final String PUSHDOWN_THROUGH_UNNEST = "pushdown_through_unnest"; + public static final String SIMPLIFY_AGGREGATIONS_OVER_CONSTANT = "simplify_aggregations_over_constant"; public static final String PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN = "push_partial_aggregation_through_join"; public static final String PARSE_DECIMAL_LITERALS_AS_DOUBLE = "parse_decimal_literals_as_double"; public static final String FORCE_SINGLE_NODE_OUTPUT = "force_single_node_output"; @@ -186,6 +191,7 @@ public final class SystemSessionProperties public static final String ADAPTIVE_PARTIAL_AGGREGATION = "adaptive_partial_aggregation"; public static final String ADAPTIVE_PARTIAL_AGGREGATION_ROWS_REDUCTION_RATIO_THRESHOLD = "adaptive_partial_aggregation_unique_rows_ratio_threshold"; public static final String OPTIMIZE_TOP_N_ROW_NUMBER = "optimize_top_n_row_number"; + public static final String OPTIMIZE_TOP_N_RANK = "optimize_top_n_rank"; public static final String OPTIMIZE_CASE_EXPRESSION_PREDICATE = "optimize_case_expression_predicate"; public static final String MAX_GROUPING_SETS = "max_grouping_sets"; public static final String LEGACY_UNNEST = "legacy_unnest"; @@ -209,6 +215,7 @@ public final class SystemSessionProperties public static final String INDEX_LOADER_TIMEOUT = "index_loader_timeout"; public static final String OPTIMIZED_REPARTITIONING_ENABLED = "optimized_repartitioning"; public static final String AGGREGATION_PARTITIONING_MERGING_STRATEGY = "aggregation_partitioning_merging_strategy"; + public static final String LOCAL_EXCHANGE_PARENT_PREFERENCE_STRATEGY = "local_exchange_parent_preference_strategy"; public static final String LIST_BUILT_IN_FUNCTIONS_ONLY = "list_built_in_functions_only"; public static final String NON_BUILT_IN_FUNCTION_NAMESPACES_TO_LIST_FUNCTIONS = "non_built_in_function_namespaces_to_list_functions"; public static final String PARTITIONING_PRECISION_STRATEGY = "partitioning_precision_strategy"; @@ -252,6 +259,8 @@ public final class SystemSessionProperties public static final String LEGACY_MATERIALIZED_VIEWS = "legacy_materialized_views"; public static final String MATERIALIZED_VIEW_ALLOW_FULL_REFRESH_ENABLED = "materialized_view_allow_full_refresh_enabled"; public static final String MATERIALIZED_VIEW_STALE_READ_BEHAVIOR = "materialized_view_stale_read_behavior"; + public static final String MATERIALIZED_VIEW_STALENESS_WINDOW = "materialized_view_staleness_window"; + public static final String MATERIALIZED_VIEW_FORCE_STALE = "materialized_view_force_stale"; public static final String AGGREGATION_IF_TO_FILTER_REWRITE_STRATEGY = "aggregation_if_to_filter_rewrite_strategy"; public static final String JOINS_NOT_NULL_INFERENCE_STRATEGY = "joins_not_null_inference_strategy"; public static final String RESOURCE_AWARE_SCHEDULING_STRATEGY = "resource_aware_scheduling_strategy"; @@ -356,11 +365,14 @@ public final class SystemSessionProperties public static final String ADD_DISTINCT_BELOW_SEMI_JOIN_BUILD = "add_distinct_below_semi_join_build"; public static final String UTILIZE_UNIQUE_PROPERTY_IN_QUERY_PLANNING = "utilize_unique_property_in_query_planning"; public static final String PUSHDOWN_SUBFIELDS_FOR_MAP_FUNCTIONS = "pushdown_subfields_for_map_functions"; + public static final String PUSHDOWN_SUBFIELDS_FOR_CARDINALITY = "pushdown_subfields_for_cardinality"; public static final String MAX_SERIALIZABLE_OBJECT_SIZE = "max_serializable_object_size"; public static final String EXPRESSION_OPTIMIZER_IN_ROW_EXPRESSION_REWRITE = "expression_optimizer_in_row_expression_rewrite"; public static final String TABLE_SCAN_SHUFFLE_PARALLELISM_THRESHOLD = "table_scan_shuffle_parallelism_threshold"; public static final String TABLE_SCAN_SHUFFLE_STRATEGY = "table_scan_shuffle_strategy"; public static final String SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION = "skip_pushdown_through_exchange_for_remote_projection"; + public static final String REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM = "remote_function_names_for_fixed_parallelism"; + public static final String REMOTE_FUNCTION_FIXED_PARALLELISM_TASK_COUNT = "remote_function_fixed_parallelism_task_count"; // TODO: Native execution related session properties that are temporarily put here. They will be relocated in the future. public static final String NATIVE_AGGREGATION_SPILL_ALL = "native_aggregation_spill_all"; @@ -374,6 +386,7 @@ public final class SystemSessionProperties public static final String NATIVE_MIN_COLUMNAR_ENCODING_CHANNELS_TO_PREFER_ROW_WISE_ENCODING = "native_min_columnar_encoding_channels_to_prefer_row_wise_encoding"; public static final String NATIVE_ENFORCE_JOIN_BUILD_INPUT_PARTITION = "native_enforce_join_build_input_partition"; public static final String NATIVE_EXECUTION_SCALE_WRITER_THREADS_ENABLED = "native_execution_scale_writer_threads_enabled"; + public static final String TRY_FUNCTION_CATCHABLE_ERRORS = "try_function_catchable_errors"; private final List> sessionProperties; @@ -917,10 +930,30 @@ public SystemSessionProperties( "Allow pushing aggregations below joins", featuresConfig.isPushAggregationThroughJoin(), false), + booleanProperty( + PUSH_SEMI_JOIN_THROUGH_UNION, + "Allow pushing semi joins through union", + featuresConfig.isPushSemiJoinThroughUnion(), + false), + booleanProperty( + SIMPLIFY_COALESCE_OVER_JOIN_KEYS, + "Simplify redundant COALESCE expressions over equi-join keys", + featuresConfig.isSimplifyCoalesceOverJoinKeys(), + false), + booleanProperty( + PUSHDOWN_THROUGH_UNNEST, + "Allow pushing projections and filters below unnest", + featuresConfig.isPushdownThroughUnnest(), + false), + booleanProperty( + SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, + "Fold aggregation functions over constant arguments to constants", + featuresConfig.isSimplifyAggregationsOverConstant(), + false), booleanProperty( PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN, "Push partial aggregations below joins", - false, + featuresConfig.isPushPartialAggregationThroughJoin(), false), booleanProperty( PARSE_DECIMAL_LITERALS_AS_DOUBLE, @@ -998,6 +1031,11 @@ public SystemSessionProperties( "Use top N row number optimization", featuresConfig.isOptimizeTopNRowNumber(), false), + booleanProperty( + OPTIMIZE_TOP_N_RANK, + "Use top N rank and dense_rank optimization", + featuresConfig.isOptimizeTopNRank(), + false), booleanProperty( OPTIMIZE_CASE_EXPRESSION_PREDICATE, "Optimize case expression predicates", @@ -1149,6 +1187,18 @@ public SystemSessionProperties( false, value -> AggregationPartitioningMergingStrategy.valueOf(((String) value).toUpperCase()), AggregationPartitioningMergingStrategy::name), + new PropertyMetadata<>( + LOCAL_EXCHANGE_PARENT_PREFERENCE_STRATEGY, + format("Strategy to use parent preferences in local exchange partitioning for aggregations. Options are %s", + Stream.of(LocalExchangeParentPreferenceStrategy.values()) + .map(LocalExchangeParentPreferenceStrategy::name) + .collect(joining(","))), + VARCHAR, + LocalExchangeParentPreferenceStrategy.class, + featuresConfig.getLocalExchangeParentPreferenceStrategy(), + false, + value -> LocalExchangeParentPreferenceStrategy.valueOf(((String) value).toUpperCase()), + LocalExchangeParentPreferenceStrategy::name), booleanProperty( LIST_BUILT_IN_FUNCTIONS_ONLY, "Only List built-in functions in SHOW FUNCTIONS", @@ -1413,6 +1463,20 @@ public SystemSessionProperties( false, value -> MaterializedViewStaleReadBehavior.valueOf(((String) value).toUpperCase()), MaterializedViewStaleReadBehavior::name), + new PropertyMetadata<>( + MATERIALIZED_VIEW_STALENESS_WINDOW, + "Default staleness window for materialized views (e.g., '1h', '30m'). Use negative values (e.g., '-1ms') to always use the view query.", + VARCHAR, + Duration.class, + null, + false, + value -> value == null ? null : Duration.valueOf((String) value), + value -> value == null ? null : ((Duration) value).toString()), + booleanProperty( + MATERIALIZED_VIEW_FORCE_STALE, + "Force materialized views to be treated as stale even when fresh, triggering the stale read behavior. For testing only.", + false, + true), stringProperty( DISTRIBUTED_TRACING_MODE, "Mode for distributed tracing. NO_TRACE, ALWAYS_TRACE, or SAMPLE_BASED", @@ -2053,6 +2117,10 @@ public SystemSessionProperties( "Enable subfield pruning for map functions, currently include map_subset and map_filter", featuresConfig.isPushdownSubfieldForMapFunctions(), false), + booleanProperty(PUSHDOWN_SUBFIELDS_FOR_CARDINALITY, + "Enable subfield pruning for cardinality() function to skip reading keys and values", + featuresConfig.isPushdownSubfieldForCardinality(), + false), longProperty(MAX_SERIALIZABLE_OBJECT_SIZE, "Configure the maximum byte size of a serializable object in expression interpreters", featuresConfig.getMaxSerializableObjectSize(), @@ -2079,6 +2147,16 @@ public SystemSessionProperties( "Skip pushing down remote projection through exchange", featuresConfig.isSkipPushdownThroughExchangeForRemoteProjection(), false), + stringProperty( + REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, + "Regex pattern to match remote function names that should use fixed parallelism", + featuresConfig.getRemoteFunctionNamesForFixedParallelism(), + false), + integerProperty( + REMOTE_FUNCTION_FIXED_PARALLELISM_TASK_COUNT, + "Number of tasks to use for remote functions matching the fixed parallelism pattern. If not set, the default hash partition count will be used.", + featuresConfig.getRemoteFunctionFixedParallelismTaskCount(), + false), new PropertyMetadata<>( QUERY_CLIENT_TIMEOUT, "Configures how long the query runs without contact from the client application, such as the CLI, before it's abandoned", @@ -2095,6 +2173,11 @@ public SystemSessionProperties( booleanProperty(ADD_DISTINCT_BELOW_SEMI_JOIN_BUILD, "Add distinct aggregation below semi join build", featuresConfig.isAddDistinctBelowSemiJoinBuild(), + false), + stringProperty( + TRY_FUNCTION_CATCHABLE_ERRORS, + "Comma-separated list of error code names that TRY function should catch (such as 'GENERIC_INTERNAL_ERROR,INVALID_ARGUMENTS')", + featuresConfig.getTryFunctionCatchableErrors(), false)); } @@ -2555,6 +2638,21 @@ public static boolean shouldPushAggregationThroughJoin(Session session) return session.getSystemProperty(PUSH_AGGREGATION_THROUGH_JOIN, Boolean.class); } + public static boolean isPushSemiJoinThroughUnion(Session session) + { + return session.getSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, Boolean.class); + } + + public static boolean isSimplifyCoalesceOverJoinKeys(Session session) + { + return session.getSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, Boolean.class); + } + + public static boolean isPushdownThroughUnnest(Session session) + { + return session.getSystemProperty(PUSHDOWN_THROUGH_UNNEST, Boolean.class); + } + public static boolean isNativeExecutionEnabled(Session session) { return session.getSystemProperty(NATIVE_EXECUTION_ENABLED, Boolean.class); @@ -2575,6 +2673,11 @@ public static boolean isPushAggregationThroughJoin(Session session) return session.getSystemProperty(PUSH_PARTIAL_AGGREGATION_THROUGH_JOIN, Boolean.class); } + public static boolean isSimplifyAggregationsOverConstant(Session session) + { + return session.getSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, Boolean.class); + } + public static boolean isParseDecimalLiteralsAsDouble(Session session) { return session.getSystemProperty(PARSE_DECIMAL_LITERALS_AS_DOUBLE, Boolean.class); @@ -2657,6 +2760,11 @@ public static boolean isOptimizeTopNRowNumber(Session session) return session.getSystemProperty(OPTIMIZE_TOP_N_ROW_NUMBER, Boolean.class); } + public static boolean isOptimizeTopNRank(Session session) + { + return session.getSystemProperty(OPTIMIZE_TOP_N_RANK, Boolean.class); + } + public static boolean isOptimizeCaseExpressionPredicate(Session session) { return session.getSystemProperty(OPTIMIZE_CASE_EXPRESSION_PREDICATE, Boolean.class); @@ -2823,6 +2931,11 @@ public static AggregationPartitioningMergingStrategy getAggregationPartitioningM return session.getSystemProperty(AGGREGATION_PARTITIONING_MERGING_STRATEGY, AggregationPartitioningMergingStrategy.class); } + public static LocalExchangeParentPreferenceStrategy getLocalExchangeParentPreferenceStrategy(Session session) + { + return session.getSystemProperty(LOCAL_EXCHANGE_PARENT_PREFERENCE_STRATEGY, LocalExchangeParentPreferenceStrategy.class); + } + public static boolean isListBuiltInFunctionsOnly(Session session) { return session.getSystemProperty(LIST_BUILT_IN_FUNCTIONS_ONLY, Boolean.class); @@ -3017,6 +3130,16 @@ public static MaterializedViewStaleReadBehavior getMaterializedViewStaleReadBeha return session.getSystemProperty(MATERIALIZED_VIEW_STALE_READ_BEHAVIOR, MaterializedViewStaleReadBehavior.class); } + public static Optional getMaterializedViewStalenessWindow(Session session) + { + return Optional.ofNullable(session.getSystemProperty(MATERIALIZED_VIEW_STALENESS_WINDOW, Duration.class)); + } + + public static boolean isMaterializedViewForceStale(Session session) + { + return session.getSystemProperty(MATERIALIZED_VIEW_FORCE_STALE, Boolean.class); + } + public static boolean isVerboseRuntimeStatsEnabled(Session session) { return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class); @@ -3520,6 +3643,11 @@ public static boolean isPushSubfieldsForMapFunctionsEnabled(Session session) return session.getSystemProperty(PUSHDOWN_SUBFIELDS_FOR_MAP_FUNCTIONS, Boolean.class); } + public static boolean isPushSubfieldsForCardinalityEnabled(Session session) + { + return session.getSystemProperty(PUSHDOWN_SUBFIELDS_FOR_CARDINALITY, Boolean.class); + } + public static boolean isUtilizeUniquePropertyInQueryPlanningEnabled(Session session) { return session.getSystemProperty(UTILIZE_UNIQUE_PROPERTY_IN_QUERY_PLANNING, Boolean.class); @@ -3564,4 +3692,19 @@ public static boolean isSkipPushdownThroughExchangeForRemoteProjection(Session s { return session.getSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, Boolean.class); } + + public static String getRemoteFunctionNamesForFixedParallelism(Session session) + { + return session.getSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, String.class); + } + + public static int getRemoteFunctionFixedParallelismTaskCount(Session session) + { + return session.getSystemProperty(REMOTE_FUNCTION_FIXED_PARALLELISM_TASK_COUNT, Integer.class); + } + + public static String getTryFunctionCatchableErrors(Session session) + { + return session.getSystemProperty(TRY_FUNCTION_CATCHABLE_ERRORS, String.class); + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java b/presto-main-base/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java index c11fedce9c259..9962d12fef241 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/AggregationStatsRule.java @@ -20,12 +20,14 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.iterative.Lookup; +import com.google.common.collect.ImmutableMap; import java.util.Collection; import java.util.Map; import java.util.Optional; -import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.INTERMEDIATE; +import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.statistics.SourceInfo.ConfidenceLevel.FACT; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; import static java.lang.Math.min; @@ -54,49 +56,81 @@ protected Optional doCalculate(AggregationNode node, Stat return Optional.empty(); } - if (node.getStep() != SINGLE) { - return Optional.empty(); - } + PlanNodeStatsEstimate estimate; - return Optional.of(groupBy( - statsProvider.getStats(node.getSource()), - node.getGroupingKeys(), - node.getAggregations())); + if (node.getStep() == PARTIAL || node.getStep() == INTERMEDIATE) { + estimate = partialGroupBy( + statsProvider.getStats(node.getSource()), + node.getGroupingKeys(), + node.getAggregations()); + } + else { + estimate = groupBy( + statsProvider.getStats(node.getSource()), + node.getGroupingKeys(), + node.getAggregations()); + } + return Optional.of(estimate); } public static PlanNodeStatsEstimate groupBy(PlanNodeStatsEstimate sourceStats, Collection groupByVariables, Map aggregations) { + // Used to estimate FINAL or SINGLE step aggregations PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.builder(); - - if (isGlobalAggregation(groupByVariables)) { + if (groupByVariables.isEmpty()) { result.setConfidence(FACT); + result.setOutputRowCount(1); } - - for (VariableReferenceExpression groupByVariable : groupByVariables) { - VariableStatsEstimate symbolStatistics = sourceStats.getVariableStatistics(groupByVariable); - result.addVariableStatistics(groupByVariable, symbolStatistics.mapNullsFraction(nullsFraction -> { - if (nullsFraction == 0.0) { - return 0.0; - } - return 1.0 / (symbolStatistics.getDistinctValuesCount() + 1); - })); + else { + result.addVariableStatistics(getGroupByVariablesStatistics(sourceStats, groupByVariables)); + double rowsCount = getRowsCount(sourceStats, groupByVariables); + result.setOutputRowCount(min(rowsCount, sourceStats.getOutputRowCount())); } + aggregations.forEach((key, value) -> result.addVariableStatistics(key, estimateAggregationStats(value, sourceStats))); + + return result.build(); + } + + public static double getRowsCount(PlanNodeStatsEstimate sourceStats, Collection groupByVariables) + { double rowsCount = 1; for (VariableReferenceExpression groupByVariable : groupByVariables) { VariableStatsEstimate symbolStatistics = sourceStats.getVariableStatistics(groupByVariable); int nullRow = (symbolStatistics.getNullsFraction() == 0.0) ? 0 : 1; rowsCount *= symbolStatistics.getDistinctValuesCount() + nullRow; } - result.setOutputRowCount(min(rowsCount, sourceStats.getOutputRowCount())); + return rowsCount; + } - for (Map.Entry aggregationEntry : aggregations.entrySet()) { - result.addVariableStatistics(aggregationEntry.getKey(), estimateAggregationStats(aggregationEntry.getValue(), sourceStats)); - } + private static PlanNodeStatsEstimate partialGroupBy(PlanNodeStatsEstimate sourceStats, Collection groupByVariables, Map aggregations) + { + // Pessimistic assumption of no reduction from PARTIAL and INTERMEDIATE aggregation, forwarding of the source statistics. + // This makes the CBO estimates in the EXPLAIN plan output easier to understand, + // even though partial aggregations are added after the CBO rules have been run. + PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.builder(); + result.setOutputRowCount(sourceStats.getOutputRowCount()); + result.addVariableStatistics(getGroupByVariablesStatistics(sourceStats, groupByVariables)); + aggregations.forEach((key, value) -> result.addVariableStatistics(key, estimateAggregationStats(value, sourceStats))); return result.build(); } + private static Map getGroupByVariablesStatistics(PlanNodeStatsEstimate sourceStats, Collection groupByVariables) + { + ImmutableMap.Builder variableStatsEstimates = ImmutableMap.builder(); + for (VariableReferenceExpression groupByVariable : groupByVariables) { + VariableStatsEstimate symbolStatistics = sourceStats.getVariableStatistics(groupByVariable); + variableStatsEstimates.put(groupByVariable, symbolStatistics.mapNullsFraction(nullsFraction -> { + if (nullsFraction == 0.0) { + return 0.0; + } + return 1.0 / (symbolStatistics.getDistinctValuesCount() + 1); + })); + } + return variableStatsEstimates.build(); + } + private static VariableStatsEstimate estimateAggregationStats(Aggregation aggregation, PlanNodeStatsEstimate sourceStats) { requireNonNull(aggregation, "aggregation is null"); @@ -105,9 +139,4 @@ private static VariableStatsEstimate estimateAggregationStats(Aggregation aggreg // TODO implement simple aggregations like: min, max, count, sum return VariableStatsEstimate.unknown(); } - - private static boolean isGlobalAggregation(Collection groupingKeys) - { - return groupingKeys.isEmpty(); - } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java b/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java index cba986d84bcff..fc04b944aaebc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/ComparisonStatsCalculator.java @@ -105,14 +105,26 @@ private PlanNodeStatsEstimate estimateExpressionNotEqualToLiteral( else { filterRange = new StatisticRange(NEGATIVE_INFINITY, true, POSITIVE_INFINITY, true, 1); } - double filterFactor = 1 - calculateFilterFactor(expressionStatistics, filterRange); + + double filterFactor; + double expressionNDV = expressionStatistics.getDistinctValuesCount(); + if (Double.compare(expressionNDV, 1D) == 0) { + // It's hard to make a meaningful estimate when we have only one distinct value + filterFactor = UNKNOWN_FILTER_COEFFICIENT; + } + else { + filterFactor = 1 - calculateFilterFactor(expressionStatistics, filterRange); + } PlanNodeStatsEstimate.Builder estimate = PlanNodeStatsEstimate.buildFrom(inputStatistics); estimate.setOutputRowCount(filterFactor * (1 - expressionStatistics.getNullsFraction()) * inputStatistics.getOutputRowCount()); if (expressionVariable.isPresent()) { + // If the original NDV was 1, we do not make any changes to the new estimate, since we're not sure if we eliminated the only distinct value + // Otherwise, we reduce the NDV by 1 (unless it was already 0) + double newNDV = Double.compare(expressionNDV, 1D) == 0 ? 1 : max(expressionNDV - 1, 0); VariableStatsEstimate symbolNewEstimate = buildFrom(expressionStatistics) .setNullsFraction(0.0) - .setDistinctValuesCount(max(expressionStatistics.getDistinctValuesCount() - 1, 0)) + .setDistinctValuesCount(newNDV) .build(); estimate = estimate.addVariableStatistics(expressionVariable.get(), symbolNewEstimate); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java b/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java index d01c5b6ba68f2..37fb1abba807f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/cost/FilterStatsCalculator.java @@ -91,6 +91,7 @@ import static java.lang.Double.NaN; import static java.lang.Double.isInfinite; import static java.lang.Double.isNaN; +import static java.lang.Double.max; import static java.lang.Double.min; import static java.lang.String.format; import static java.util.Collections.emptyMap; @@ -98,6 +99,13 @@ public class FilterStatsCalculator { + /** + * + * This value applies a filter factor to upper-bound the size of the variable range selected for an IN predicate + * Since the estimator sums up the individual estimates, we don't want to go beyond 1.0 + * This also impacts NOT IN similarly, we never apply a filter factor of 0.0 for a NOT IN clause + */ + static final double CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT = 0.8; static final double UNKNOWN_FILTER_COEFFICIENT = 0.9; private final Metadata metadata; @@ -403,9 +411,11 @@ protected PlanNodeStatsEstimate visitInPredicate(InPredicate node, Void context) } double notNullValuesBeforeIn = input.getOutputRowCount() * (1 - valueStats.getNullsFraction()); + double ceiledInEstimated = max(notNullValuesBeforeIn * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT, 1.0); + double inEstimateRowCount = min(inEstimate.getOutputRowCount(), ceiledInEstimated); PlanNodeStatsEstimate.Builder result = PlanNodeStatsEstimate.buildFrom(input); - result.setOutputRowCount(min(inEstimate.getOutputRowCount(), notNullValuesBeforeIn)); + result.setOutputRowCount(inEstimateRowCount); if (node.getValue() instanceof SymbolReference) { VariableReferenceExpression valueVariable = toVariable(node.getValue()); @@ -774,9 +784,11 @@ private PlanNodeStatsEstimate estimateIn(RowExpression value, List, Type> getExpressionTypes(Session session, Expression expression, TypeProvider types) @@ -555,6 +565,13 @@ protected VariableStatsEstimate visitCoalesceExpression(CoalesceExpression node, } } + private static void setConstantSizeEstimate(Object value, VariableStatsEstimate.Builder statsEstimate) + { + if (value instanceof Slice) { + statsEstimate.setAverageRowSize(((Slice) value).length()); + } + } + private static VariableStatsEstimate estimateCoalesce(PlanNodeStatsEstimate input, VariableStatsEstimate left, VariableStatsEstimate right) { // Question to reviewer: do you have a method to check if fraction is empty or saturated? diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java index 5649c893c2cff..3ab62ab71b85f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/AccessControlCheckerExecution.java @@ -271,7 +271,7 @@ private ListenableFuture executeTask() } stateMachine.beginColumnAccessPermissionChecking(); - checkAccessPermissions(queryAnalysis.getAccessControlReferences(), query, getSession().getPreparedStatements()); + checkAccessPermissions(queryAnalysis.getAccessControlReferences(), queryAnalysis.getViewDefinitionReferences(), query, getSession().getPreparedStatements(), getSession().getIdentity(), accessControl, getSession().getAccessControlContext()); stateMachine.endColumnAccessPermissionChecking(); return immediateFuture(null); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/AlterFunctionTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/AlterFunctionTask.java index 1e3592edc089c..57442b4fca918 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/AlterFunctionTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/AlterFunctionTask.java @@ -17,10 +17,16 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.AlterRoutineCharacteristics; import com.facebook.presto.spi.function.RoutineCharacteristics.NullCallClause; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.AlterFunction; @@ -36,6 +42,7 @@ import java.util.Optional; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.util.Objects.requireNonNull; @@ -67,8 +74,9 @@ public String explain(AlterFunction statement, List parameters) public ListenableFuture execute(AlterFunction statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List parameters, WarningCollector warningCollector, String query) { Map, Expression> parameterLookup = parameterExtractor(statement, parameters); - Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, warningCollector, query); - analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, warningCollector, query, new ViewDefinitionReferences()); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); QualifiedObjectName functionName = metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver().qualifyObjectName(statement.getFunctionName()); AlterRoutineCharacteristics alterRoutineCharacteristics = new AlterRoutineCharacteristics( @@ -83,4 +91,7 @@ public ListenableFuture execute(AlterFunction statement, TransactionManager t alterRoutineCharacteristics); return immediateFuture(null); } + + @Override + public void queryPermissionCheck(AccessControl accessControl, Identity identity, AccessControlContext context, String query, Map preparedStatements, Map viewDefinitions, Map materializedViewDefinitions) {} } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateBranchTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateBranchTask.java new file mode 100644 index 0000000000000..e15ae892a922e --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateBranchTask.java @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.TimestampType; +import com.facebook.presto.common.type.TimestampWithTimeZoneType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.connector.ConnectorTableVersion; +import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; +import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.analyzer.SemanticException; +import com.facebook.presto.sql.tree.CreateBranch; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.sql.tree.Parameter; +import com.facebook.presto.sql.tree.TableVersionExpression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; +import static com.facebook.presto.metadata.MetadataUtil.getConnectorIdOrThrow; +import static com.facebook.presto.spi.connector.ConnectorTableVersion.VersionOperator; +import static com.facebook.presto.spi.connector.ConnectorTableVersion.VersionType; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH; +import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.sql.planner.ExpressionInterpreter.evaluateConstantExpression; +import static com.facebook.presto.sql.tree.TableVersionExpression.TableVersionType.TIMESTAMP; +import static com.facebook.presto.sql.tree.TableVersionExpression.TableVersionType.VERSION; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +public class CreateBranchTask + implements DDLDefinitionTask +{ + @Override + public String getName() + { + return "CREATE BRANCH"; + } + + @Override + public ListenableFuture execute(CreateBranch statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List parameters, WarningCollector warningCollector, String query) + { + QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName(), metadata); + Optional tableHandleOptional = metadata.getMetadataResolver(session).getTableHandle(tableName); + + if (statement.isTableExists() && !tableHandleOptional.isPresent()) { + return immediateFuture(null); + } + + Optional optionalMaterializedView = metadata.getMetadataResolver(session).getMaterializedView(tableName); + if (optionalMaterializedView.isPresent()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and create branch is not supported", tableName); + } + + getConnectorIdOrThrow(session, metadata, tableName.getCatalogName()); + accessControl.checkCanCreateBranch(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName); + + if (statement.isReplace() && statement.isIfNotExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, + "Cannot specify both OR REPLACE and IF NOT EXISTS in CREATE BRANCH statement"); + } + + Optional tableVersion = Optional.empty(); + + if (statement.getTableVersion().isPresent()) { + TableVersionExpression tableVersionExpr = statement.getTableVersion().get(); + Expression stateExpr = tableVersionExpr.getStateExpression(); + TableVersionExpression.TableVersionType tableVersionType = tableVersionExpr.getTableVersionType(); + TableVersionExpression.TableVersionOperator tableVersionOperator = tableVersionExpr.getTableVersionOperator(); + + Map, Expression> parameterLookup = parameterExtractor(statement, parameters); + + ExpressionAnalyzer analyzer = ExpressionAnalyzer.createConstantAnalyzer( + metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver(), + session, + parameterLookup, + WarningCollector.NOOP); + analyzer.analyze(stateExpr, Scope.create()); + Type stateExprType = analyzer.getExpressionTypes().get(NodeRef.of(stateExpr)); + + if (tableVersionType == TIMESTAMP) { + if (!(stateExprType instanceof TimestampWithTimeZoneType || stateExprType instanceof TimestampType)) { + throw new SemanticException(TYPE_MISMATCH, stateExpr, + "Type %s is invalid. Supported table version AS OF/BEFORE expression type is Timestamp or Timestamp with Time Zone.", + stateExprType.getDisplayName()); + } + } + else if (tableVersionType == VERSION) { + if (!(stateExprType instanceof BigintType || stateExprType instanceof VarcharType)) { + throw new SemanticException(TYPE_MISMATCH, stateExpr, "Type %s is invalid. Supported table version AS OF/BEFORE expression type is BIGINT or VARCHAR", stateExprType.getDisplayName()); + } + } + + Object evalStateExpr = evaluateConstantExpression(stateExpr, stateExprType, metadata, session, parameterLookup); + VersionType versionType = tableVersionType == TIMESTAMP ? VersionType.TIMESTAMP : VersionType.VERSION; + VersionOperator versionOperator = tableVersionOperator == TableVersionExpression.TableVersionOperator.EQUAL + ? VersionOperator.EQUAL : VersionOperator.LESS_THAN; + + tableVersion = Optional.of(new ConnectorTableVersion(versionType, versionOperator, stateExprType, evalStateExpr)); + } + + metadata.createBranch( + session, + tableHandleOptional.get(), + statement.getBranchName(), + statement.isReplace(), + statement.isIfNotExists(), + tableVersion, + statement.getRetainDays(), + statement.getMinSnapshotsToKeep(), + statement.getMaxSnapshotAgeDays()); + + return immediateFuture(null); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java index 8ae5e5670f80c..779d74f06ea33 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateFunctionTask.java @@ -18,13 +18,18 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.Parameter; import com.facebook.presto.spi.function.RoutineCharacteristics; import com.facebook.presto.spi.function.SqlFunctionHandle; import com.facebook.presto.spi.function.SqlFunctionId; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.parser.SqlParser; @@ -50,6 +55,7 @@ import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; import static com.facebook.presto.sql.SqlFormatter.formatSql; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.lang.String.format; @@ -83,8 +89,10 @@ public ListenableFuture execute(CreateFunction statement, TransactionManager { Map, Expression> parameterLookup = parameterExtractor(statement, parameters); Session session = stateMachine.getSession(); - Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, stateMachine.getWarningCollector(), query); - Analysis analysis = analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, stateMachine.getWarningCollector(), query, new ViewDefinitionReferences()); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + if (analysis.getFunctionHandles().values().stream() .anyMatch(SqlFunctionHandle.class::isInstance)) { throw new PrestoException(NOT_SUPPORTED, "Invoking a dynamically registered function in SQL function body is not supported"); @@ -101,6 +109,9 @@ public ListenableFuture execute(CreateFunction statement, TransactionManager return immediateFuture(null); } + @Override + public void queryPermissionCheck(AccessControl accessControl, Identity identity, AccessControlContext context, String query, Map preparedStatements, Map viewDefinitions, Map materializedViewDefinitions) {} + private SqlInvokedFunction createSqlInvokedFunction(CreateFunction statement, Metadata metadata, Analysis analysis) { QualifiedObjectName functionName = statement.isTemporary() ? diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java index 7b146cfb0415b..99ca276ae6813 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateMaterializedViewTask.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.spi.security.ViewSecurity; import com.facebook.presto.sql.analyzer.Analysis; @@ -53,6 +54,7 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MATERIALIZED_VIEW_ALREADY_EXISTS; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.util.Objects.requireNonNull; @@ -91,8 +93,9 @@ public ListenableFuture execute(CreateMaterializedView statement, Transaction accessControl.checkCanCreateView(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), viewName); Map, Expression> parameterLookup = parameterExtractor(statement, parameters); - Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, warningCollector, query); - Analysis analysis = analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, warningCollector, query, new ViewDefinitionReferences()); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); List columnMetadata = analysis.getOutputDescriptor(statement.getQuery()) .getVisibleFields().stream() diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTableTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTableTask.java index 306bbde275f6e..a41e4a64f8620 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTableTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTableTask.java @@ -87,6 +87,9 @@ public String getName() @Override public String explain(CreateTable statement, List parameters) { + if (statement.isNotExists()) { + return "CREATE TABLE IF NOT EXISTS " + statement.getName(); + } return "CREATE TABLE " + statement.getName(); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTagTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTagTask.java new file mode 100644 index 0000000000000..d7adab88527db --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateTagTask.java @@ -0,0 +1,138 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.TimestampType; +import com.facebook.presto.common.type.TimestampWithTimeZoneType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.connector.ConnectorTableVersion; +import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.sql.analyzer.ExpressionAnalyzer; +import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.analyzer.SemanticException; +import com.facebook.presto.sql.tree.CreateTag; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.sql.tree.Parameter; +import com.facebook.presto.sql.tree.TableVersionExpression; +import com.facebook.presto.transaction.TransactionManager; +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; +import static com.facebook.presto.metadata.MetadataUtil.getConnectorIdOrThrow; +import static com.facebook.presto.spi.connector.ConnectorTableVersion.VersionOperator; +import static com.facebook.presto.spi.connector.ConnectorTableVersion.VersionType; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH; +import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.sql.planner.ExpressionInterpreter.evaluateConstantExpression; +import static com.facebook.presto.sql.tree.TableVersionExpression.TableVersionType.TIMESTAMP; +import static com.facebook.presto.sql.tree.TableVersionExpression.TableVersionType.VERSION; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +public class CreateTagTask + implements DDLDefinitionTask +{ + @Override + public String getName() + { + return "CREATE TAG"; + } + + @Override + public ListenableFuture execute(CreateTag statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, Session session, List parameters, WarningCollector warningCollector, String query) + { + QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName(), metadata); + Optional tableHandleOptional = metadata.getMetadataResolver(session).getTableHandle(tableName); + + if (statement.isTableExists() && !tableHandleOptional.isPresent()) { + return immediateFuture(null); + } + + Optional optionalMaterializedView = metadata.getMetadataResolver(session).getMaterializedView(tableName); + if (optionalMaterializedView.isPresent()) { + throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is a materialized view, and create tag is not supported", tableName); + } + + getConnectorIdOrThrow(session, metadata, tableName.getCatalogName()); + accessControl.checkCanCreateTag(session.getRequiredTransactionId(), session.getIdentity(), session.getAccessControlContext(), tableName); + + if (statement.isReplace() && statement.isIfNotExists()) { + throw new SemanticException(NOT_SUPPORTED, statement, + "Cannot specify both OR REPLACE and IF NOT EXISTS in CREATE TAG statement"); + } + + Optional tableVersion = Optional.empty(); + + if (statement.getTableVersion().isPresent()) { + TableVersionExpression tableVersionExpr = statement.getTableVersion().get(); + Expression stateExpr = tableVersionExpr.getStateExpression(); + TableVersionExpression.TableVersionType tableVersionType = tableVersionExpr.getTableVersionType(); + TableVersionExpression.TableVersionOperator tableVersionOperator = tableVersionExpr.getTableVersionOperator(); + + Map, Expression> parameterLookup = parameterExtractor(statement, parameters); + + ExpressionAnalyzer analyzer = ExpressionAnalyzer.createConstantAnalyzer( + metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver(), + session, + parameterLookup, + WarningCollector.NOOP); + analyzer.analyze(stateExpr, Scope.create()); + Type stateExprType = analyzer.getExpressionTypes().get(NodeRef.of(stateExpr)); + + if (tableVersionType == TIMESTAMP) { + if (!(stateExprType instanceof TimestampWithTimeZoneType || stateExprType instanceof TimestampType)) { + throw new SemanticException(TYPE_MISMATCH, stateExpr, + "Type %s is invalid. Supported table version AS OF/BEFORE expression type is Timestamp or Timestamp with Time Zone.", + stateExprType.getDisplayName()); + } + } + else if (tableVersionType == VERSION) { + if (!(stateExprType instanceof BigintType || stateExprType instanceof VarcharType)) { + throw new SemanticException(TYPE_MISMATCH, stateExpr, "Type %s is invalid. Supported table version AS OF/BEFORE expression type is BIGINT or VARCHAR", stateExprType.getDisplayName()); + } + } + + Object evalStateExpr = evaluateConstantExpression(stateExpr, stateExprType, metadata, session, parameterLookup); + VersionType versionType = tableVersionType == TIMESTAMP ? VersionType.TIMESTAMP : VersionType.VERSION; + VersionOperator versionOperator = tableVersionOperator == TableVersionExpression.TableVersionOperator.EQUAL + ? VersionOperator.EQUAL : VersionOperator.LESS_THAN; + + tableVersion = Optional.of(new ConnectorTableVersion(versionType, versionOperator, stateExprType, evalStateExpr)); + } + + metadata.createTag( + session, + tableHandleOptional.get(), + statement.getTagName(), + statement.isReplace(), + statement.isIfNotExists(), + tableVersion, + statement.getRetainDays()); + + return immediateFuture(null); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateViewTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateViewTask.java index 82e4c69e2a1ff..594854dba08e1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/CreateViewTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/CreateViewTask.java @@ -19,9 +19,13 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.ViewSecurity; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; @@ -35,6 +39,7 @@ import jakarta.inject.Inject; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.getDefaultViewSecurityMode; @@ -44,6 +49,7 @@ import static com.facebook.presto.spi.security.ViewSecurity.INVOKER; import static com.facebook.presto.sql.SqlFormatterUtil.getFormattedSql; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.util.Objects.requireNonNull; @@ -115,9 +121,15 @@ public ListenableFuture execute(CreateView statement, TransactionManager tran return immediateFuture(null); } + @Override + public void queryPermissionCheck(AccessControl accessControl, Identity identity, AccessControlContext context, String query, Map preparedStatements, Map viewDefinitions, Map materializedViewDefinitions) {} + private Analysis analyzeStatement(Statement statement, Session session, Metadata metadata, AccessControl accessControl, List parameters, WarningCollector warningCollector, String query) { - Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterExtractor(statement, parameters), warningCollector, query); - return analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterExtractor(statement, parameters), warningCollector, query, new ViewDefinitionReferences()); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + + return analysis; } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java index aacc06d983562..8fb65531f65fe 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java @@ -62,8 +62,7 @@ private DDLDefinitionExecution( @Override protected ListenableFuture executeTask() { - accessControl.checkQueryIntegrity(stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, stateMachine.getSession().getPreparedStatements(), ImmutableMap.of(), ImmutableMap.of()); - + task.queryPermissionCheck(accessControl, stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, stateMachine.getSession().getPreparedStatements(), ImmutableMap.of(), ImmutableMap.of()); return task.execute(statement, transactionManager, metadata, accessControl, stateMachine.getSession(), parameters, stateMachine.getWarningCollector(), query); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/DataDefinitionTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/DataDefinitionTask.java index a8b6aea7b7f08..ed806bfcf2959 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/DataDefinitionTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/DataDefinitionTask.java @@ -13,12 +13,19 @@ */ package com.facebook.presto.execution; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; import com.facebook.presto.sql.SqlFormatter; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Prepare; import com.facebook.presto.sql.tree.Statement; import java.util.List; +import java.util.Map; import java.util.Optional; public interface DataDefinitionTask @@ -33,4 +40,9 @@ default String explain(T statement, List parameters) return SqlFormatter.formatSql(statement, Optional.of(parameters)); } + + default void queryPermissionCheck(AccessControl accessControl, Identity identity, AccessControlContext context, String query, Map preparedStatements, Map viewDefinitions, Map materializedViewDefinitions) + { + accessControl.checkQueryIntegrity(identity, context, query, preparedStatements, viewDefinitions, materializedViewDefinitions); + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/DropFunctionTask.java b/presto-main-base/src/main/java/com/facebook/presto/execution/DropFunctionTask.java index e898c80dde890..ed5d4224e9286 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/DropFunctionTask.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/DropFunctionTask.java @@ -16,8 +16,14 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.SqlFunctionId; import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.DropFunction; @@ -34,6 +40,7 @@ import static com.facebook.presto.metadata.SessionFunctionHandle.SESSION_NAMESPACE; import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.lang.String.format; @@ -67,8 +74,10 @@ public String explain(DropFunction statement, List parameters) public ListenableFuture execute(DropFunction statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl, QueryStateMachine stateMachine, List parameters, String query) { Map, Expression> parameterLookup = parameterExtractor(statement, parameters); - Analyzer analyzer = new Analyzer(stateMachine.getSession(), metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, stateMachine.getWarningCollector(), query); - analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(stateMachine.getSession(), metadata, sqlParser, accessControl, Optional.empty(), parameters, parameterLookup, stateMachine.getWarningCollector(), query, new ViewDefinitionReferences()); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, stateMachine.getSession().getPreparedStatements(), stateMachine.getSession().getIdentity(), accessControl, stateMachine.getSession().getAccessControlContext()); + Optional> parameterTypes = statement.getParameterTypes().map(types -> types.stream().map(TypeSignature::parseTypeSignature).collect(toImmutableList())); if (statement.isTemporary()) { @@ -87,4 +96,7 @@ public ListenableFuture execute(DropFunction statement, TransactionManager tr return immediateFuture(null); } + + @Override + public void queryPermissionCheck(AccessControl accessControl, Identity identity, AccessControlContext context, String query, Map preparedStatements, Map viewDefinitions, Map materializedViewDefinitions) {} } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java b/presto-main-base/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java index 8eb4ced151805..c3c6547629097 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/QueryManagerConfig.java @@ -102,6 +102,10 @@ public class QueryManagerConfig private int minColumnarEncodingChannelsToPreferRowWiseEncoding = 1000; + private int maxQueryAdmissionsPerSecond = Integer.MAX_VALUE; + + private int minRunningQueriesForPacing = 30; + @Min(1) public int getScheduleSplitBatchSize() { @@ -317,7 +321,8 @@ public int getMaxQueryRunningTaskCount() return maxQueryRunningTaskCount; } - @Config("experimental.max-total-running-task-count-to-not-execute-new-query") + @LegacyConfig("experimental.max-total-running-task-count-to-not-execute-new-query") + @Config("max-total-running-task-count-to-not-execute-new-query") @ConfigDescription("Keep new queries in the queue if total task count exceeds this threshold") public QueryManagerConfig setMaxTotalRunningTaskCountToNotExecuteNewQuery(int maxTotalRunningTaskCountToNotExecuteNewQuery) { @@ -766,6 +771,34 @@ public QueryManagerConfig setMinColumnarEncodingChannelsToPreferRowWiseEncoding( return this; } + @Min(1) + public int getMaxQueryAdmissionsPerSecond() + { + return maxQueryAdmissionsPerSecond; + } + + @Config("query-manager.query-pacing.max-queries-per-second") + @ConfigDescription("Maximum number of queries that can be admitted per second globally for admission pacing. Default is unlimited (Integer.MAX_VALUE). Set to a lower value (e.g., 1) to pace query admissions to one per second.") + public QueryManagerConfig setMaxQueryAdmissionsPerSecond(int maxQueryAdmissionsPerSecond) + { + this.maxQueryAdmissionsPerSecond = maxQueryAdmissionsPerSecond; + return this; + } + + @Min(0) + public int getMinRunningQueriesForPacing() + { + return minRunningQueriesForPacing; + } + + @Config("query-manager.query-pacing.min-running-queries") + @ConfigDescription("Minimum number of running queries before admission pacing is applied. Default is 30. Set to a higher value to only pace when cluster is busy.") + public QueryManagerConfig setMinRunningQueriesForPacing(int minRunningQueriesForPacing) + { + this.minRunningQueriesForPacing = minRunningQueriesForPacing; + return this; + } + public enum ExchangeMaterializationStrategy { NONE, diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/SafeEventLoopGroup.java b/presto-main-base/src/main/java/com/facebook/presto/execution/SafeEventLoopGroup.java index bca2888c6b395..1bb157db5910e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/SafeEventLoopGroup.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/SafeEventLoopGroup.java @@ -52,6 +52,7 @@ public SafeEventLoopGroup(int nThreads, ThreadFactory threadFactory, long slowMe @Override protected EventLoop newChild(Executor executor, Object... args) + throws Exception { return new SafeEventLoop(this, executor); } @@ -71,7 +72,7 @@ protected void run() Runnable task = takeTask(); if (task != null) { try { - runTask(task); + task.run(); } catch (Throwable t) { log.error(t, "Error executing task on event loop"); diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java index a30ecc62b8673..f0c88ab90a6eb 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java @@ -63,8 +63,7 @@ private SessionDefinitionExecution( @Override protected ListenableFuture executeTask() { - accessControl.checkQueryIntegrity(stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, stateMachine.getSession().getPreparedStatements(), ImmutableMap.of(), ImmutableMap.of()); - + task.queryPermissionCheck(accessControl, stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, stateMachine.getSession().getPreparedStatements(), ImmutableMap.of(), ImmutableMap.of()); return task.execute(statement, transactionManager, metadata, accessControl, stateMachine, parameters, query); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index b403ad33a45e7..dfbfc86ae0f54 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -223,7 +223,7 @@ private SqlQueryExecution( stateMachine.setExpandedQuery(queryAnalysis.getExpandedQuery()); stateMachine.beginColumnAccessPermissionChecking(); - checkAccessPermissions(queryAnalysis.getAccessControlReferences(), query, getSession().getPreparedStatements()); + checkAccessPermissions(queryAnalysis.getAccessControlReferences(), queryAnalysis.getViewDefinitionReferences(), query, getSession().getPreparedStatements(), getSession().getIdentity(), accessControl, getSession().getAccessControlContext()); stateMachine.endColumnAccessPermissionChecking(); // when the query finishes cache the final query info, and clear the reference to the output stage diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java index 82274c04b3550..b13edb21603e9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroup.java @@ -98,6 +98,7 @@ public class InternalResourceGroup private final Predicate shouldWaitForResourceManagerUpdate; private final InternalNodeManager nodeManager; private final ClusterResourceChecker clusterResourceChecker; + private final QueryPacingContext queryPacingContext; // Configuration // ============= @@ -169,13 +170,15 @@ protected InternalResourceGroup( Function> additionalRuntimeInfo, Predicate shouldWaitForResourceManagerUpdate, InternalNodeManager nodeManager, - ClusterResourceChecker clusterResourceChecker) + ClusterResourceChecker clusterResourceChecker, + QueryPacingContext queryPacingContext) { this.parent = requireNonNull(parent, "parent is null"); this.jmxExportListener = requireNonNull(jmxExportListener, "jmxExportListener is null"); this.executor = requireNonNull(executor, "executor is null"); this.nodeManager = requireNonNull(nodeManager, "node manager is null"); this.clusterResourceChecker = requireNonNull(clusterResourceChecker, "clusterResourceChecker is null"); + this.queryPacingContext = requireNonNull(queryPacingContext, "queryPacingContext is null"); requireNonNull(name, "name is null"); if (parent.isPresent()) { id = new ResourceGroupId(parent.get().id, name); @@ -676,7 +679,8 @@ public InternalResourceGroup getOrCreateSubGroup(String name, boolean staticSegm additionalRuntimeInfo, shouldWaitForResourceManagerUpdate, nodeManager, - clusterResourceChecker); + clusterResourceChecker, + queryPacingContext); // Sub group must use query priority to ensure ordering if (schedulingPolicy == QUERY_PRIORITY) { subGroup.setSchedulingPolicy(QUERY_PRIORITY); @@ -735,12 +739,25 @@ public void run(ManagedQueryExecution query) } else { query.setResourceGroupQueryLimits(perQueryLimits); - if (canRun && queuedQueries.isEmpty()) { + boolean immediateStartCandidate = canRun && queuedQueries.isEmpty(); + boolean startQuery = immediateStartCandidate; + if (immediateStartCandidate) { + // Check for coordinator overload (task limit exceeded or denied admission) + //isTaskLimitExceeded MUST be checked before tryAcquireAdmissionSlot, or else admission slots will be acquired but not started + boolean coordOverloaded = ((RootInternalResourceGroup) root).isTaskLimitExceeded() + || !queryPacingContext.tryAcquireAdmissionSlot(); + if (coordOverloaded) { + startQuery = false; + } + } + + if (startQuery) { startInBackground(query); } else { enqueueQuery(query); } + query.addStateChangeListener(state -> { if (state.isDone()) { queryFinished(query); @@ -807,6 +824,8 @@ private void startInBackground(ManagedQueryExecution query) group = group.parent.get(); } updateEligibility(); + // Increment global running query counter for pacing + queryPacingContext.onQueryStarted(); executor.execute(query::startWaitingForResources); group = this; long lastRunningQueryStartTimeMillis = currentTimeMillis(); @@ -840,6 +859,8 @@ private void queryFinished(ManagedQueryExecution query) group.parent.get().descendantRunningQueries--; group = group.parent.get(); } + // Decrement global running query counter for pacing + queryPacingContext.onQueryFinished(); } else { queuedQueries.remove(query); @@ -904,12 +925,21 @@ protected boolean internalStartNext() { checkState(Thread.holdsLock(root), "Must hold lock to find next query"); synchronized (root) { + if (((RootInternalResourceGroup) root).isTaskLimitExceeded()) { + return false; + } + if (!canRunMore()) { return false; } - ManagedQueryExecution query = queuedQueries.poll(); + ManagedQueryExecution query = queuedQueries.peek(); if (query != null) { + if (!queryPacingContext.tryAcquireAdmissionSlot()) { + return false; + } + + queuedQueries.poll(); // Remove from queue; use query from peek() above startInBackground(query); return true; } @@ -1037,10 +1067,6 @@ private boolean canRunMore() return false; } - if (((RootInternalResourceGroup) root).isTaskLimitExceeded()) { - return false; - } - int hardConcurrencyLimit = getHardConcurrencyLimitBasedOnCpuUsage(); int totalRunningQueries = runningQueries.size() + descendantRunningQueries; @@ -1146,7 +1172,8 @@ public RootInternalResourceGroup( Function> additionalRuntimeInfo, Predicate shouldWaitForResourceManagerUpdate, InternalNodeManager nodeManager, - ClusterResourceChecker clusterResourceChecker) + ClusterResourceChecker clusterResourceChecker, + QueryPacingContext queryPacingContext) { super(Optional.empty(), name, @@ -1156,7 +1183,8 @@ public RootInternalResourceGroup( additionalRuntimeInfo, shouldWaitForResourceManagerUpdate, nodeManager, - clusterResourceChecker); + clusterResourceChecker, + queryPacingContext); } public synchronized void updateEligibilityRecursively(InternalResourceGroup group) @@ -1172,7 +1200,7 @@ public synchronized void processQueuedQueries() internalRefreshStats(); while (internalStartNext()) { - // start all the queries we can + // start all the queries we can (subject to limits and pacing) } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java index f760cfa19b031..806e82e1e64d9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java @@ -59,6 +59,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; @@ -90,6 +91,18 @@ public final class InternalResourceGroupManager private static final String CONFIGURATION_MANAGER_PROPERTY_NAME = "resource-groups.configuration-manager"; private static final int REFRESH_EXECUTOR_POOL_SIZE = 2; + private final int maxQueryAdmissionsPerSecond; + private final int minRunningQueriesForPacing; + private final long queryAdmissionIntervalNanos; + private final AtomicLong lastAdmittedQueryNanos = new AtomicLong(0L); + + // Pacing metrics - use AtomicLong/AtomicInteger for lock-free updates to avoid deadlock + // with resource group locks (see tryAcquireAdmissionSlot for details) + private final AtomicLong totalAdmissionAttempts = new AtomicLong(0L); + private final AtomicLong totalAdmissionsGranted = new AtomicLong(0L); + private final AtomicLong totalAdmissionsDenied = new AtomicLong(0L); + private final AtomicInteger totalRunningQueriesCounter = new AtomicInteger(0); + private final ScheduledExecutorService refreshExecutor = newScheduledThreadPool(REFRESH_EXECUTOR_POOL_SIZE, daemonThreadsNamed("resource-group-manager-refresher-%d-" + REFRESH_EXECUTOR_POOL_SIZE)); private final PeriodicTaskExecutor resourceGroupRuntimeExecutor; private final List rootGroups = new CopyOnWriteArrayList<>(); @@ -115,6 +128,7 @@ public final class InternalResourceGroupManager private final InternalNodeManager nodeManager; private AtomicBoolean isConfigurationManagerLoaded; private final ClusterResourceChecker clusterResourceChecker; + private final QueryPacingContext queryPacingContext; @Inject public InternalResourceGroupManager( @@ -141,7 +155,101 @@ public InternalResourceGroupManager( this.resourceGroupRuntimeExecutor = new PeriodicTaskExecutor(resourceGroupRuntimeInfoRefreshInterval.toMillis(), refreshExecutor, this::refreshResourceGroupRuntimeInfo); configurationManagerFactories.putIfAbsent(LegacyResourceGroupConfigurationManager.NAME, new LegacyResourceGroupConfigurationManager.Factory()); this.isConfigurationManagerLoaded = new AtomicBoolean(false); - this.clusterResourceChecker = clusterResourceChecker; + this.clusterResourceChecker = requireNonNull(clusterResourceChecker, "clusterResourceChecker is null"); + this.maxQueryAdmissionsPerSecond = queryManagerConfig.getMaxQueryAdmissionsPerSecond(); + this.minRunningQueriesForPacing = queryManagerConfig.getMinRunningQueriesForPacing(); + this.queryAdmissionIntervalNanos = (maxQueryAdmissionsPerSecond == Integer.MAX_VALUE) + ? 0L + : 1_000_000_000L / maxQueryAdmissionsPerSecond; + this.queryPacingContext = new QueryPacingContext() + { + @Override + public boolean tryAcquireAdmissionSlot() + { + return InternalResourceGroupManager.this.tryAcquireAdmissionSlot(); + } + + @Override + public void onQueryStarted() + { + incrementRunningQueries(); + } + + @Override + public void onQueryFinished() + { + decrementRunningQueries(); + } + }; + } + + /** + * Global rate limiter for query admissions. Enforces maxQueryAdmissionsPerSecond + * when running queries exceed minRunningQueriesForPacing threshold. + * + * @return true if query can be admitted, false if rate limit exceeded + */ + boolean tryAcquireAdmissionSlot() + { + // Pacing disabled - return early without tracking metrics + if (queryAdmissionIntervalNanos == 0L) { + return true; + } + + // Running queries below threshold - bypass pacing + int currentRunningQueries = getTotalRunningQueries(); + if (currentRunningQueries < minRunningQueriesForPacing) { + return true; + } + + totalAdmissionAttempts.incrementAndGet(); + + // Atomic update for global rate limiting. With multiple root resource groups, + // concurrent threads may call this method simultaneously (each holding their + // own root group's lock). Compare-and-swap ensures correctness in that scenario. + // With a single root group, the root lock serializes access, making the atomic + // update redundant but harmless. + for (int attempt = 0; attempt < 10; attempt++) { + long now = System.nanoTime(); + long last = lastAdmittedQueryNanos.get(); + + // Check if enough time has elapsed since last admission + if (last != 0L && (now - last) < queryAdmissionIntervalNanos) { + totalAdmissionsDenied.incrementAndGet(); + return false; + } + + // Atomically update timestamp if unchanged; retry if another thread won + if (lastAdmittedQueryNanos.compareAndSet(last, now)) { + totalAdmissionsGranted.incrementAndGet(); + return true; + } + } + + // Exhausted retries - deny to prevent starvation under extreme contention + totalAdmissionsDenied.incrementAndGet(); + return false; + } + + /** + * Returns total running queries across all resource groups. + * Uses atomic counter updated via callbacks to avoid locking resource groups. + */ + private int getTotalRunningQueries() + { + return totalRunningQueriesCounter.get(); + } + + /** Called by InternalResourceGroup when a query starts execution. */ + public void incrementRunningQueries() + { + totalRunningQueriesCounter.incrementAndGet(); + } + + /** Called by InternalResourceGroup when a query finishes execution. */ + public void decrementRunningQueries() + { + totalRunningQueriesCounter.decrementAndGet(); } @Override @@ -406,7 +514,15 @@ private synchronized void createGroupIfNecessary(SelectionContext context, Ex else { RootInternalResourceGroup root; if (!isResourceManagerEnabled) { - root = new RootInternalResourceGroup(id.getSegments().get(0), this::exportGroup, executor, ignored -> Optional.empty(), rg -> false, nodeManager, clusterResourceChecker); + root = new RootInternalResourceGroup( + id.getSegments().get(0), + this::exportGroup, + executor, + ignored -> Optional.empty(), + rg -> false, + nodeManager, + clusterResourceChecker, + queryPacingContext); } else { root = new RootInternalResourceGroup( @@ -420,7 +536,8 @@ private synchronized void createGroupIfNecessary(SelectionContext context, Ex lastUpdatedResourceGroupRuntimeInfo::get, concurrencyThreshold), nodeManager, - clusterResourceChecker); + clusterResourceChecker, + queryPacingContext); } group = root; rootGroups.add(root); @@ -500,6 +617,57 @@ public long getLastSchedulingCycleRuntimeDelayMs() return lastSchedulingCycleRunTimeMs.get() == 0L ? lastSchedulingCycleRunTimeMs.get() : currentTimeMillis() - lastSchedulingCycleRunTimeMs.get(); } + @Managed + public int getMaxQueryAdmissionsPerSecond() + { + return maxQueryAdmissionsPerSecond; + } + + @Managed + public long getTotalAdmissionAttempts() + { + return totalAdmissionAttempts.get(); + } + + @Managed + public long getTotalAdmissionsGranted() + { + return totalAdmissionsGranted.get(); + } + + @Managed + public long getTotalAdmissionsDenied() + { + return totalAdmissionsDenied.get(); + } + + @Managed + public int getMinRunningQueriesForPacing() + { + return minRunningQueriesForPacing; + } + + @Managed + public double getAdmissionGrantRate() + { + long attempts = totalAdmissionAttempts.get(); + return attempts > 0 ? (double) totalAdmissionsGranted.get() / attempts : 0.0; + } + + @Managed + public double getAdmissionDenyRate() + { + long attempts = totalAdmissionAttempts.get(); + return attempts > 0 ? (double) totalAdmissionsDenied.get() / attempts : 0.0; + } + + @Managed + public long getMillisSinceLastAdmission() + { + long last = lastAdmittedQueryNanos.get(); + return last == 0L ? -1L : (System.nanoTime() - last) / 1_000_000; + } + private int getQueriesQueuedOnInternal(InternalResourceGroup resourceGroup) { if (resourceGroup.subGroups().isEmpty()) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/QueryPacingContext.java b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/QueryPacingContext.java new file mode 100644 index 0000000000000..02eca0b4ac607 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/resourceGroups/QueryPacingContext.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.execution.resourceGroups; + +/** + * Context for query admission pacing. Provides a single interface for + * global rate limiting and running query tracking to prevent worker overload. + *

+ * This interface consolidates the pacing-related callbacks that are shared + * across all resource groups, keeping resource group objects smaller. + */ +public interface QueryPacingContext +{ + /** + * A no-op implementation that allows all queries and tracks nothing. + */ + QueryPacingContext NOOP = new QueryPacingContext() + { + @Override + public boolean tryAcquireAdmissionSlot() + { + return true; + } + + @Override + public void onQueryStarted() + { + } + + @Override + public void onQueryFinished() + { + } + }; + + /** + * Attempts to acquire an admission slot for starting a new query. + * Enforces global rate limiting when running queries exceed threshold. + * + * @return true if query can be admitted, false if rate limit exceeded + */ + boolean tryAcquireAdmissionSlot(); + + /** + * Called when a query starts running. Used to track global running query count. + */ + void onQueryStarted(); + + /** + * Called when a query finishes (success or failure). Used to track global running query count. + */ + void onQueryFinished(); +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index 4984cd8fe065d..4bca843875cef 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -317,6 +317,8 @@ import static com.facebook.presto.common.type.DoubleType.OLD_NAN_DOUBLE; import static com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.KdbTreeType.KDB_TREE; import static com.facebook.presto.common.type.KllSketchParametricType.KLL_SKETCH; @@ -504,8 +506,6 @@ import static com.facebook.presto.type.FunctionParametricType.FUNCTION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.JoniRegexpType.JONI_REGEXP; import static com.facebook.presto.type.JsonPathType.JSON_PATH; import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java index efab521446259..6165940a70f92 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java @@ -714,6 +714,34 @@ public void dropBranch(Session session, TableHandle tableHandle, String branchNa delegate.dropBranch(session, tableHandle, branchName, branchExists); } + @Override + public void createBranch( + Session session, + TableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + delegate.createBranch(session, tableHandle, branchName, replace, ifNotExists, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + + @Override + public void createTag( + Session session, + TableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays) + { + delegate.createTag(session, tableHandle, tagName, replace, ifNotExists, tableVersion, retainDays); + } + @Override public void dropTag(Session session, TableHandle tableHandle, String tagName, boolean tagExists) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java index 41c53f31d775f..edb2285ba23d5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -593,6 +593,24 @@ default TableLayoutFilterCoverage getTableLayoutFilterCoverage(Session session, void dropBranch(Session session, TableHandle tableHandle, String branchName, boolean branchExists); + void createBranch(Session session, + TableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays); + + void createTag(Session session, + TableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays); + void dropTag(Session session, TableHandle tableHandle, String tagName, boolean tagExists); void dropConstraint(Session session, TableHandle tableHandle, Optional constraintName, Optional columnName); diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java index 6c8ef22ebb878..183cbe91f252b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -1746,6 +1746,22 @@ public void dropBranch(Session session, TableHandle tableHandle, String branchNa metadata.dropBranch(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), branchName, branchExists); } + @Override + public void createBranch(Session session, TableHandle tableHandle, String branchName, boolean replace, boolean ifNotExists, Optional tableVersion, Optional retainDays, Optional minSnapshotsToKeep, Optional maxSnapshotAgeDays) + { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + metadata.createBranch(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), branchName, replace, ifNotExists, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + + @Override + public void createTag(Session session, TableHandle tableHandle, String tagName, boolean replace, boolean ifNotExists, Optional tableVersion, Optional retainDays) + { + ConnectorId connectorId = tableHandle.getConnectorId(); + ConnectorMetadata metadata = getMetadataForWrite(session, connectorId); + metadata.createTag(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), tagName, replace, ifNotExists, tableVersion, retainDays); + } + @Override public void dropTag(Session session, TableHandle tableHandle, String tagName, boolean tagExists) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManagerStats.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManagerStats.java new file mode 100644 index 0000000000000..3f4c80c01cfda --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManagerStats.java @@ -0,0 +1,1847 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.airlift.stats.TimeStat; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +public class MetadataManagerStats +{ + private final AtomicLong applyTableFunctionCalls = new AtomicLong(); + private final AtomicLong verifyComparableOrderableContractCalls = new AtomicLong(); + private final AtomicLong getTypeCalls = new AtomicLong(); + private final AtomicLong registerBuiltInFunctionsCalls = new AtomicLong(); + private final AtomicLong registerConnectorFunctionsCalls = new AtomicLong(); + private final AtomicLong listSchemaNamesCalls = new AtomicLong(); + private final AtomicLong getSchemaPropertiesCalls = new AtomicLong(); + private final AtomicLong getSystemTableCalls = new AtomicLong(); + private final AtomicLong getHandleVersionCalls = new AtomicLong(); + private final AtomicLong getTableHandleForStatisticsCollectionCalls = new AtomicLong(); + private final AtomicLong getLayoutCalls = new AtomicLong(); + private final AtomicLong getAlternativeTableHandleCalls = new AtomicLong(); + private final AtomicLong isLegacyGetLayoutSupportedCalls = new AtomicLong(); + private final AtomicLong getCommonPartitioningCalls = new AtomicLong(); + private final AtomicLong isRefinedPartitioningOverCalls = new AtomicLong(); + private final AtomicLong getPartitioningHandleForExchangeCalls = new AtomicLong(); + private final AtomicLong getInfoCalls = new AtomicLong(); + private final AtomicLong getTableMetadataCalls = new AtomicLong(); + private final AtomicLong listTablesCalls = new AtomicLong(); + private final AtomicLong getColumnHandlesCalls = new AtomicLong(); + private final AtomicLong getColumnMetadataCalls = new AtomicLong(); + private final AtomicLong toExplainIOConstraintsCalls = new AtomicLong(); + private final AtomicLong listTableColumnsCalls = new AtomicLong(); + private final AtomicLong createSchemaCalls = new AtomicLong(); + private final AtomicLong dropSchemaCalls = new AtomicLong(); + private final AtomicLong renameSchemaCalls = new AtomicLong(); + private final AtomicLong createTableCalls = new AtomicLong(); + private final AtomicLong createTemporaryTableCalls = new AtomicLong(); + private final AtomicLong dropTableCalls = new AtomicLong(); + private final AtomicLong truncateTableCalls = new AtomicLong(); + private final AtomicLong getNewTableLayoutCalls = new AtomicLong(); + private final AtomicLong beginCreateTableCalls = new AtomicLong(); + private final AtomicLong finishCreateTableCalls = new AtomicLong(); + private final AtomicLong getInsertLayoutCalls = new AtomicLong(); + private final AtomicLong getStatisticsCollectionMetadataForWriteCalls = new AtomicLong(); + private final AtomicLong getStatisticsCollectionMetadataCalls = new AtomicLong(); + private final AtomicLong beginStatisticsCollectionCalls = new AtomicLong(); + private final AtomicLong finishStatisticsCollectionCalls = new AtomicLong(); + private final AtomicLong beginQueryCalls = new AtomicLong(); + private final AtomicLong cleanupQueryCalls = new AtomicLong(); + private final AtomicLong beginInsertCalls = new AtomicLong(); + private final AtomicLong finishInsertCalls = new AtomicLong(); + private final AtomicLong getDeleteRowIdColumnCalls = new AtomicLong(); + private final AtomicLong getUpdateRowIdColumnCalls = new AtomicLong(); + private final AtomicLong supportsMetadataDeleteCalls = new AtomicLong(); + private final AtomicLong metadataDeleteCalls = new AtomicLong(); + private final AtomicLong beginDeleteCalls = new AtomicLong(); + private final AtomicLong finishDeleteWithOutputCalls = new AtomicLong(); + private final AtomicLong beginCallDistributedProcedureCalls = new AtomicLong(); + private final AtomicLong finishCallDistributedProcedureCalls = new AtomicLong(); + private final AtomicLong beginUpdateCalls = new AtomicLong(); + private final AtomicLong finishUpdateCalls = new AtomicLong(); + private final AtomicLong getRowChangeParadigmCalls = new AtomicLong(); + private final AtomicLong getMergeTargetTableRowIdColumnHandleCalls = new AtomicLong(); + private final AtomicLong beginMergeCalls = new AtomicLong(); + private final AtomicLong finishMergeCalls = new AtomicLong(); + private final AtomicLong getCatalogHandleCalls = new AtomicLong(); + private final AtomicLong getCatalogNamesCalls = new AtomicLong(); + private final AtomicLong listViewsCalls = new AtomicLong(); + private final AtomicLong getViewsCalls = new AtomicLong(); + private final AtomicLong createViewCalls = new AtomicLong(); + private final AtomicLong renameViewCalls = new AtomicLong(); + private final AtomicLong dropViewCalls = new AtomicLong(); + private final AtomicLong createMaterializedViewCalls = new AtomicLong(); + private final AtomicLong dropMaterializedViewCalls = new AtomicLong(); + private final AtomicLong listMaterializedViewsCalls = new AtomicLong(); + private final AtomicLong getMaterializedViewsCalls = new AtomicLong(); + private final AtomicLong beginRefreshMaterializedViewCalls = new AtomicLong(); + private final AtomicLong finishRefreshMaterializedViewCalls = new AtomicLong(); + private final AtomicLong getReferencedMaterializedViewsCalls = new AtomicLong(); + private final AtomicLong getMaterializedViewStatusCalls = new AtomicLong(); + private final AtomicLong resolveIndexCalls = new AtomicLong(); + private final AtomicLong createRoleCalls = new AtomicLong(); + private final AtomicLong dropRoleCalls = new AtomicLong(); + private final AtomicLong listRolesCalls = new AtomicLong(); + private final AtomicLong listRoleGrantsCalls = new AtomicLong(); + private final AtomicLong grantRolesCalls = new AtomicLong(); + private final AtomicLong revokeRolesCalls = new AtomicLong(); + private final AtomicLong listApplicableRolesCalls = new AtomicLong(); + private final AtomicLong listEnabledRolesCalls = new AtomicLong(); + private final AtomicLong grantTablePrivilegesCalls = new AtomicLong(); + private final AtomicLong revokeTablePrivilegesCalls = new AtomicLong(); + private final AtomicLong listTablePrivilegesCalls = new AtomicLong(); + private final AtomicLong commitPageSinkAsyncCalls = new AtomicLong(); + private final AtomicLong getFunctionAndTypeManagerCalls = new AtomicLong(); + private final AtomicLong getProcedureRegistryCalls = new AtomicLong(); + private final AtomicLong getBlockEncodingSerdeCalls = new AtomicLong(); + private final AtomicLong getSessionPropertyManagerCalls = new AtomicLong(); + private final AtomicLong getSchemaPropertyManagerCalls = new AtomicLong(); + private final AtomicLong getTablePropertyManagerCalls = new AtomicLong(); + private final AtomicLong getColumnPropertyManagerCalls = new AtomicLong(); + private final AtomicLong getAnalyzePropertyManagerCalls = new AtomicLong(); + private final AtomicLong getMetadataResolverCalls = new AtomicLong(); + private final AtomicLong getConnectorCapabilitiesCalls = new AtomicLong(); + private final AtomicLong dropBranchCalls = new AtomicLong(); + private final AtomicLong createBranchCalls = new AtomicLong(); + private final AtomicLong createTagCalls = new AtomicLong(); + private final AtomicLong dropTagCalls = new AtomicLong(); + private final AtomicLong dropConstraintCalls = new AtomicLong(); + private final AtomicLong addConstraintCalls = new AtomicLong(); + private final AtomicLong renameTableCalls = new AtomicLong(); + private final AtomicLong setTablePropertiesCalls = new AtomicLong(); + private final AtomicLong addColumnCalls = new AtomicLong(); + private final AtomicLong dropColumnCalls = new AtomicLong(); + private final AtomicLong renameColumnCalls = new AtomicLong(); + private final AtomicLong normalizeIdentifierCalls = new AtomicLong(); + private final AtomicLong getTableLayoutFilterCoverageCalls = new AtomicLong(); + private final AtomicLong getTableStatisticsCalls = new AtomicLong(); + private final AtomicLong getCatalogNamesWithConnectorContextCalls = new AtomicLong(); + private final AtomicLong isPushdownSupportedForFilterCalls = new AtomicLong(); + private final TimeStat applyTableFunctionTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat verifyComparableOrderableContractTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTypeTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat registerBuiltInFunctionsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat registerConnectorFunctionsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listSchemaNamesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getSchemaPropertiesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getSystemTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getHandleVersionTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTableHandleForStatisticsCollectionTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getLayoutTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getAlternativeTableHandleTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat isLegacyGetLayoutSupportedTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getCommonPartitioningTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat isRefinedPartitioningOverTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getPartitioningHandleForExchangeTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getInfoTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTableMetadataTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listTablesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getColumnHandlesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getColumnMetadataTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat toExplainIOConstraintsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listTableColumnsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createSchemaTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropSchemaTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat renameSchemaTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createTemporaryTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat truncateTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getNewTableLayoutTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginCreateTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishCreateTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getInsertLayoutTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getStatisticsCollectionMetadataForWriteTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getStatisticsCollectionMetadataTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginStatisticsCollectionTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishStatisticsCollectionTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginQueryTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat cleanupQueryTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginInsertTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishInsertTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getDeleteRowIdColumnTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getUpdateRowIdColumnTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat supportsMetadataDeleteTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat metadataDeleteTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginDeleteTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishDeleteWithOutputTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginCallDistributedProcedureTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishCallDistributedProcedureTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginUpdateTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishUpdateTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getRowChangeParadigmTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getMergeTargetTableRowIdColumnHandleTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginMergeTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishMergeTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getCatalogHandleTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getCatalogNamesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listViewsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getViewsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat renameViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createMaterializedViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropMaterializedViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listMaterializedViewsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getMaterializedViewsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat beginRefreshMaterializedViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat finishRefreshMaterializedViewTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getReferencedMaterializedViewsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getMaterializedViewStatusTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat resolveIndexTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createRoleTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropRoleTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listRolesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listRoleGrantsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat grantRolesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat revokeRolesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listApplicableRolesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listEnabledRolesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat grantTablePrivilegesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat revokeTablePrivilegesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat listTablePrivilegesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat commitPageSinkAsyncTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getFunctionAndTypeManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getProcedureRegistryTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getBlockEncodingSerdeTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getSessionPropertyManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getSchemaPropertyManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTablePropertyManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getColumnPropertyManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getAnalyzePropertyManagerTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getMetadataResolverTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getConnectorCapabilitiesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropBranchTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createBranchTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat createTagTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropTagTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropConstraintTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat addConstraintTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat renameTableTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat setTablePropertiesTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat addColumnTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat dropColumnTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat renameColumnTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat normalizeIdentifierTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTableLayoutFilterCoverageTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getTableStatisticsTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat getCatalogNamesWithConnectorContextTime = new TimeStat(TimeUnit.NANOSECONDS); + private final TimeStat isPushdownSupportedForFilterTime = new TimeStat(TimeUnit.NANOSECONDS); + + @Managed + public long getApplyTableFunctionCalls() + { + return applyTableFunctionCalls.get(); + } + + @Managed + public long getVerifyComparableOrderableContractCalls() + { + return verifyComparableOrderableContractCalls.get(); + } + + @Managed + public long getGetTypeCalls() + { + return getTypeCalls.get(); + } + + @Managed + public long getRegisterBuiltInFunctionsCalls() + { + return registerBuiltInFunctionsCalls.get(); + } + + @Managed + public long getRegisterConnectorFunctionsCalls() + { + return registerConnectorFunctionsCalls.get(); + } + + @Managed + public long getListSchemaNamesCalls() + { + return listSchemaNamesCalls.get(); + } + + @Managed + public long getGetSchemaPropertiesCalls() + { + return getSchemaPropertiesCalls.get(); + } + + @Managed + public long getGetSystemTableCalls() + { + return getSystemTableCalls.get(); + } + + @Managed + public long getGetHandleVersionCalls() + { + return getHandleVersionCalls.get(); + } + + @Managed + public long getGetTableHandleForStatisticsCollectionCalls() + { + return getTableHandleForStatisticsCollectionCalls.get(); + } + + @Managed + public long getGetLayoutCalls() + { + return getLayoutCalls.get(); + } + + @Managed + public long getGetAlternativeTableHandleCalls() + { + return getAlternativeTableHandleCalls.get(); + } + + @Managed + public long getIsLegacyGetLayoutSupportedCalls() + { + return isLegacyGetLayoutSupportedCalls.get(); + } + + @Managed + public long getGetCommonPartitioningCalls() + { + return getCommonPartitioningCalls.get(); + } + + @Managed + public long getIsRefinedPartitioningOverCalls() + { + return isRefinedPartitioningOverCalls.get(); + } + + @Managed + public long getGetPartitioningHandleForExchangeCalls() + { + return getPartitioningHandleForExchangeCalls.get(); + } + + @Managed + public long getGetInfoCalls() + { + return getInfoCalls.get(); + } + + @Managed + public long getGetTableMetadataCalls() + { + return getTableMetadataCalls.get(); + } + + @Managed + public long getListTablesCalls() + { + return listTablesCalls.get(); + } + + @Managed + public long getGetColumnHandlesCalls() + { + return getColumnHandlesCalls.get(); + } + + @Managed + public long getGetColumnMetadataCalls() + { + return getColumnMetadataCalls.get(); + } + + @Managed + public long getToExplainIOConstraintsCalls() + { + return toExplainIOConstraintsCalls.get(); + } + + @Managed + public long getListTableColumnsCalls() + { + return listTableColumnsCalls.get(); + } + + @Managed + public long getCreateSchemaCalls() + { + return createSchemaCalls.get(); + } + + @Managed + public long getDropSchemaCalls() + { + return dropSchemaCalls.get(); + } + + @Managed + public long getRenameSchemaCalls() + { + return renameSchemaCalls.get(); + } + + @Managed + public long getNormalizeIdentifierCalls() + { + return normalizeIdentifierCalls.get(); + } + + @Managed + public long getGetTableLayoutFilterCoverageCalls() + { + return getTableLayoutFilterCoverageCalls.get(); + } + + @Managed + public long getGetTableStatisticsCalls() + { + return getTableStatisticsCalls.get(); + } + + @Managed + @Nested + public TimeStat getApplyTableFunctionTime() + { + return applyTableFunctionTime; + } + + @Managed + @Nested + public TimeStat getVerifyComparableOrderableContractTime() + { + return verifyComparableOrderableContractTime; + } + + @Managed + @Nested + public TimeStat getGetTypeTime() + { + return getTypeTime; + } + + @Managed + @Nested + public TimeStat getRegisterBuiltInFunctionsTime() + { + return registerBuiltInFunctionsTime; + } + + @Managed + @Nested + public TimeStat getRegisterConnectorFunctionsTime() + { + return registerConnectorFunctionsTime; + } + + @Managed + @Nested + public TimeStat getListSchemaNamesTime() + { + return listSchemaNamesTime; + } + + @Managed + @Nested + public TimeStat getGetSchemaPropertiesTime() + { + return getSchemaPropertiesTime; + } + + @Managed + @Nested + public TimeStat getGetSystemTableTime() + { + return getSystemTableTime; + } + + @Managed + @Nested + public TimeStat getGetHandleVersionTime() + { + return getHandleVersionTime; + } + + @Managed + @Nested + public TimeStat getGetTableHandleForStatisticsCollectionTime() + { + return getTableHandleForStatisticsCollectionTime; + } + + @Managed + @Nested + public TimeStat getGetLayoutTime() + { + return getLayoutTime; + } + + @Managed + @Nested + public TimeStat getGetAlternativeTableHandleTime() + { + return getAlternativeTableHandleTime; + } + + @Managed + @Nested + public TimeStat getIsLegacyGetLayoutSupportedTime() + { + return isLegacyGetLayoutSupportedTime; + } + + @Managed + @Nested + public TimeStat getGetCommonPartitioningTime() + { + return getCommonPartitioningTime; + } + + @Managed + @Nested + public TimeStat getIsRefinedPartitioningOverTime() + { + return isRefinedPartitioningOverTime; + } + + @Managed + @Nested + public TimeStat getGetPartitioningHandleForExchangeTime() + { + return getPartitioningHandleForExchangeTime; + } + + @Managed + @Nested + public TimeStat getGetInfoTime() + { + return getInfoTime; + } + + @Managed + @Nested + public TimeStat getGetTableMetadataTime() + { + return getTableMetadataTime; + } + + @Managed + @Nested + public TimeStat getListTablesTime() + { + return listTablesTime; + } + + @Managed + @Nested + public TimeStat getGetColumnHandlesTime() + { + return getColumnHandlesTime; + } + + @Managed + @Nested + public TimeStat getGetColumnMetadataTime() + { + return getColumnMetadataTime; + } + + @Managed + @Nested + public TimeStat getToExplainIOConstraintsTime() + { + return toExplainIOConstraintsTime; + } + + @Managed + @Nested + public TimeStat getListTableColumnsTime() + { + return listTableColumnsTime; + } + + @Managed + @Nested + public TimeStat getCreateSchemaTime() + { + return createSchemaTime; + } + + @Managed + @Nested + public TimeStat getDropSchemaTime() + { + return dropSchemaTime; + } + + @Managed + @Nested + public TimeStat getRenameSchemaTime() + { + return renameSchemaTime; + } + + @Managed + @Nested + public TimeStat getCreateTableTime() + { + return createTableTime; + } + + @Managed + @Nested + public TimeStat getCreateTemporaryTableTime() + { + return createTemporaryTableTime; + } + + @Managed + @Nested + public TimeStat getDropTableTime() + { + return dropTableTime; + } + + @Managed + @Nested + public TimeStat getTruncateTableTime() + { + return truncateTableTime; + } + + @Managed + @Nested + public TimeStat getGetNewTableLayoutTime() + { + return getNewTableLayoutTime; + } + + @Managed + @Nested + public TimeStat getBeginCreateTableTime() + { + return beginCreateTableTime; + } + + @Managed + @Nested + public TimeStat getFinishCreateTableTime() + { + return finishCreateTableTime; + } + + @Managed + @Nested + public TimeStat getGetInsertLayoutTime() + { + return getInsertLayoutTime; + } + + @Managed + @Nested + public TimeStat getGetStatisticsCollectionMetadataForWriteTime() + { + return getStatisticsCollectionMetadataForWriteTime; + } + + @Managed + @Nested + public TimeStat getGetStatisticsCollectionMetadataTime() + { + return getStatisticsCollectionMetadataTime; + } + + @Managed + @Nested + public TimeStat getBeginStatisticsCollectionTime() + { + return beginStatisticsCollectionTime; + } + + @Managed + @Nested + public TimeStat getFinishStatisticsCollectionTime() + { + return finishStatisticsCollectionTime; + } + + @Managed + @Nested + public TimeStat getBeginQueryTime() + { + return beginQueryTime; + } + + @Managed + @Nested + public TimeStat getCleanupQueryTime() + { + return cleanupQueryTime; + } + + @Managed + @Nested + public TimeStat getBeginInsertTime() + { + return beginInsertTime; + } + + @Managed + @Nested + public TimeStat getFinishInsertTime() + { + return finishInsertTime; + } + + @Managed + @Nested + public TimeStat getGetDeleteRowIdColumnTime() + { + return getDeleteRowIdColumnTime; + } + + @Managed + @Nested + public TimeStat getGetUpdateRowIdColumnTime() + { + return getUpdateRowIdColumnTime; + } + + @Managed + @Nested + public TimeStat getSupportsMetadataDeleteTime() + { + return supportsMetadataDeleteTime; + } + + @Managed + @Nested + public TimeStat getMetadataDeleteTime() + { + return metadataDeleteTime; + } + + @Managed + @Nested + public TimeStat getBeginDeleteTime() + { + return beginDeleteTime; + } + + @Managed + @Nested + public TimeStat getFinishDeleteWithOutputTime() + { + return finishDeleteWithOutputTime; + } + + @Managed + @Nested + public TimeStat getBeginCallDistributedProcedureTime() + { + return beginCallDistributedProcedureTime; + } + + @Managed + @Nested + public TimeStat getFinishCallDistributedProcedureTime() + { + return finishCallDistributedProcedureTime; + } + + @Managed + @Nested + public TimeStat getBeginUpdateTime() + { + return beginUpdateTime; + } + + @Managed + @Nested + public TimeStat getFinishUpdateTime() + { + return finishUpdateTime; + } + + @Managed + @Nested + public TimeStat getGetRowChangeParadigmTime() + { + return getRowChangeParadigmTime; + } + + @Managed + @Nested + public TimeStat getGetMergeTargetTableRowIdColumnHandleTime() + { + return getMergeTargetTableRowIdColumnHandleTime; + } + + @Managed + @Nested + public TimeStat getBeginMergeTime() + { + return beginMergeTime; + } + + @Managed + @Nested + public TimeStat getFinishMergeTime() + { + return finishMergeTime; + } + + @Managed + @Nested + public TimeStat getGetCatalogHandleTime() + { + return getCatalogHandleTime; + } + + @Managed + @Nested + public TimeStat getGetCatalogNamesTime() + { + return getCatalogNamesTime; + } + + @Managed + @Nested + public TimeStat getListViewsTime() + { + return listViewsTime; + } + + @Managed + @Nested + public TimeStat getGetViewsTime() + { + return getViewsTime; + } + + @Managed + @Nested + public TimeStat getCreateViewTime() + { + return createViewTime; + } + + @Managed + @Nested + public TimeStat getRenameViewTime() + { + return renameViewTime; + } + + @Managed + @Nested + public TimeStat getDropViewTime() + { + return dropViewTime; + } + + @Managed + @Nested + public TimeStat getCreateMaterializedViewTime() + { + return createMaterializedViewTime; + } + + @Managed + @Nested + public TimeStat getDropMaterializedViewTime() + { + return dropMaterializedViewTime; + } + + @Managed + @Nested + public TimeStat getListMaterializedViewsTime() + { + return listMaterializedViewsTime; + } + + @Managed + @Nested + public TimeStat getGetMaterializedViewsTime() + { + return getMaterializedViewsTime; + } + + @Managed + @Nested + public TimeStat getBeginRefreshMaterializedViewTime() + { + return beginRefreshMaterializedViewTime; + } + + @Managed + @Nested + public TimeStat getFinishRefreshMaterializedViewTime() + { + return finishRefreshMaterializedViewTime; + } + + @Managed + @Nested + public TimeStat getGetReferencedMaterializedViewsTime() + { + return getReferencedMaterializedViewsTime; + } + + @Managed + @Nested + public TimeStat getGetMaterializedViewStatusTime() + { + return getMaterializedViewStatusTime; + } + + @Managed + @Nested + public TimeStat getResolveIndexTime() + { + return resolveIndexTime; + } + + @Managed + @Nested + public TimeStat getCreateRoleTime() + { + return createRoleTime; + } + + @Managed + @Nested + public TimeStat getDropRoleTime() + { + return dropRoleTime; + } + + @Managed + @Nested + public TimeStat getListRolesTime() + { + return listRolesTime; + } + + @Managed + @Nested + public TimeStat getListRoleGrantsTime() + { + return listRoleGrantsTime; + } + + @Managed + @Nested + public TimeStat getGrantRolesTime() + { + return grantRolesTime; + } + + @Managed + @Nested + public TimeStat getRevokeRolesTime() + { + return revokeRolesTime; + } + + @Managed + @Nested + public TimeStat getListApplicableRolesTime() + { + return listApplicableRolesTime; + } + + @Managed + @Nested + public TimeStat getListEnabledRolesTime() + { + return listEnabledRolesTime; + } + + @Managed + @Nested + public TimeStat getGrantTablePrivilegesTime() + { + return grantTablePrivilegesTime; + } + + @Managed + @Nested + public TimeStat getRevokeTablePrivilegesTime() + { + return revokeTablePrivilegesTime; + } + + @Managed + @Nested + public TimeStat getListTablePrivilegesTime() + { + return listTablePrivilegesTime; + } + + @Managed + @Nested + public TimeStat getCommitPageSinkAsyncTime() + { + return commitPageSinkAsyncTime; + } + + @Managed + @Nested + public TimeStat getGetFunctionAndTypeManagerTime() + { + return getFunctionAndTypeManagerTime; + } + + @Managed + @Nested + public TimeStat getGetProcedureRegistryTime() + { + return getProcedureRegistryTime; + } + + @Managed + @Nested + public TimeStat getGetBlockEncodingSerdeTime() + { + return getBlockEncodingSerdeTime; + } + + @Managed + @Nested + public TimeStat getGetSessionPropertyManagerTime() + { + return getSessionPropertyManagerTime; + } + + @Managed + @Nested + public TimeStat getGetSchemaPropertyManagerTime() + { + return getSchemaPropertyManagerTime; + } + + @Managed + @Nested + public TimeStat getGetTablePropertyManagerTime() + { + return getTablePropertyManagerTime; + } + + @Managed + @Nested + public TimeStat getGetColumnPropertyManagerTime() + { + return getColumnPropertyManagerTime; + } + + @Managed + @Nested + public TimeStat getGetAnalyzePropertyManagerTime() + { + return getAnalyzePropertyManagerTime; + } + + @Managed + @Nested + public TimeStat getGetMetadataResolverTime() + { + return getMetadataResolverTime; + } + + @Managed + @Nested + public TimeStat getGetConnectorCapabilitiesTime() + { + return getConnectorCapabilitiesTime; + } + + @Managed + @Nested + public TimeStat getDropBranchTime() + { + return dropBranchTime; + } + + @Managed + @Nested + public TimeStat getDropTagTime() + { + return dropTagTime; + } + + @Managed + @Nested + public TimeStat getDropConstraintTime() + { + return dropConstraintTime; + } + + @Managed + @Nested + public TimeStat getAddConstraintTime() + { + return addConstraintTime; + } + + @Managed + @Nested + public TimeStat getRenameTableTime() + { + return renameTableTime; + } + + @Managed + @Nested + public TimeStat getSetTablePropertiesTime() + { + return setTablePropertiesTime; + } + + @Managed + @Nested + public TimeStat getAddColumnTime() + { + return addColumnTime; + } + + @Managed + @Nested + public TimeStat getDropColumnTime() + { + return dropColumnTime; + } + + @Managed + @Nested + public TimeStat getRenameColumnTime() + { + return renameColumnTime; + } + + @Managed + @Nested + public TimeStat getNormalizeIdentifierTime() + { + return normalizeIdentifierTime; + } + + @Managed + @Nested + public TimeStat getGetTableLayoutFilterCoverageTime() + { + return getTableLayoutFilterCoverageTime; + } + + @Managed + @Nested + public TimeStat getGetTableStatisticsTime() + { + return getTableStatisticsTime; + } + + public void recordApplyTableFunctionCall(long duration) + { + applyTableFunctionCalls.incrementAndGet(); + applyTableFunctionTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordVerifyComparableOrderableContractCall(long duration) + { + verifyComparableOrderableContractCalls.incrementAndGet(); + verifyComparableOrderableContractTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTypeCall(long duration) + { + getTypeCalls.incrementAndGet(); + getTypeTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRegisterBuiltInFunctionsCall(long duration) + { + registerBuiltInFunctionsCalls.incrementAndGet(); + registerBuiltInFunctionsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRegisterConnectorFunctionsCall(long duration) + { + registerConnectorFunctionsCalls.incrementAndGet(); + registerConnectorFunctionsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListSchemaNamesCall(long duration) + { + listSchemaNamesCalls.incrementAndGet(); + listSchemaNamesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetSchemaPropertiesCall(long duration) + { + getSchemaPropertiesCalls.incrementAndGet(); + getSchemaPropertiesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetSystemTableCall(long duration) + { + getSystemTableCalls.incrementAndGet(); + getSystemTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetHandleVersionCall(long duration) + { + getHandleVersionCalls.incrementAndGet(); + getHandleVersionTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTableHandleForStatisticsCollectionCall(long duration) + { + getTableHandleForStatisticsCollectionCalls.incrementAndGet(); + getTableHandleForStatisticsCollectionTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetLayoutCall(long duration) + { + getLayoutCalls.incrementAndGet(); + getLayoutTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetAlternativeTableHandleCall(long duration) + { + getAlternativeTableHandleCalls.incrementAndGet(); + getAlternativeTableHandleTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordIsLegacyGetLayoutSupportedCall(long duration) + { + isLegacyGetLayoutSupportedCalls.incrementAndGet(); + isLegacyGetLayoutSupportedTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetCommonPartitioningCall(long duration) + { + getCommonPartitioningCalls.incrementAndGet(); + getCommonPartitioningTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordIsRefinedPartitioningOverCall(long duration) + { + isRefinedPartitioningOverCalls.incrementAndGet(); + isRefinedPartitioningOverTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetPartitioningHandleForExchangeCall(long duration) + { + getPartitioningHandleForExchangeCalls.incrementAndGet(); + getPartitioningHandleForExchangeTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetInfoCall(long duration) + { + getInfoCalls.incrementAndGet(); + getInfoTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTableMetadataCall(long duration) + { + getTableMetadataCalls.incrementAndGet(); + getTableMetadataTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListTablesCall(long duration) + { + listTablesCalls.incrementAndGet(); + listTablesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetColumnHandlesCall(long duration) + { + getColumnHandlesCalls.incrementAndGet(); + getColumnHandlesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetColumnMetadataCall(long duration) + { + getColumnMetadataCalls.incrementAndGet(); + getColumnMetadataTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordToExplainIOConstraintsCall(long duration) + { + toExplainIOConstraintsCalls.incrementAndGet(); + toExplainIOConstraintsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListTableColumnsCall(long duration) + { + listTableColumnsCalls.incrementAndGet(); + listTableColumnsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateSchemaCall(long duration) + { + createSchemaCalls.incrementAndGet(); + createSchemaTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropSchemaCall(long duration) + { + dropSchemaCalls.incrementAndGet(); + dropSchemaTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRenameSchemaCall(long duration) + { + renameSchemaCalls.incrementAndGet(); + renameSchemaTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateTableCall(long duration) + { + createTableCalls.incrementAndGet(); + createTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateTemporaryTableCall(long duration) + { + createTemporaryTableCalls.incrementAndGet(); + createTemporaryTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropTableCall(long duration) + { + dropTableCalls.incrementAndGet(); + dropTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordTruncateTableCall(long duration) + { + truncateTableCalls.incrementAndGet(); + truncateTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetNewTableLayoutCall(long duration) + { + getNewTableLayoutCalls.incrementAndGet(); + getNewTableLayoutTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginCreateTableCall(long duration) + { + beginCreateTableCalls.incrementAndGet(); + beginCreateTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishCreateTableCall(long duration) + { + finishCreateTableCalls.incrementAndGet(); + finishCreateTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetInsertLayoutCall(long duration) + { + getInsertLayoutCalls.incrementAndGet(); + getInsertLayoutTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetStatisticsCollectionMetadataForWriteCall(long duration) + { + getStatisticsCollectionMetadataForWriteCalls.incrementAndGet(); + getStatisticsCollectionMetadataForWriteTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetStatisticsCollectionMetadataCall(long duration) + { + getStatisticsCollectionMetadataCalls.incrementAndGet(); + getStatisticsCollectionMetadataTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginStatisticsCollectionCall(long duration) + { + beginStatisticsCollectionCalls.incrementAndGet(); + beginStatisticsCollectionTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishStatisticsCollectionCall(long duration) + { + finishStatisticsCollectionCalls.incrementAndGet(); + finishStatisticsCollectionTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginQueryCall(long duration) + { + beginQueryCalls.incrementAndGet(); + beginQueryTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCleanupQueryCall(long duration) + { + cleanupQueryCalls.incrementAndGet(); + cleanupQueryTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginInsertCall(long duration) + { + beginInsertCalls.incrementAndGet(); + beginInsertTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishInsertCall(long duration) + { + finishInsertCalls.incrementAndGet(); + finishInsertTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetDeleteRowIdColumnCall(long duration) + { + getDeleteRowIdColumnCalls.incrementAndGet(); + getDeleteRowIdColumnTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetUpdateRowIdColumnCall(long duration) + { + getUpdateRowIdColumnCalls.incrementAndGet(); + getUpdateRowIdColumnTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordSupportsMetadataDeleteCall(long duration) + { + supportsMetadataDeleteCalls.incrementAndGet(); + supportsMetadataDeleteTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordMetadataDeleteCall(long duration) + { + metadataDeleteCalls.incrementAndGet(); + metadataDeleteTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginDeleteCall(long duration) + { + beginDeleteCalls.incrementAndGet(); + beginDeleteTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishDeleteWithOutputCall(long duration) + { + finishDeleteWithOutputCalls.incrementAndGet(); + finishDeleteWithOutputTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginCallDistributedProcedureCall(long duration) + { + beginCallDistributedProcedureCalls.incrementAndGet(); + beginCallDistributedProcedureTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishCallDistributedProcedureCall(long duration) + { + finishCallDistributedProcedureCalls.incrementAndGet(); + finishCallDistributedProcedureTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginUpdateCall(long duration) + { + beginUpdateCalls.incrementAndGet(); + beginUpdateTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishUpdateCall(long duration) + { + finishUpdateCalls.incrementAndGet(); + finishUpdateTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetRowChangeParadigmCall(long duration) + { + getRowChangeParadigmCalls.incrementAndGet(); + getRowChangeParadigmTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetMergeTargetTableRowIdColumnHandleCall(long duration) + { + getMergeTargetTableRowIdColumnHandleCalls.incrementAndGet(); + getMergeTargetTableRowIdColumnHandleTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginMergeCall(long duration) + { + beginMergeCalls.incrementAndGet(); + beginMergeTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishMergeCall(long duration) + { + finishMergeCalls.incrementAndGet(); + finishMergeTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetCatalogHandleCall(long duration) + { + getCatalogHandleCalls.incrementAndGet(); + getCatalogHandleTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetCatalogNamesCall(long duration) + { + getCatalogNamesCalls.incrementAndGet(); + getCatalogNamesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListViewsCall(long duration) + { + listViewsCalls.incrementAndGet(); + listViewsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetViewsCall(long duration) + { + getViewsCalls.incrementAndGet(); + getViewsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateViewCall(long duration) + { + createViewCalls.incrementAndGet(); + createViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRenameViewCall(long duration) + { + renameViewCalls.incrementAndGet(); + renameViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropViewCall(long duration) + { + dropViewCalls.incrementAndGet(); + dropViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateMaterializedViewCall(long duration) + { + createMaterializedViewCalls.incrementAndGet(); + createMaterializedViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropMaterializedViewCall(long duration) + { + dropMaterializedViewCalls.incrementAndGet(); + dropMaterializedViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListMaterializedViewsCall(long duration) + { + listMaterializedViewsCalls.incrementAndGet(); + listMaterializedViewsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetMaterializedViewsCall(long duration) + { + getMaterializedViewsCalls.incrementAndGet(); + getMaterializedViewsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordBeginRefreshMaterializedViewCall(long duration) + { + beginRefreshMaterializedViewCalls.incrementAndGet(); + beginRefreshMaterializedViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordFinishRefreshMaterializedViewCall(long duration) + { + finishRefreshMaterializedViewCalls.incrementAndGet(); + finishRefreshMaterializedViewTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetReferencedMaterializedViewsCall(long duration) + { + getReferencedMaterializedViewsCalls.incrementAndGet(); + getReferencedMaterializedViewsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetMaterializedViewStatusCall(long duration) + { + getMaterializedViewStatusCalls.incrementAndGet(); + getMaterializedViewStatusTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordResolveIndexCall(long duration) + { + resolveIndexCalls.incrementAndGet(); + resolveIndexTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateRoleCall(long duration) + { + createRoleCalls.incrementAndGet(); + createRoleTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropRoleCall(long duration) + { + dropRoleCalls.incrementAndGet(); + dropRoleTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListRolesCall(long duration) + { + listRolesCalls.incrementAndGet(); + listRolesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListRoleGrantsCall(long duration) + { + listRoleGrantsCalls.incrementAndGet(); + listRoleGrantsTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGrantRolesCall(long duration) + { + grantRolesCalls.incrementAndGet(); + grantRolesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRevokeRolesCall(long duration) + { + revokeRolesCalls.incrementAndGet(); + revokeRolesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListApplicableRolesCall(long duration) + { + listApplicableRolesCalls.incrementAndGet(); + listApplicableRolesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListEnabledRolesCall(long duration) + { + listEnabledRolesCalls.incrementAndGet(); + listEnabledRolesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGrantTablePrivilegesCall(long duration) + { + grantTablePrivilegesCalls.incrementAndGet(); + grantTablePrivilegesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRevokeTablePrivilegesCall(long duration) + { + revokeTablePrivilegesCalls.incrementAndGet(); + revokeTablePrivilegesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordListTablePrivilegesCall(long duration) + { + listTablePrivilegesCalls.incrementAndGet(); + listTablePrivilegesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCommitPageSinkAsyncCall(long duration) + { + commitPageSinkAsyncCalls.incrementAndGet(); + commitPageSinkAsyncTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetFunctionAndTypeManagerCall(long duration) + { + getFunctionAndTypeManagerCalls.incrementAndGet(); + getFunctionAndTypeManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetProcedureRegistryCall(long duration) + { + getProcedureRegistryCalls.incrementAndGet(); + getProcedureRegistryTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetBlockEncodingSerdeCall(long duration) + { + getBlockEncodingSerdeCalls.incrementAndGet(); + getBlockEncodingSerdeTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetSessionPropertyManagerCall(long duration) + { + getSessionPropertyManagerCalls.incrementAndGet(); + getSessionPropertyManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetSchemaPropertyManagerCall(long duration) + { + getSchemaPropertyManagerCalls.incrementAndGet(); + getSchemaPropertyManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTablePropertyManagerCall(long duration) + { + getTablePropertyManagerCalls.incrementAndGet(); + getTablePropertyManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetColumnPropertyManagerCall(long duration) + { + getColumnPropertyManagerCalls.incrementAndGet(); + getColumnPropertyManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetAnalyzePropertyManagerCall(long duration) + { + getAnalyzePropertyManagerCalls.incrementAndGet(); + getAnalyzePropertyManagerTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetMetadataResolverCall(long duration) + { + getMetadataResolverCalls.incrementAndGet(); + getMetadataResolverTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetConnectorCapabilitiesCall(long duration) + { + getConnectorCapabilitiesCalls.incrementAndGet(); + getConnectorCapabilitiesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropBranchCall(long duration) + { + dropBranchCalls.incrementAndGet(); + dropBranchTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateBranchCall(long duration) + { + createBranchCalls.incrementAndGet(); + createBranchTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordCreateTagCall(long duration) + { + createTagCalls.incrementAndGet(); + createTagTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropTagCall(long duration) + { + dropTagCalls.incrementAndGet(); + dropTagTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropConstraintCall(long duration) + { + dropConstraintCalls.incrementAndGet(); + dropConstraintTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordAddConstraintCall(long duration) + { + addConstraintCalls.incrementAndGet(); + addConstraintTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRenameTableCall(long duration) + { + renameTableCalls.incrementAndGet(); + renameTableTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordSetTablePropertiesCall(long duration) + { + setTablePropertiesCalls.incrementAndGet(); + setTablePropertiesTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordAddColumnCall(long duration) + { + addColumnCalls.incrementAndGet(); + addColumnTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordDropColumnCall(long duration) + { + dropColumnCalls.incrementAndGet(); + dropColumnTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordRenameColumnCall(long duration) + { + renameColumnCalls.incrementAndGet(); + renameColumnTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordNormalizeIdentifierCall(long duration) + { + normalizeIdentifierCalls.incrementAndGet(); + normalizeIdentifierTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTableLayoutFilterCoverageCall(long duration) + { + getTableLayoutFilterCoverageCalls.incrementAndGet(); + getTableLayoutFilterCoverageTime.add(duration, TimeUnit.NANOSECONDS); + } + + public void recordGetTableStatisticsCall(long duration) + { + getTableStatisticsCalls.incrementAndGet(); + getTableStatisticsTime.add(duration, TimeUnit.NANOSECONDS); + } + + @Managed + public long getGetCatalogNamesWithConnectorContextCalls() + { + return getCatalogNamesWithConnectorContextCalls.get(); + } + + @Managed + @Nested + public TimeStat getGetCatalogNamesWithConnectorContextTime() + { + return getCatalogNamesWithConnectorContextTime; + } + + public void recordGetCatalogNamesWithConnectorContextCall(long duration) + { + getCatalogNamesWithConnectorContextCalls.incrementAndGet(); + getCatalogNamesWithConnectorContextTime.add(duration, TimeUnit.NANOSECONDS); + } + + @Managed + public long getIsPushdownSupportedForFilterCalls() + { + return isPushdownSupportedForFilterCalls.get(); + } + + @Managed + @Nested + public TimeStat getIsPushdownSupportedForFilterTime() + { + return isPushdownSupportedForFilterTime; + } + + public void recordIsPushdownSupportedForFilterCall(long duration) + { + isPushdownSupportedForFilterCalls.incrementAndGet(); + isPushdownSupportedForFilterTime.add(duration, TimeUnit.NANOSECONDS); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/StatsRecordingMetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/StatsRecordingMetadataManager.java new file mode 100644 index 0000000000000..1fa6ea8ffd3a1 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/StatsRecordingMetadataManager.java @@ -0,0 +1,1459 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.Session; +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.block.BlockEncodingSerde; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewStatus; +import com.facebook.presto.spi.MergeHandle; +import com.facebook.presto.spi.NewTableLayout; +import com.facebook.presto.spi.SystemTable; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.TableLayoutFilterCoverage; +import com.facebook.presto.spi.TableMetadata; +import com.facebook.presto.spi.analyzer.MetadataResolver; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.connector.ConnectorCapabilities; +import com.facebook.presto.spi.connector.ConnectorOutputMetadata; +import com.facebook.presto.spi.connector.ConnectorTableVersion; +import com.facebook.presto.spi.connector.RowChangeParadigm; +import com.facebook.presto.spi.connector.TableFunctionApplicationResult; +import com.facebook.presto.spi.constraints.TableConstraint; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.plan.PartitioningHandle; +import com.facebook.presto.spi.procedure.ProcedureRegistry; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.security.GrantInfo; +import com.facebook.presto.spi.security.PrestoPrincipal; +import com.facebook.presto.spi.security.Privilege; +import com.facebook.presto.spi.security.RoleGrant; +import com.facebook.presto.spi.statistics.ComputedStatistics; +import com.facebook.presto.spi.statistics.TableStatistics; +import com.facebook.presto.spi.statistics.TableStatisticsMetadata; +import com.google.common.util.concurrent.ListenableFuture; +import io.airlift.slice.Slice; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +public class StatsRecordingMetadataManager + implements Metadata +{ + private final Metadata delegate; + private final MetadataManagerStats stats; + + public StatsRecordingMetadataManager(Metadata delegate, MetadataManagerStats stats) + { + this.delegate = delegate; + this.stats = stats; + } + + @Override + public void createSchema(Session session, CatalogSchemaName schema, Map properties) + { + long startTime = System.nanoTime(); + try { + delegate.createSchema(session, schema, properties); + } + finally { + stats.recordCreateSchemaCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropSchema(Session session, CatalogSchemaName schema) + { + long startTime = System.nanoTime(); + try { + delegate.dropSchema(session, schema); + } + finally { + stats.recordDropSchemaCall(System.nanoTime() - startTime); + } + } + + @Override + public void renameSchema(Session session, CatalogSchemaName source, String target) + { + long startTime = System.nanoTime(); + try { + delegate.renameSchema(session, source, target); + } + finally { + stats.recordRenameSchemaCall(System.nanoTime() - startTime); + } + } + + @Override + public void createTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) + { + long startTime = System.nanoTime(); + try { + delegate.createTable(session, catalogName, tableMetadata, ignoreExisting); + } + finally { + stats.recordCreateTableCall(System.nanoTime() - startTime); + } + } + + @Override + public TableHandle createTemporaryTable(Session session, String catalogName, List columns, Optional partitioningMetadata) + { + long startTime = System.nanoTime(); + try { + return delegate.createTemporaryTable(session, catalogName, columns, partitioningMetadata); + } + finally { + stats.recordCreateTemporaryTableCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropTable(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + delegate.dropTable(session, tableHandle); + } + finally { + stats.recordDropTableCall(System.nanoTime() - startTime); + } + } + + @Override + public void truncateTable(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + delegate.truncateTable(session, tableHandle); + } + finally { + stats.recordTruncateTableCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata) + { + long startTime = System.nanoTime(); + try { + return delegate.getNewTableLayout(session, catalogName, tableMetadata); + } + finally { + stats.recordGetNewTableLayoutCall(System.nanoTime() - startTime); + } + } + + @Override + public OutputTableHandle beginCreateTable(Session session, String catalogName, ConnectorTableMetadata tableMetadata, Optional layout) + { + long startTime = System.nanoTime(); + try { + return delegate.beginCreateTable(session, catalogName, tableMetadata, layout); + } + finally { + stats.recordBeginCreateTableCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional finishCreateTable(Session session, OutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) + { + long startTime = System.nanoTime(); + try { + return delegate.finishCreateTable(session, tableHandle, fragments, computedStatistics); + } + finally { + stats.recordFinishCreateTableCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getInsertLayout(Session session, TableHandle target) + { + long startTime = System.nanoTime(); + try { + return delegate.getInsertLayout(session, target); + } + finally { + stats.recordGetInsertLayoutCall(System.nanoTime() - startTime); + } + } + + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, String catalogName, ConnectorTableMetadata tableMetadata) + { + long startTime = System.nanoTime(); + try { + return delegate.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata); + } + finally { + stats.recordGetStatisticsCollectionMetadataForWriteCall(System.nanoTime() - startTime); + } + } + + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadata(Session session, String catalogName, ConnectorTableMetadata tableMetadata) + { + long startTime = System.nanoTime(); + try { + return delegate.getStatisticsCollectionMetadata(session, catalogName, tableMetadata); + } + finally { + stats.recordGetStatisticsCollectionMetadataCall(System.nanoTime() - startTime); + } + } + + @Override + public AnalyzeTableHandle beginStatisticsCollection(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.beginStatisticsCollection(session, tableHandle); + } + finally { + stats.recordBeginStatisticsCollectionCall(System.nanoTime() - startTime); + } + } + + @Override + public void finishStatisticsCollection(Session session, AnalyzeTableHandle tableHandle, Collection computedStatistics) + { + long startTime = System.nanoTime(); + try { + delegate.finishStatisticsCollection(session, tableHandle, computedStatistics); + } + finally { + stats.recordFinishStatisticsCollectionCall(System.nanoTime() - startTime); + } + } + + @Override + public void beginQuery(Session session, Set connectors) + { + long startTime = System.nanoTime(); + try { + delegate.beginQuery(session, connectors); + } + finally { + stats.recordBeginQueryCall(System.nanoTime() - startTime); + } + } + + @Override + public void cleanupQuery(Session session) + { + long startTime = System.nanoTime(); + try { + delegate.cleanupQuery(session); + } + finally { + stats.recordCleanupQueryCall(System.nanoTime() - startTime); + } + } + + @Override + public InsertTableHandle beginInsert(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.beginInsert(session, tableHandle); + } + finally { + stats.recordBeginInsertCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional finishInsert(Session session, InsertTableHandle tableHandle, Collection fragments, Collection computedStatistics) + { + long startTime = System.nanoTime(); + try { + return delegate.finishInsert(session, tableHandle, fragments, computedStatistics); + } + finally { + stats.recordFinishInsertCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getDeleteRowIdColumn(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getDeleteRowIdColumn(session, tableHandle); + } + finally { + stats.recordGetDeleteRowIdColumnCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getUpdateRowIdColumn(Session session, TableHandle tableHandle, List updatedColumns) + { + long startTime = System.nanoTime(); + try { + return delegate.getUpdateRowIdColumn(session, tableHandle, updatedColumns); + } + finally { + stats.recordGetUpdateRowIdColumnCall(System.nanoTime() - startTime); + } + } + + @Override + public boolean supportsMetadataDelete(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.supportsMetadataDelete(session, tableHandle); + } + finally { + stats.recordSupportsMetadataDeleteCall(System.nanoTime() - startTime); + } + } + + @Override + public OptionalLong metadataDelete(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.metadataDelete(session, tableHandle); + } + finally { + stats.recordMetadataDeleteCall(System.nanoTime() - startTime); + } + } + + @Override + public DeleteTableHandle beginDelete(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.beginDelete(session, tableHandle); + } + finally { + stats.recordBeginDeleteCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional finishDeleteWithOutput(Session session, DeleteTableHandle tableHandle, Collection fragments) + { + long startTime = System.nanoTime(); + try { + return delegate.finishDeleteWithOutput(session, tableHandle, fragments); + } + finally { + stats.recordFinishDeleteWithOutputCall(System.nanoTime() - startTime); + } + } + + @Override + public DistributedProcedureHandle beginCallDistributedProcedure(Session session, QualifiedObjectName procedureName, TableHandle tableHandle, Object[] arguments, boolean sourceTableEliminated) + { + long startTime = System.nanoTime(); + try { + return delegate.beginCallDistributedProcedure(session, procedureName, tableHandle, arguments, sourceTableEliminated); + } + finally { + stats.recordBeginCallDistributedProcedureCall(System.nanoTime() - startTime); + } + } + + @Override + public void finishCallDistributedProcedure(Session session, DistributedProcedureHandle procedureHandle, QualifiedObjectName procedureName, Collection fragments) + { + long startTime = System.nanoTime(); + try { + delegate.finishCallDistributedProcedure(session, procedureHandle, procedureName, fragments); + } + finally { + stats.recordFinishCallDistributedProcedureCall(System.nanoTime() - startTime); + } + } + + @Override + public TableHandle beginUpdate(Session session, TableHandle tableHandle, List updatedColumns) + { + long startTime = System.nanoTime(); + try { + return delegate.beginUpdate(session, tableHandle, updatedColumns); + } + finally { + stats.recordBeginUpdateCall(System.nanoTime() - startTime); + } + } + + @Override + public void finishUpdate(Session session, TableHandle tableHandle, Collection fragments) + { + long startTime = System.nanoTime(); + try { + delegate.finishUpdate(session, tableHandle, fragments); + } + finally { + stats.recordFinishUpdateCall(System.nanoTime() - startTime); + } + } + + @Override + public RowChangeParadigm getRowChangeParadigm(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getRowChangeParadigm(session, tableHandle); + } + finally { + stats.recordGetRowChangeParadigmCall(System.nanoTime() - startTime); + } + } + + @Override + public ColumnHandle getMergeTargetTableRowIdColumnHandle(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getMergeTargetTableRowIdColumnHandle(session, tableHandle); + } + finally { + stats.recordGetMergeTargetTableRowIdColumnHandleCall(System.nanoTime() - startTime); + } + } + + @Override + public MergeHandle beginMerge(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.beginMerge(session, tableHandle); + } + finally { + stats.recordBeginMergeCall(System.nanoTime() - startTime); + } + } + + @Override + public void finishMerge(Session session, MergeHandle tableHandle, Collection fragments, Collection computedStatistics) + { + long startTime = System.nanoTime(); + try { + delegate.finishMerge(session, tableHandle, fragments, computedStatistics); + } + finally { + stats.recordFinishMergeCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getCatalogHandle(Session session, String catalogName) + { + long startTime = System.nanoTime(); + try { + return delegate.getCatalogHandle(session, catalogName); + } + finally { + stats.recordGetCatalogHandleCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getCatalogNames(Session session) + { + long startTime = System.nanoTime(); + try { + return delegate.getCatalogNames(session); + } + finally { + stats.recordGetCatalogNamesCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getCatalogNamesWithConnectorContext(Session session) + { + long startTime = System.nanoTime(); + try { + return delegate.getCatalogNamesWithConnectorContext(session); + } + finally { + stats.recordGetCatalogNamesWithConnectorContextCall(System.nanoTime() - startTime); + } + } + + @Override + public List listViews(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.listViews(session, prefix); + } + finally { + stats.recordListViewsCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getViews(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.getViews(session, prefix); + } + finally { + stats.recordGetViewsCall(System.nanoTime() - startTime); + } + } + + @Override + public void createView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, String viewData, boolean replace) + { + long startTime = System.nanoTime(); + try { + delegate.createView(session, catalogName, viewMetadata, viewData, replace); + } + finally { + stats.recordCreateViewCall(System.nanoTime() - startTime); + } + } + + @Override + public void renameView(Session session, QualifiedObjectName existingViewName, QualifiedObjectName newViewName) + { + long startTime = System.nanoTime(); + try { + delegate.renameView(session, existingViewName, newViewName); + } + finally { + stats.recordRenameViewCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropView(Session session, QualifiedObjectName viewName) + { + long startTime = System.nanoTime(); + try { + delegate.dropView(session, viewName); + } + finally { + stats.recordDropViewCall(System.nanoTime() - startTime); + } + } + + @Override + public void createMaterializedView(Session session, String catalogName, ConnectorTableMetadata viewMetadata, MaterializedViewDefinition viewDefinition, boolean ignoreExisting) + { + long startTime = System.nanoTime(); + try { + delegate.createMaterializedView(session, catalogName, viewMetadata, viewDefinition, ignoreExisting); + } + finally { + stats.recordCreateMaterializedViewCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropMaterializedView(Session session, QualifiedObjectName viewName) + { + long startTime = System.nanoTime(); + try { + delegate.dropMaterializedView(session, viewName); + } + finally { + stats.recordDropMaterializedViewCall(System.nanoTime() - startTime); + } + } + + @Override + public List listMaterializedViews(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.listMaterializedViews(session, prefix); + } + finally { + stats.recordListMaterializedViewsCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getMaterializedViews(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.getMaterializedViews(session, prefix); + } + finally { + stats.recordGetMaterializedViewsCall(System.nanoTime() - startTime); + } + } + + @Override + public InsertTableHandle beginRefreshMaterializedView(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.beginRefreshMaterializedView(session, tableHandle); + } + finally { + stats.recordBeginRefreshMaterializedViewCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional finishRefreshMaterializedView(Session session, InsertTableHandle tableHandle, Collection fragments, Collection computedStatistics) + { + long startTime = System.nanoTime(); + try { + return delegate.finishRefreshMaterializedView(session, tableHandle, fragments, computedStatistics); + } + finally { + stats.recordFinishRefreshMaterializedViewCall(System.nanoTime() - startTime); + } + } + + @Override + public List getReferencedMaterializedViews(Session session, QualifiedObjectName tableName) + { + long startTime = System.nanoTime(); + try { + return delegate.getReferencedMaterializedViews(session, tableName); + } + finally { + stats.recordGetReferencedMaterializedViewsCall(System.nanoTime() - startTime); + } + } + + @Override + public MaterializedViewStatus getMaterializedViewStatus(Session session, QualifiedObjectName viewName, TupleDomain baseQueryDomain) + { + long startTime = System.nanoTime(); + try { + return delegate.getMaterializedViewStatus(session, viewName, baseQueryDomain); + } + finally { + stats.recordGetMaterializedViewStatusCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional resolveIndex(Session session, TableHandle tableHandle, Set indexableColumns, Set outputColumns, TupleDomain tupleDomain) + { + long startTime = System.nanoTime(); + try { + return delegate.resolveIndex(session, tableHandle, indexableColumns, outputColumns, tupleDomain); + } + finally { + stats.recordResolveIndexCall(System.nanoTime() - startTime); + } + } + + @Override + public void createRole(Session session, String role, Optional grantor, String catalog) + { + long startTime = System.nanoTime(); + try { + delegate.createRole(session, role, grantor, catalog); + } + finally { + stats.recordCreateRoleCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropRole(Session session, String role, String catalog) + { + long startTime = System.nanoTime(); + try { + delegate.dropRole(session, role, catalog); + } + finally { + stats.recordDropRoleCall(System.nanoTime() - startTime); + } + } + + @Override + public Set listRoles(Session session, String catalog) + { + long startTime = System.nanoTime(); + try { + return delegate.listRoles(session, catalog); + } + finally { + stats.recordListRolesCall(System.nanoTime() - startTime); + } + } + + @Override + public Set listRoleGrants(Session session, String catalog, PrestoPrincipal principal) + { + long startTime = System.nanoTime(); + try { + return delegate.listRoleGrants(session, catalog, principal); + } + finally { + stats.recordListRoleGrantsCall(System.nanoTime() - startTime); + } + } + + @Override + public void grantRoles(Session session, Set roles, Set grantees, boolean withAdminOption, Optional grantor, String catalog) + { + long startTime = System.nanoTime(); + try { + delegate.grantRoles(session, roles, grantees, withAdminOption, grantor, catalog); + } + finally { + stats.recordGrantRolesCall(System.nanoTime() - startTime); + } + } + + @Override + public void revokeRoles(Session session, Set roles, Set grantees, boolean adminOptionFor, Optional grantor, String catalog) + { + long startTime = System.nanoTime(); + try { + delegate.revokeRoles(session, roles, grantees, adminOptionFor, grantor, catalog); + } + finally { + stats.recordRevokeRolesCall(System.nanoTime() - startTime); + } + } + + @Override + public Set listApplicableRoles(Session session, PrestoPrincipal principal, String catalog) + { + long startTime = System.nanoTime(); + try { + return delegate.listApplicableRoles(session, principal, catalog); + } + finally { + stats.recordListApplicableRolesCall(System.nanoTime() - startTime); + } + } + + @Override + public Set listEnabledRoles(Session session, String catalog) + { + long startTime = System.nanoTime(); + try { + return delegate.listEnabledRoles(session, catalog); + } + finally { + stats.recordListEnabledRolesCall(System.nanoTime() - startTime); + } + } + + @Override + public void grantTablePrivileges(Session session, QualifiedObjectName tableName, Set privileges, PrestoPrincipal grantee, boolean grantOption) + { + long startTime = System.nanoTime(); + try { + delegate.grantTablePrivileges(session, tableName, privileges, grantee, grantOption); + } + finally { + stats.recordGrantTablePrivilegesCall(System.nanoTime() - startTime); + } + } + + @Override + public void revokeTablePrivileges(Session session, QualifiedObjectName tableName, Set privileges, PrestoPrincipal grantee, boolean grantOption) + { + long startTime = System.nanoTime(); + try { + delegate.revokeTablePrivileges(session, tableName, privileges, grantee, grantOption); + } + finally { + stats.recordRevokeTablePrivilegesCall(System.nanoTime() - startTime); + } + } + + @Override + public List listTablePrivileges(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.listTablePrivileges(session, prefix); + } + finally { + stats.recordListTablePrivilegesCall(System.nanoTime() - startTime); + } + } + + @Override + public ListenableFuture commitPageSinkAsync(Session session, OutputTableHandle tableHandle, Collection fragments) + { + long startTime = System.nanoTime(); + try { + return delegate.commitPageSinkAsync(session, tableHandle, fragments); + } + finally { + stats.recordCommitPageSinkAsyncCall(System.nanoTime() - startTime); + } + } + + @Override + public ListenableFuture commitPageSinkAsync(Session session, InsertTableHandle tableHandle, Collection fragments) + { + long startTime = System.nanoTime(); + try { + return delegate.commitPageSinkAsync(session, tableHandle, fragments); + } + finally { + stats.recordCommitPageSinkAsyncCall(System.nanoTime() - startTime); + } + } + + @Override + public ListenableFuture commitPageSinkAsync(Session session, DeleteTableHandle tableHandle, Collection fragments) + { + long startTime = System.nanoTime(); + try { + return delegate.commitPageSinkAsync(session, tableHandle, fragments); + } + finally { + stats.recordCommitPageSinkAsyncCall(System.nanoTime() - startTime); + } + } + + @Override + public FunctionAndTypeManager getFunctionAndTypeManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getFunctionAndTypeManager(); + } + finally { + stats.recordGetFunctionAndTypeManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public ProcedureRegistry getProcedureRegistry() + { + long startTime = System.nanoTime(); + try { + return delegate.getProcedureRegistry(); + } + finally { + stats.recordGetProcedureRegistryCall(System.nanoTime() - startTime); + } + } + + @Override + public BlockEncodingSerde getBlockEncodingSerde() + { + long startTime = System.nanoTime(); + try { + return delegate.getBlockEncodingSerde(); + } + finally { + stats.recordGetBlockEncodingSerdeCall(System.nanoTime() - startTime); + } + } + + @Override + public SessionPropertyManager getSessionPropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getSessionPropertyManager(); + } + finally { + stats.recordGetSessionPropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public SchemaPropertyManager getSchemaPropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getSchemaPropertyManager(); + } + finally { + stats.recordGetSchemaPropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public TablePropertyManager getTablePropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getTablePropertyManager(); + } + finally { + stats.recordGetTablePropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public MaterializedViewPropertyManager getMaterializedViewPropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getMaterializedViewPropertyManager(); + } + finally { + stats.recordGetTablePropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public ColumnPropertyManager getColumnPropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getColumnPropertyManager(); + } + finally { + stats.recordGetColumnPropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public AnalyzePropertyManager getAnalyzePropertyManager() + { + long startTime = System.nanoTime(); + try { + return delegate.getAnalyzePropertyManager(); + } + finally { + stats.recordGetAnalyzePropertyManagerCall(System.nanoTime() - startTime); + } + } + + @Override + public MetadataResolver getMetadataResolver(Session session) + { + long startTime = System.nanoTime(); + try { + return delegate.getMetadataResolver(session); + } + finally { + stats.recordGetMetadataResolverCall(System.nanoTime() - startTime); + } + } + + @Override + public Set getConnectorCapabilities(Session session, ConnectorId catalogName) + { + long startTime = System.nanoTime(); + try { + return delegate.getConnectorCapabilities(session, catalogName); + } + finally { + stats.recordGetConnectorCapabilitiesCall(System.nanoTime() - startTime); + } + } + + @Override + public TableLayoutFilterCoverage getTableLayoutFilterCoverage(Session session, TableHandle tableHandle, Set relevantPartitionColumn) + { + long startTime = System.nanoTime(); + try { + return delegate.getTableLayoutFilterCoverage(session, tableHandle, relevantPartitionColumn); + } + finally { + stats.recordGetTableLayoutFilterCoverageCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropBranch(Session session, TableHandle tableHandle, String branchName, boolean branchExists) + { + long startTime = System.nanoTime(); + try { + delegate.dropBranch(session, tableHandle, branchName, branchExists); + } + finally { + stats.recordDropBranchCall(System.nanoTime() - startTime); + } + } + + @Override + public void createBranch(Session session, + TableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + long startTime = System.nanoTime(); + try { + delegate.createBranch(session, tableHandle, branchName, replace, ifNotExists, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + finally { + stats.recordCreateBranchCall(System.nanoTime() - startTime); + } + } + + @Override + public void createTag(Session session, + TableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays) + { + long startTime = System.nanoTime(); + try { + delegate.createTag(session, tableHandle, tagName, replace, ifNotExists, tableVersion, retainDays); + } + finally { + stats.recordCreateTagCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropTag(Session session, TableHandle tableHandle, String tagName, boolean tagExists) + { + long startTime = System.nanoTime(); + try { + delegate.dropTag(session, tableHandle, tagName, tagExists); + } + finally { + stats.recordDropTagCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropConstraint(Session session, TableHandle tableHandle, Optional constraintName, Optional columnName) + { + long startTime = System.nanoTime(); + try { + delegate.dropConstraint(session, tableHandle, constraintName, columnName); + } + finally { + stats.recordDropConstraintCall(System.nanoTime() - startTime); + } + } + + @Override + public void addConstraint(Session session, TableHandle tableHandle, TableConstraint tableConstraint) + { + long startTime = System.nanoTime(); + try { + delegate.addConstraint(session, tableHandle, tableConstraint); + } + finally { + stats.recordAddConstraintCall(System.nanoTime() - startTime); + } + } + + @Override + public boolean isPushdownSupportedForFilter(Session session, TableHandle tableHandle, RowExpression filter, Map symbolToColumnHandleMap) + { + long startTime = System.nanoTime(); + try { + return delegate.isPushdownSupportedForFilter(session, tableHandle, filter, symbolToColumnHandleMap); + } + finally { + stats.recordIsPushdownSupportedForFilterCall(System.nanoTime() - startTime); + } + } + + @Override + public void renameTable(Session session, TableHandle tableHandle, QualifiedObjectName newTableName) + { + long startTime = System.nanoTime(); + try { + delegate.renameTable(session, tableHandle, newTableName); + } + finally { + stats.recordRenameTableCall(System.nanoTime() - startTime); + } + } + + @Override + public void setTableProperties(Session session, TableHandle tableHandle, Map properties) + { + long startTime = System.nanoTime(); + try { + delegate.setTableProperties(session, tableHandle, properties); + } + finally { + stats.recordSetTablePropertiesCall(System.nanoTime() - startTime); + } + } + + @Override + public void addColumn(Session session, TableHandle tableHandle, ColumnMetadata column) + { + long startTime = System.nanoTime(); + try { + delegate.addColumn(session, tableHandle, column); + } + finally { + stats.recordAddColumnCall(System.nanoTime() - startTime); + } + } + + @Override + public void dropColumn(Session session, TableHandle tableHandle, ColumnHandle column) + { + long startTime = System.nanoTime(); + try { + delegate.dropColumn(session, tableHandle, column); + } + finally { + stats.recordDropColumnCall(System.nanoTime() - startTime); + } + } + + @Override + public void renameColumn(Session session, TableHandle tableHandle, ColumnHandle source, String target) + { + long startTime = System.nanoTime(); + try { + delegate.renameColumn(session, tableHandle, source, target); + } + finally { + stats.recordRenameColumnCall(System.nanoTime() - startTime); + } + } + + @Override + public String normalizeIdentifier(Session session, String catalogName, String identifier) + { + long startTime = System.nanoTime(); + try { + return delegate.normalizeIdentifier(session, catalogName, identifier); + } + finally { + stats.recordNormalizeIdentifierCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional> applyTableFunction(Session session, TableFunctionHandle handle) + { + long startTime = System.nanoTime(); + try { + return delegate.applyTableFunction(session, handle); + } + finally { + stats.recordApplyTableFunctionCall(System.nanoTime() - startTime); + } + } + + @Override + public void verifyComparableOrderableContract() + { + long startTime = System.nanoTime(); + try { + delegate.verifyComparableOrderableContract(); + } + finally { + stats.recordVerifyComparableOrderableContractCall(System.nanoTime() - startTime); + } + } + + @Override + public Type getType(TypeSignature signature) + { + long startTime = System.nanoTime(); + try { + return delegate.getType(signature); + } + finally { + stats.recordGetTypeCall(System.nanoTime() - startTime); + } + } + + @Override + public void registerBuiltInFunctions(List functions) + { + long startTime = System.nanoTime(); + try { + delegate.registerBuiltInFunctions(functions); + } + finally { + stats.recordRegisterBuiltInFunctionsCall(System.nanoTime() - startTime); + } + } + + @Override + public void registerConnectorFunctions(String catalogName, List functionInfos) + { + long startTime = System.nanoTime(); + try { + delegate.registerConnectorFunctions(catalogName, functionInfos); + } + finally { + stats.recordRegisterConnectorFunctionsCall(System.nanoTime() - startTime); + } + } + + @Override + public List listSchemaNames(Session session, String catalogName) + { + long startTime = System.nanoTime(); + try { + return delegate.listSchemaNames(session, catalogName); + } + finally { + stats.recordListSchemaNamesCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getSchemaProperties(Session session, CatalogSchemaName schemaName) + { + long startTime = System.nanoTime(); + try { + return delegate.getSchemaProperties(session, schemaName); + } + finally { + stats.recordGetSchemaPropertiesCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getSystemTable(Session session, QualifiedObjectName tableName) + { + long startTime = System.nanoTime(); + try { + return delegate.getSystemTable(session, tableName); + } + finally { + stats.recordGetSystemTableCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getHandleVersion(Session session, QualifiedObjectName tableName, Optional tableVersion) + { + long startTime = System.nanoTime(); + try { + return delegate.getHandleVersion(session, tableName, tableVersion); + } + finally { + stats.recordGetHandleVersionCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getTableHandleForStatisticsCollection(Session session, QualifiedObjectName tableName, Map analyzeProperties) + { + long startTime = System.nanoTime(); + try { + return delegate.getTableHandleForStatisticsCollection(session, tableName, analyzeProperties); + } + finally { + stats.recordGetTableHandleForStatisticsCollectionCall(System.nanoTime() - startTime); + } + } + + @Override + public TableLayoutResult getLayout(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns) + { + long startTime = System.nanoTime(); + try { + return delegate.getLayout(session, tableHandle, constraint, desiredColumns); + } + finally { + stats.recordGetLayoutCall(System.nanoTime() - startTime); + } + } + + @Override + public TableLayout getLayout(Session session, TableHandle handle) + { + long startTime = System.nanoTime(); + try { + return delegate.getLayout(session, handle); + } + finally { + stats.recordGetLayoutCall(System.nanoTime() - startTime); + } + } + + @Override + public TableHandle getAlternativeTableHandle(Session session, TableHandle tableHandle, PartitioningHandle partitioningHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getAlternativeTableHandle(session, tableHandle, partitioningHandle); + } + finally { + stats.recordGetAlternativeTableHandleCall(System.nanoTime() - startTime); + } + } + + @Override + public boolean isLegacyGetLayoutSupported(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.isLegacyGetLayoutSupported(session, tableHandle); + } + finally { + stats.recordIsLegacyGetLayoutSupportedCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getCommonPartitioning(Session session, PartitioningHandle left, PartitioningHandle right) + { + long startTime = System.nanoTime(); + try { + return delegate.getCommonPartitioning(session, left, right); + } + finally { + stats.recordGetCommonPartitioningCall(System.nanoTime() - startTime); + } + } + + @Override + public boolean isRefinedPartitioningOver(Session session, PartitioningHandle a, PartitioningHandle b) + { + long startTime = System.nanoTime(); + try { + return delegate.isRefinedPartitioningOver(session, a, b); + } + finally { + stats.recordIsRefinedPartitioningOverCall(System.nanoTime() - startTime); + } + } + + @Override + public PartitioningHandle getPartitioningHandleForExchange(Session session, String catalogName, int partitionCount, List partitionTypes) + { + long startTime = System.nanoTime(); + try { + return delegate.getPartitioningHandleForExchange(session, catalogName, partitionCount, partitionTypes); + } + finally { + stats.recordGetPartitioningHandleForExchangeCall(System.nanoTime() - startTime); + } + } + + @Override + public Optional getInfo(Session session, TableHandle handle) + { + long startTime = System.nanoTime(); + try { + return delegate.getInfo(session, handle); + } + finally { + stats.recordGetInfoCall(System.nanoTime() - startTime); + } + } + + @Override + public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getTableMetadata(session, tableHandle); + } + finally { + stats.recordGetTableMetadataCall(System.nanoTime() - startTime); + } + } + + @Override + public TableStatistics getTableStatistics(Session session, TableHandle tableHandle, List columnHandles, Constraint constraint) + { + long startTime = System.nanoTime(); + try { + return delegate.getTableStatistics(session, tableHandle, columnHandles, constraint); + } + finally { + stats.recordGetTableStatisticsCall(System.nanoTime() - startTime); + } + } + + @Override + public List listTables(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.listTables(session, prefix); + } + finally { + stats.recordListTablesCall(System.nanoTime() - startTime); + } + } + + @Override + public Map getColumnHandles(Session session, TableHandle tableHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getColumnHandles(session, tableHandle); + } + finally { + stats.recordGetColumnHandlesCall(System.nanoTime() - startTime); + } + } + + @Override + public ColumnMetadata getColumnMetadata(Session session, TableHandle tableHandle, ColumnHandle columnHandle) + { + long startTime = System.nanoTime(); + try { + return delegate.getColumnMetadata(session, tableHandle, columnHandle); + } + finally { + stats.recordGetColumnMetadataCall(System.nanoTime() - startTime); + } + } + + @Override + public TupleDomain toExplainIOConstraints(Session session, TableHandle tableHandle, TupleDomain constraints) + { + long startTime = System.nanoTime(); + try { + return delegate.toExplainIOConstraints(session, tableHandle, constraints); + } + finally { + stats.recordToExplainIOConstraintsCall(System.nanoTime() - startTime); + } + } + + @Override + public Map> listTableColumns(Session session, QualifiedTablePrefix prefix) + { + long startTime = System.nanoTime(); + try { + return delegate.listTableColumns(session, prefix); + } + finally { + stats.recordListTableColumnsCall(System.nanoTime() - startTime); + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/PageBuffer.java b/presto-main-base/src/main/java/com/facebook/presto/operator/PageBuffer.java index 82fe581a61db3..ed68df6f8d0c9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/PageBuffer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/PageBuffer.java @@ -18,7 +18,6 @@ import static com.facebook.presto.operator.WorkProcessor.ProcessState.finished; import static com.facebook.presto.operator.WorkProcessor.ProcessState.ofResult; -import static com.facebook.presto.operator.WorkProcessor.ProcessState.yield; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -41,7 +40,7 @@ public WorkProcessor pages() return ofResult(result); } - return yield(); + return WorkProcessor.ProcessState.yield(); }); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java index 5f7f9a27d19f7..ace3200d78e8f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/IpPrefixFunctions.java @@ -32,15 +32,15 @@ import java.util.Comparator; import java.util.List; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.operator.scalar.ArraySortFunction.sort; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.type.IpAddressOperators.between; import static com.facebook.presto.type.IpAddressOperators.castFromVarcharToIpAddress; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.facebook.presto.type.IpPrefixOperators.castFromIpPrefixToIpAddress; import static com.facebook.presto.type.IpPrefixOperators.castFromVarcharToIpPrefix; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.util.Failures.checkCondition; import static io.airlift.slice.Slices.utf8Slice; import static io.airlift.slice.Slices.wrappedBuffer; diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java index 9aebc92850fd3..df034876a9af3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.common.block.Block; +import com.facebook.presto.common.function.SqlFunctionProperties; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.ScalarFunction; @@ -24,6 +25,7 @@ import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; import io.airlift.slice.Slice; +import java.util.Set; import java.util.function.Supplier; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; @@ -38,13 +40,13 @@ private TryFunction() {} @TypeParameterSpecialization(name = "T", nativeContainerType = long.class) @SqlNullable @SqlType("T") - public static Long tryLong(@SqlType("function(T)") TryLongLambda function) + public static Long tryLong(SqlFunctionProperties properties, @SqlType("function(T)") TryLongLambda function) { try { return function.apply(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, properties); return null; } } @@ -53,13 +55,13 @@ public static Long tryLong(@SqlType("function(T)") TryLongLambda function) @TypeParameterSpecialization(name = "T", nativeContainerType = double.class) @SqlNullable @SqlType("T") - public static Double tryDouble(@SqlType("function(T)") TryDoubleLambda function) + public static Double tryDouble(SqlFunctionProperties properties, @SqlType("function(T)") TryDoubleLambda function) { try { return function.apply(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, properties); return null; } } @@ -68,13 +70,13 @@ public static Double tryDouble(@SqlType("function(T)") TryDoubleLambda function) @TypeParameterSpecialization(name = "T", nativeContainerType = boolean.class) @SqlNullable @SqlType("T") - public static Boolean tryBoolean(@SqlType("function(T)") TryBooleanLambda function) + public static Boolean tryBoolean(SqlFunctionProperties properties, @SqlType("function(T)") TryBooleanLambda function) { try { return function.apply(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, properties); return null; } } @@ -83,13 +85,13 @@ public static Boolean tryBoolean(@SqlType("function(T)") TryBooleanLambda functi @TypeParameterSpecialization(name = "T", nativeContainerType = Slice.class) @SqlNullable @SqlType("T") - public static Slice trySlice(@SqlType("function(T)") TrySliceLambda function) + public static Slice trySlice(SqlFunctionProperties properties, @SqlType("function(T)") TrySliceLambda function) { try { return function.apply(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, properties); return null; } } @@ -98,13 +100,13 @@ public static Slice trySlice(@SqlType("function(T)") TrySliceLambda function) @TypeParameterSpecialization(name = "T", nativeContainerType = Block.class) @SqlNullable @SqlType("T") - public static Block tryBlock(@SqlType("function(T)") TryBlockLambda function) + public static Block tryBlock(SqlFunctionProperties properties, @SqlType("function(T)") TryBlockLambda function) { try { return function.apply(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, properties); return null; } } @@ -150,18 +152,27 @@ public static T evaluate(Supplier supplier, T defaultValue) return supplier.get(); } catch (PrestoException e) { - propagateIfUnhandled(e); + propagateIfUnhandled(e, null); return defaultValue; } } - private static void propagateIfUnhandled(PrestoException e) + private static void propagateIfUnhandled(PrestoException e, SqlFunctionProperties properties) throws PrestoException { + // Check if error is catchable by TRY function if (e.getErrorCode().isCatchableByTry()) { return; } + // Check if the error code is in the session-specified list of catchable errors + if (properties != null) { + Set catchableErrorCodes = properties.getTryCatchableErrorCodes(); + if (catchableErrorCodes != null && catchableErrorCodes.contains(e.getErrorCode().getName())) { + return; + } + } + throw e; } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java b/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java index 6bc0a9b1a197d..5b7fe3b943485 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java @@ -863,6 +863,38 @@ public void checkCanDropBranch(TransactionId transactionId, Identity identity, A } } + @Override + public void checkCanCreateBranch(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + + authorizationCheck(() -> systemAccessControl.checkCanCreateBranch(identity, context, toCatalogSchemaTableName(tableName))); + + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateBranch(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } + } + + @Override + public void checkCanCreateTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + requireNonNull(identity, "identity is null"); + requireNonNull(tableName, "tableName is null"); + + authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); + + authorizationCheck(() -> systemAccessControl.checkCanCreateTag(identity, context, toCatalogSchemaTableName(tableName))); + + CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); + if (entry != null) { + authorizationCheck(() -> entry.getAccessControl().checkCanCreateTag(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName))); + } + } + @Override public void checkCanDropTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/AllowAllSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/AllowAllSystemAccessControl.java index 2b7459ef6b1c6..b5c3441bb88d7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/AllowAllSystemAccessControl.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/AllowAllSystemAccessControl.java @@ -250,6 +250,16 @@ public void checkCanRevokeTablePrivilege(Identity identity, AccessControlContext { } + @Override + public void checkCanCreateBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + } + + @Override + public void checkCanCreateTag(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + } + @Override public void checkCanDropBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java index c6164210ea050..07412f651b902 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java @@ -94,6 +94,11 @@ public void checkCanCreateTable(Identity identity, AccessControlContext context, { } + @Override + public void checkCanCreateView(Identity identity, AccessControlContext context, CatalogSchemaTableName view) + { + } + @Override public void checkCanShowCreateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java index d248e838ff8ad..cbea6676a0dce 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/FileBasedSystemAccessControl.java @@ -57,8 +57,10 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; import static com.facebook.presto.spi.security.AccessDeniedException.denyCatalogAccess; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -479,6 +481,22 @@ public void checkCanRevokeTablePrivilege(Identity identity, AccessControlContext } } + @Override + public void checkCanCreateBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + if (!canAccessCatalog(identity, table.getCatalogName(), ALL)) { + denyCreateBranch(table.toString()); + } + } + + @Override + public void checkCanCreateTag(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + if (!canAccessCatalog(identity, table.getCatalogName(), ALL)) { + denyCreateTag(table.toString()); + } + } + @Override public void checkCanDropBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java index 6c57e4204f063..82865ce6de482 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java @@ -710,6 +710,42 @@ public void checkCanRevokeTablePrivilege(Identity identity, AccessControlContext } } + @Override + public void checkCanCreateBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateBranch(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanCreateBranch.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateBranch", RuntimeUnit.NANO, duration); + stats.checkCanCreateBranch.record(duration); + } + } + + @Override + public void checkCanCreateTag(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateTag(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanCreateTag.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateTag", RuntimeUnit.NANO, duration); + stats.checkCanCreateTag.record(duration); + } + } + @Override public void checkCanDropBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) { @@ -860,6 +896,8 @@ public static class Stats final SystemAccessControlStats checkCanDropTag = new SystemAccessControlStats(); final SystemAccessControlStats checkCanDropConstraint = new SystemAccessControlStats(); final SystemAccessControlStats checkCanAddConstraint = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateBranch = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateTag = new SystemAccessControlStats(); final SystemAccessControlStats getRowFilters = new SystemAccessControlStats(); final SystemAccessControlStats getColumnMasks = new SystemAccessControlStats(); @@ -954,6 +992,13 @@ public SystemAccessControlStats getCheckCanCreateTable() return checkCanCreateTable; } + @Managed + @Nested + public SystemAccessControlStats getCheckCanCreateBranch() + { + return checkCanCreateBranch; + } + @Managed @Nested public SystemAccessControlStats getCheckCanSetTableProperties() diff --git a/presto-main-base/src/main/java/com/facebook/presto/server/ServerConfig.java b/presto-main-base/src/main/java/com/facebook/presto/server/ServerConfig.java index 7e7bcbbffd1d4..e629733d916a1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/server/ServerConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/server/ServerConfig.java @@ -43,6 +43,7 @@ public class ServerConfig private boolean nestedDataSerializationEnabled = true; private Duration clusterResourceGroupStateInfoExpirationDuration = new Duration(0, MILLISECONDS); private String clusterTag; + private boolean webUIEnabled = true; public boolean isResourceManager() { @@ -116,6 +117,18 @@ public ServerConfig setCoordinator(boolean coordinator) return this; } + public boolean isWebUIEnabled() + { + return webUIEnabled; + } + + @Config("webui-enabled") + public ServerConfig setWebUIEnabled(boolean webUIEnabled) + { + this.webUIEnabled = webUIEnabled; + return this; + } + @NotNull(message = "presto.version must be provided when it cannot be automatically determined") public String getPrestoVersion() { diff --git a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 30d6818998cf5..054f458460b03 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -39,6 +39,8 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_SPILL_COMPRESSION_CODEC = "native_spill_compression_codec"; public static final String NATIVE_SPILL_WRITE_BUFFER_SIZE = "native_spill_write_buffer_size"; public static final String NATIVE_SPILL_FILE_CREATE_CONFIG = "native_spill_file_create_config"; + public static final String NATIVE_AGGREGATION_SPILL_FILE_CREATE_CONFIG = "native_aggregation_spill_file_create_config"; + public static final String NATIVE_HASH_JOIN_SPILL_FILE_CREATE_CONFIG = "native_hash_join_spill_file_create_config"; public static final String NATIVE_JOIN_SPILL_ENABLED = "native_join_spill_enabled"; public static final String NATIVE_WINDOW_SPILL_ENABLED = "native_window_spill_enabled"; public static final String NATIVE_WRITER_SPILL_ENABLED = "native_writer_spill_enabled"; @@ -62,6 +64,7 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_MAX_EXTENDED_PARTIAL_AGGREGATION_MEMORY = "native_max_extended_partial_aggregation_memory"; public static final String NATIVE_MAX_SPILL_BYTES = "native_max_spill_bytes"; public static final String NATIVE_MAX_PAGE_PARTITIONING_BUFFER_SIZE = "native_max_page_partitioning_buffer_size"; + public static final String NATIVE_PARTITIONED_OUTPUT_EAGER_FLUSH = "native_partitioned_output_eager_flush"; public static final String NATIVE_MAX_OUTPUT_BUFFER_SIZE = "native_max_output_buffer_size"; public static final String NATIVE_QUERY_TRACE_ENABLED = "native_query_trace_enabled"; public static final String NATIVE_QUERY_TRACE_DIR = "native_query_trace_dir"; @@ -90,6 +93,8 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_USE_VELOX_GEOSPATIAL_JOIN = "native_use_velox_geospatial_join"; public static final String NATIVE_AGGREGATION_COMPACTION_BYTES_THRESHOLD = "native_aggregation_compaction_bytes_threshold"; public static final String NATIVE_AGGREGATION_COMPACTION_UNUSED_MEMORY_RATIO = "native_aggregation_compaction_unused_memory_ratio"; + public static final String NATIVE_AGGREGATION_MEMORY_COMPACTION_RECLAIM_ENABLED = "native_aggregation_memory_compaction_reclaim_enabled"; + public static final String NATIVE_MERGE_JOIN_OUTPUT_BATCH_START_SIZE = "native_merge_join_output_batch_start_size"; private final List> sessionProperties; @@ -144,6 +149,20 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "defined by the underlying file system.", "", !nativeExecution), + stringProperty( + NATIVE_AGGREGATION_SPILL_FILE_CREATE_CONFIG, + "Native Execution only. Config used to create aggregation spill files. This config is \n" + + "provided to underlying file system and the config is free form. The form should be\n" + + "defined by the underlying file system.", + "", + !nativeExecution), + stringProperty( + NATIVE_HASH_JOIN_SPILL_FILE_CREATE_CONFIG, + "Native Execution only. Config used to create hash join spill files. This config is \n" + + "provided to underlying file system and the config is free form. The form should be\n" + + "defined by the underlying file system.", + "", + !nativeExecution), booleanProperty( NATIVE_JOIN_SPILL_ENABLED, "Native Execution only. Enable join spilling on native engine", @@ -317,6 +336,11 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "producing a SerializedPage.", 24L << 20, !nativeExecution), + booleanProperty(NATIVE_PARTITIONED_OUTPUT_EAGER_FLUSH, + "Native Execution only. If true, the PartitionedOutput operator will flush rows eagerly, without " + + "waiting until buffers reach certain size. Default is false.", + false, + !nativeExecution), integerProperty( NATIVE_MAX_LOCAL_EXCHANGE_PARTITION_COUNT, "Maximum number of partitions created by a local exchange. " + @@ -450,6 +474,21 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "The value is in the range of [0, 1). NOTE: Currently only applies to approx_most_frequent " + "aggregate with StringView type during global aggregation.", 0.25, + !nativeExecution), + booleanProperty( + NATIVE_AGGREGATION_MEMORY_COMPACTION_RECLAIM_ENABLED, + "If true, enables lightweight memory compaction before spilling during " + + "memory reclaim in aggregation. When enabled, the aggregation operator " + + "will try to compact aggregate function state before resorting to spilling.", + false, + !nativeExecution), + integerProperty( + NATIVE_MERGE_JOIN_OUTPUT_BATCH_START_SIZE, + "Initial output batch size in rows for MergeJoin operator. When non-zero, " + + "the batch size starts at this value and is dynamically adjusted based on " + + "the average row size of previous output batches. When zero (default), " + + "dynamic adjustment is disabled and the batch size is fixed at preferred_output_batch_rows.", + 0, !nativeExecution)); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java b/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java index dcc80d61ab1ad..98399ccc19ef4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java @@ -15,6 +15,7 @@ package com.facebook.presto.sql; import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.predicate.NullableValue; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.metadata.Metadata; @@ -40,7 +41,9 @@ import com.facebook.presto.sql.tree.IsNullPredicate; import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.Relation; import com.facebook.presto.sql.tree.SymbolReference; +import com.facebook.presto.sql.tree.Table; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -60,6 +63,7 @@ import static com.facebook.presto.common.predicate.TupleDomain.extractFixedValues; import static com.facebook.presto.common.type.StandardTypes.HYPER_LOG_LOG; import static com.facebook.presto.common.type.StandardTypes.VARBINARY; +import static com.facebook.presto.metadata.MetadataUtil.createQualifiedObjectName; import static com.facebook.presto.sql.ExpressionUtils.combineDisjuncts; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.tree.ArithmeticBinaryExpression.Operator.DIVIDE; @@ -399,6 +403,15 @@ public static Expression convertMaterializedDataPredicatesToExpression( } } + public static Relation resolveTableName(Relation relation, Session session, Metadata metadata) + { + if (!(relation instanceof Table)) { + return relation; + } + QualifiedObjectName qualifiedTableName = createQualifiedObjectName(session, relation, ((Table) relation).getName(), metadata); + return new Table(QualifiedName.of(qualifiedTableName.getSchemaName(), qualifiedTableName.getObjectName())); + } + private static Expression convertSymbolReferencesToIdentifiers(Expression expression) { return ExpressionTreeRewriter.rewriteWith(new ExpressionRewriter() diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java index d46addaccf585..10d6c37bdec79 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java @@ -17,7 +17,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.spi.analyzer.AccessControlReferences; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.parser.SqlParser; @@ -46,8 +46,6 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.CANNOT_HAVE_AGGREGATIONS_WINDOWS_OR_GROUPING; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.UtilizedColumnsAnalyzer.analyzeForUtilizedColumns; -import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForColumns; -import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForTable; import static java.util.Objects.requireNonNull; public class Analyzer @@ -62,6 +60,7 @@ public class Analyzer private final WarningCollector warningCollector; private final MetadataExtractor metadataExtractor; private final String query; + private final ViewDefinitionReferences viewDefinitionReferences; public Analyzer( Session session, @@ -72,9 +71,10 @@ public Analyzer( List parameters, Map, Expression> parameterLookup, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { - this(session, metadata, sqlParser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, Optional.empty(), query); + this(session, metadata, sqlParser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, Optional.empty(), query, viewDefinitionReferences); } public Analyzer( @@ -87,7 +87,8 @@ public Analyzer( Map, Expression> parameterLookup, WarningCollector warningCollector, Optional metadataExtractorExecutor, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { this.session = requireNonNull(session, "session is null"); this.metadata = requireNonNull(metadata, "metadata is null"); @@ -100,21 +101,7 @@ public Analyzer( requireNonNull(metadataExtractorExecutor, "metadataExtractorExecutor is null"); this.metadataExtractor = new MetadataExtractor(session, metadata, metadataExtractorExecutor, sqlParser, warningCollector); this.query = requireNonNull(query, "query is null"); - } - - public Analysis analyze(Statement statement) - { - return analyze(statement, false); - } - - // TODO: Remove this method once all calls are moved to analyzer interface, as this call is overloaded with analyze and columnCheckPermissions - public Analysis analyze(Statement statement, boolean isDescribe) - { - Analysis analysis = analyzeSemantic(statement, isDescribe); - AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); - checkAccessPermissionsForTable(accessControlReferences); - checkAccessPermissionsForColumns(accessControlReferences); - return analysis; + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } public Analysis analyzeSemantic(Statement statement, boolean isDescribe) @@ -127,8 +114,8 @@ public Analysis analyzeSemantic( Optional procedureName, boolean isDescribe) { - Statement rewrittenStatement = StatementRewrite.rewrite(session, metadata, sqlParser, queryExplainer, statement, parameters, parameterLookup, accessControl, warningCollector, query); - Analysis analysis = new Analysis(rewrittenStatement, parameterLookup, isDescribe); + Statement rewrittenStatement = StatementRewrite.rewrite(session, metadata, sqlParser, queryExplainer, statement, parameters, parameterLookup, accessControl, warningCollector, query, viewDefinitionReferences); + Analysis analysis = new Analysis(rewrittenStatement, parameterLookup, isDescribe, viewDefinitionReferences); metadataExtractor.populateMetadataHandle(session, rewrittenStatement, analysis.getMetadataHandle()); analysis.setProcedureName(procedureName); @@ -139,6 +126,11 @@ public Analysis analyzeSemantic( return analysis; } + public ViewDefinitionReferences getViewDefinitionReferences() + { + return viewDefinitionReferences; + } + static void verifyNoAggregateWindowOrGroupingFunctions( Map, FunctionHandle> functionHandles, FunctionAndTypeResolver functionAndTypeResolver, diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java index d5ab8c544c14c..aa2002e7cfd88 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryAnalyzer.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.analyzer.MetadataResolver; import com.facebook.presto.spi.analyzer.QueryAnalysis; import com.facebook.presto.spi.analyzer.QueryAnalyzer; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; @@ -90,7 +91,8 @@ public QueryAnalysis analyze(AnalyzerContext analyzerContext, PreparedQuery prep parameterExtractor(builtInPreparedQuery.getStatement(), builtInPreparedQuery.getParameters()), session.getWarningCollector(), Optional.of(metadataExtractorExecutor), - analyzerContext.getQuery()); + analyzerContext.getQuery(), + new ViewDefinitionReferences()); Analysis analysis = analyzer.analyzeSemantic( ((BuiltInQueryPreparer.BuiltInPreparedQuery) preparedQuery).getStatement(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index 177e99402ca75..a926c0243fdd2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -40,6 +40,7 @@ import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.SqlFunctionId; @@ -2021,7 +2022,7 @@ public static ExpressionAnalysis analyzeExpressions( { // expressions at this point can not have sub queries so deny all access checks // in the future, we will need a full access controller here to verify access to functions - Analysis analysis = new Analysis(null, parameters, isDescribe); + Analysis analysis = new Analysis(null, parameters, isDescribe, new ViewDefinitionReferences()); ExpressionAnalyzer analyzer = create(analysis, session, metadata, sqlParser, new DenyAllAccessControl(), types, warningCollector); for (Expression expression : expressions) { analyzer.analyze(expression, Scope.builder().withRelationType(RelationId.anonymous(), new RelationType()).build()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 0aea60a29b806..0f1d0dede3bec 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -155,16 +155,24 @@ public class FeaturesConfig private double defaultJoinSelectivityCoefficient; private double defaultWriterReplicationCoefficient = 3; private boolean pushAggregationThroughJoin = true; + private boolean pushPartialAggregationThroughJoin; + private boolean pushSemiJoinThroughUnion; + private boolean simplifyCoalesceOverJoinKeys; + private boolean pushdownThroughUnnest; + private boolean simplifyAggregationsOverConstant; private double memoryRevokingTarget = 0.5; private double memoryRevokingThreshold = 0.9; private boolean useMarkDistinct = true; private boolean exploitConstraints = true; private boolean preferPartialAggregation = true; private PartialAggregationStrategy partialAggregationStrategy = PartialAggregationStrategy.ALWAYS; + private LocalExchangeParentPreferenceStrategy localExchangeParentPreferenceStrategy = LocalExchangeParentPreferenceStrategy.ALWAYS; private double partialAggregationByteReductionThreshold = 0.5; private boolean adaptivePartialAggregationEnabled; private double adaptivePartialAggregationRowsReductionRatioThreshold = 0.8; private boolean optimizeTopNRowNumber = true; + + private boolean optimizeTopNRank; private boolean pushLimitThroughOuterJoin = true; private boolean optimizeConstantGroupingKeys = true; @@ -324,14 +332,18 @@ public class FeaturesConfig private boolean addExchangeBelowPartialAggregationOverGroupId; private boolean addDistinctBelowSemiJoinBuild; private boolean pushdownSubfieldForMapFunctions = true; + private boolean pushdownSubfieldForCardinality; private long maxSerializableObjectSize = 1000; private boolean utilizeUniquePropertyInQueryPlanning = true; private String expressionOptimizerUsedInRowExpressionRewrite = ""; private double tableScanShuffleParallelismThreshold = 0.1; private ShuffleForTableScanStrategy tableScanShuffleStrategy = ShuffleForTableScanStrategy.DISABLED; private boolean skipPushdownThroughExchangeForRemoteProjection; + private String remoteFunctionNamesForFixedParallelism = ""; + private int remoteFunctionFixedParallelismTaskCount = 10; private boolean builtInSidecarFunctionsEnabled; + private String tryFunctionCatchableErrors = ""; public enum PartitioningPrecisionStrategy { @@ -418,6 +430,13 @@ public enum PartialAggregationStrategy AUTOMATIC // Let the optimizer decide for each aggregation } + public enum LocalExchangeParentPreferenceStrategy + { + ALWAYS, // Always use parent preferences for local exchange partitioning + NEVER, // Never use parent preferences, use aggregation's own grouping keys + AUTOMATIC // Cost-based: use parent preferences only if cardinality >= taskConcurrency + } + public enum AggregationIfToFilterRewriteStrategy { DISABLED, @@ -1130,6 +1149,18 @@ public FeaturesConfig setPartialAggregationStrategy(PartialAggregationStrategy p return this; } + public LocalExchangeParentPreferenceStrategy getLocalExchangeParentPreferenceStrategy() + { + return localExchangeParentPreferenceStrategy; + } + + @Config("optimizer.local-exchange-parent-preference-strategy") + public FeaturesConfig setLocalExchangeParentPreferenceStrategy(LocalExchangeParentPreferenceStrategy localExchangeParentPreferenceStrategy) + { + this.localExchangeParentPreferenceStrategy = localExchangeParentPreferenceStrategy; + return this; + } + public double getPartialAggregationByteReductionThreshold() { return partialAggregationByteReductionThreshold; @@ -1171,6 +1202,11 @@ public boolean isOptimizeTopNRowNumber() return optimizeTopNRowNumber; } + public boolean isOptimizeTopNRank() + { + return optimizeTopNRank; + } + @Config("optimizer.optimize-top-n-row-number") public FeaturesConfig setOptimizeTopNRowNumber(boolean optimizeTopNRowNumber) { @@ -1178,6 +1214,13 @@ public FeaturesConfig setOptimizeTopNRowNumber(boolean optimizeTopNRowNumber) return this; } + @Config("optimizer.optimize-top-n-rank") + public FeaturesConfig setOptimizeTopNRank(boolean optimizeTopNRank) + { + this.optimizeTopNRank = optimizeTopNRank; + return this; + } + public boolean isOptimizeCaseExpressionPredicate() { return optimizeCaseExpressionPredicate; @@ -1638,6 +1681,69 @@ public FeaturesConfig setPushAggregationThroughJoin(boolean value) return this; } + public boolean isPushPartialAggregationThroughJoin() + { + return pushPartialAggregationThroughJoin; + } + + @Config("optimizer.push-partial-aggregation-through-join") + @ConfigDescription("Push partial aggregations below joins") + public FeaturesConfig setPushPartialAggregationThroughJoin(boolean pushPartialAggregationThroughJoin) + { + this.pushPartialAggregationThroughJoin = pushPartialAggregationThroughJoin; + return this; + } + + public boolean isPushSemiJoinThroughUnion() + { + return pushSemiJoinThroughUnion; + } + + @Config("optimizer.push-semi-join-through-union") + @ConfigDescription("Push semi join through union to allow parallel semi join execution") + public FeaturesConfig setPushSemiJoinThroughUnion(boolean pushSemiJoinThroughUnion) + { + this.pushSemiJoinThroughUnion = pushSemiJoinThroughUnion; + return this; + } + + public boolean isSimplifyCoalesceOverJoinKeys() + { + return simplifyCoalesceOverJoinKeys; + } + + @Config("optimizer.simplify-coalesce-over-join-keys") + @ConfigDescription("Simplify redundant COALESCE expressions over equi-join keys based on join type") + public FeaturesConfig setSimplifyCoalesceOverJoinKeys(boolean simplifyCoalesceOverJoinKeys) + { + this.simplifyCoalesceOverJoinKeys = simplifyCoalesceOverJoinKeys; + return this; + } + + public boolean isPushdownThroughUnnest() + { + return pushdownThroughUnnest; + } + + @Config("optimizer.pushdown-through-unnest") + public FeaturesConfig setPushdownThroughUnnest(boolean value) + { + this.pushdownThroughUnnest = value; + return this; + } + + public boolean isSimplifyAggregationsOverConstant() + { + return simplifyAggregationsOverConstant; + } + + @Config("optimizer.simplify-aggregations-over-constant") + public FeaturesConfig setSimplifyAggregationsOverConstant(boolean simplifyAggregationsOverConstant) + { + this.simplifyAggregationsOverConstant = simplifyAggregationsOverConstant; + return this; + } + public boolean isForceSingleNodeOutput() { return forceSingleNodeOutput; @@ -3269,6 +3375,19 @@ public boolean isPushdownSubfieldForMapFunctions() return pushdownSubfieldForMapFunctions; } + @Config("optimizer.pushdown-subfield-for-cardinality") + @ConfigDescription("Enable subfield pruning for cardinality() function to skip reading keys and values") + public FeaturesConfig setPushdownSubfieldForCardinality(boolean pushdownSubfieldForCardinality) + { + this.pushdownSubfieldForCardinality = pushdownSubfieldForCardinality; + return this; + } + + public boolean isPushdownSubfieldForCardinality() + { + return pushdownSubfieldForCardinality; + } + @Config("optimizer.utilize-unique-property-in-query-planning") @ConfigDescription("Utilize the unique property of input columns in query planning") public FeaturesConfig setUtilizeUniquePropertyInQueryPlanning(boolean utilizeUniquePropertyInQueryPlanning) @@ -3359,4 +3478,44 @@ public boolean isBuiltInSidecarFunctionsEnabled() { return this.builtInSidecarFunctionsEnabled; } + + public String getRemoteFunctionNamesForFixedParallelism() + { + return remoteFunctionNamesForFixedParallelism; + } + + @Config("optimizer.remote-function-names-for-fixed-parallelism") + @ConfigDescription("Regex pattern to match remote function names that should use fixed parallelism") + public FeaturesConfig setRemoteFunctionNamesForFixedParallelism(String remoteFunctionNamesForFixedParallelism) + { + this.remoteFunctionNamesForFixedParallelism = remoteFunctionNamesForFixedParallelism; + return this; + } + + @Min(1) + public int getRemoteFunctionFixedParallelismTaskCount() + { + return remoteFunctionFixedParallelismTaskCount; + } + + @Config("optimizer.remote-function-fixed-parallelism-task-count") + @ConfigDescription("Number of tasks to use for remote functions matching the fixed parallelism pattern. If not set (0), the default hash partition count will be used.") + public FeaturesConfig setRemoteFunctionFixedParallelismTaskCount(int remoteFunctionFixedParallelismTaskCount) + { + this.remoteFunctionFixedParallelismTaskCount = remoteFunctionFixedParallelismTaskCount; + return this; + } + + public String getTryFunctionCatchableErrors() + { + return tryFunctionCatchableErrors; + } + + @Config("try-function-catchable-errors") + @ConfigDescription("Comma-separated list of error code names that TRY function should catch") + public FeaturesConfig setTryFunctionCatchableErrors(String tryFunctionCatchableErrors) + { + this.tryFunctionCatchableErrors = tryFunctionCatchableErrors; + return this; + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java index ede26ac0b319d..ac0b7591ab38b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/MaterializedViewQueryOptimizer.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.MetadataResolver; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.security.AccessControl; @@ -100,6 +101,7 @@ import static com.facebook.presto.sql.MaterializedViewUtils.COUNT; import static com.facebook.presto.sql.MaterializedViewUtils.NON_ASSOCIATIVE_REWRITE_FUNCTIONS; import static com.facebook.presto.sql.MaterializedViewUtils.SUM; +import static com.facebook.presto.sql.MaterializedViewUtils.resolveTableName; import static com.facebook.presto.sql.analyzer.MaterializedViewInformationExtractor.MaterializedViewInfo; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_TABLE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; @@ -640,7 +642,8 @@ protected Node visitAliasedRelation(AliasedRelation node, Void context) @Override protected Node visitRelation(Relation node, Void context) { - if (materializedViewInfo.getBaseTable().isPresent() && node.equals(materializedViewInfo.getBaseTable().get())) { + if (materializedViewInfo.getBaseTable().isPresent() && resolveTableName(node, session, metadata) + .equals(resolveTableName(materializedViewInfo.getBaseTable().get(), session, metadata))) { return materializedView; } throw new IllegalStateException("Mismatching table or non-supporting relation format in base query"); @@ -771,7 +774,7 @@ ExpressionAnalysis getExpressionAnalysis(Expression expression, Scope scope) accessControl, sqlParser, scope, - new Analysis(null, ImmutableMap.of(), false), + new Analysis(null, ImmutableMap.of(), false, new ViewDefinitionReferences()), expression, WarningCollector.NOOP); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java index 8edf6fcb57c0f..7bb90d8619f4d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/QueryExplainer.java @@ -21,6 +21,8 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; @@ -58,6 +60,7 @@ import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.graphvizLogicalPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonDistributedPlan; import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.jsonLogicalPlan; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForTablesAndColumns; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -119,13 +122,17 @@ public QueryExplainer( this.planChecker = requireNonNull(planChecker, "planChecker is null"); } - public Analysis analyze(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query) + public Analysis analyze(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { - Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(this), parameters, parameterExtractor(statement, parameters), warningCollector, query); - return analyzer.analyze(statement); + Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(this), parameters, parameterExtractor(statement, parameters), warningCollector, query, viewDefinitionReferences); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissionsForTablesAndColumns(accessControlReferences); + + return analysis; } - public String getPlan(Session session, Statement statement, Type planType, List parameters, boolean verbose, WarningCollector warningCollector, String query) + public String getPlan(Session session, Statement statement, Type planType, List parameters, boolean verbose, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { DataDefinitionTask task = dataDefinitionTask.get(statement.getClass()); if (task != null) { @@ -134,13 +141,13 @@ public String getPlan(Session session, Statement statement, Type planType, List< switch (planType) { case LOGICAL: - Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, query); + Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session, 0, verbose, isVerboseOptimizerInfoEnabled(session)); case DISTRIBUTED: - SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query); + SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return PlanPrinter.textDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session, verbose); case IO: - return IOPlanPrinter.textIOPlan(getLogicalPlan(session, statement, parameters, warningCollector, query).getRoot(), metadata, session); + return IOPlanPrinter.textIOPlan(getLogicalPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences).getRoot(), metadata, session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); } @@ -150,7 +157,7 @@ private static String explainTask(Statement statement, Dat return task.explain((T) statement, parameters); } - public String getGraphvizPlan(Session session, Statement statement, Type planType, List parameters, WarningCollector warningCollector, String query) + public String getGraphvizPlan(Session session, Statement statement, Type planType, List parameters, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { DataDefinitionTask task = dataDefinitionTask.get(statement.getClass()); if (task != null) { @@ -160,16 +167,16 @@ public String getGraphvizPlan(Session session, Statement statement, Type planTyp switch (planType) { case LOGICAL: - Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, query); + Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return graphvizLogicalPlan(plan.getRoot(), plan.getTypes(), plan.getStatsAndCosts(), metadata.getFunctionAndTypeManager(), session); case DISTRIBUTED: - SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query); + SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return graphvizDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session); } throw new IllegalArgumentException("Unhandled plan type: " + planType); } - public String getJsonPlan(Session session, Statement statement, Type planType, List parameters, WarningCollector warningCollector, String query) + public String getJsonPlan(Session session, Statement statement, Type planType, List parameters, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { DataDefinitionTask task = dataDefinitionTask.get(statement.getClass()); if (task != null) { @@ -180,29 +187,29 @@ public String getJsonPlan(Session session, Statement statement, Type planType, L Plan plan; switch (planType) { case IO: - plan = getLogicalPlan(session, statement, parameters, warningCollector, query); + plan = getLogicalPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return textIOPlan(plan.getRoot(), metadata, session); case LOGICAL: - plan = getLogicalPlan(session, statement, parameters, warningCollector, query); + plan = getLogicalPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return jsonLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session); case DISTRIBUTED: - SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query); + SubPlan subPlan = getDistributedPlan(session, statement, parameters, warningCollector, query, viewDefinitionReferences); return jsonDistributedPlan(subPlan, metadata.getFunctionAndTypeManager(), session); default: throw new PrestoException(NOT_SUPPORTED, format("Unsupported explain plan type %s for JSON format", planType)); } } - public Plan getLogicalPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query) + public Plan getLogicalPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { - return getLogicalPlan(session, statement, parameters, warningCollector, new PlanNodeIdAllocator(), query); + return getLogicalPlan(session, statement, parameters, warningCollector, new PlanNodeIdAllocator(), query, viewDefinitionReferences); } - public Plan getLogicalPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, PlanNodeIdAllocator idAllocator, String query) + public Plan getLogicalPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, PlanNodeIdAllocator idAllocator, String query, ViewDefinitionReferences viewDefinitionReferences) { // analyze statement Analysis analysis = session.getRuntimeStats() - .recordWallAndCpuTime(ANALYZE_TIME_NANOS, () -> analyze(session, statement, parameters, warningCollector, query)); + .recordWallAndCpuTime(ANALYZE_TIME_NANOS, () -> analyze(session, statement, parameters, warningCollector, query, viewDefinitionReferences)); final VariableAllocator planVariableAllocator = new VariableAllocator(); LogicalPlanner logicalPlanner = new LogicalPlanner( @@ -233,10 +240,10 @@ public Plan getLogicalPlan(Session session, Statement statement, List optimizer.validateAndOptimizePlan(planNode, OPTIMIZED_AND_VALIDATED)); } - public SubPlan getDistributedPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query) + public SubPlan getDistributedPlan(Session session, Statement statement, List parameters, WarningCollector warningCollector, String query, ViewDefinitionReferences viewDefinitionReferences) { PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); - Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, idAllocator, query); + Plan plan = getLogicalPlan(session, statement, parameters, warningCollector, idAllocator, query, viewDefinitionReferences); return session.getRuntimeStats() .recordWallAndCpuTime(FRAGMENT_PLAN_TIME_NANOS, () -> planFragmenter.createSubPlans(session, plan, false, idAllocator, warningCollector)); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index d72a1ad680bbb..4fe1e2f3e5f4e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -46,9 +46,9 @@ import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.StandardErrorCode; +import com.facebook.presto.spi.StandardWarningCode; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.spi.analyzer.AccessControlInfo; import com.facebook.presto.spi.analyzer.AccessControlInfoForTable; import com.facebook.presto.spi.analyzer.MetadataResolver; import com.facebook.presto.spi.analyzer.ViewDefinition; @@ -103,6 +103,7 @@ import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateSchema; @@ -412,8 +413,6 @@ public StatementAnalyzer( this.metadataResolver = requireNonNull(metadata.getMetadataResolver(session), "metadataResolver is null"); requireNonNull(metadata.getFunctionAndTypeManager(), "functionAndTypeManager is null"); this.functionAndTypeResolver = requireNonNull(metadata.getFunctionAndTypeManager().getFunctionAndTypeResolver(), "functionAndTypeResolver is null"); - - analysis.addQueryAccessControlInfo(new AccessControlInfo(accessControl, session.getIdentity(), session.getTransactionId(), session.getAccessControlContext())); } public Scope analyze(Node node, Scope outerQueryScope) @@ -765,6 +764,9 @@ protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional scope) return createAndAssignScope(node, scope); } + @Override + protected Scope visitCreateBranch(CreateBranch node, Optional scope) + { + return createAndAssignScope(node, scope); + } + @Override protected Scope visitDropTag(DropTag node, Optional scope) { @@ -2195,15 +2203,20 @@ protected Scope visitTable(Table table, Optional scope) } Optional optionalMaterializedView = getMaterializedViewDefinition(session, metadataResolver, analysis.getMetadataHandle(), name); - // Prevent INSERT and CREATE TABLE when selecting from a materialized view. - if (optionalMaterializedView.isPresent() - && (analysis.getStatement() instanceof Insert || analysis.getStatement() instanceof CreateTableAsSelect)) { - throw new SemanticException( - NOT_SUPPORTED, - table, - "%s by selecting from a materialized view %s is not supported", - analysis.getStatement().getClass().getSimpleName(), - optionalMaterializedView.get().getTable()); + // Prevent INSERT when selecting from a materialized view into one of its base tables (circular dependency). + if (optionalMaterializedView.isPresent() && analysis.getStatement() instanceof Insert) { + Insert insert = (Insert) analysis.getStatement(); + QualifiedObjectName targetTable = createQualifiedObjectName(session, insert, insert.getTarget(), metadata); + SchemaTableName targetSchemaTable = new SchemaTableName(targetTable.getSchemaName(), targetTable.getObjectName()); + if (optionalMaterializedView.get().getBaseTables().contains(targetSchemaTable)) { + throw new SemanticException( + NOT_SUPPORTED, + table, + "INSERT into table %s by selecting from materialized view %s is not supported because %s is a base table of the materialized view", + targetTable, + optionalMaterializedView.get().getTable(), + targetTable); + } } Statement statement = analysis.getStatement(); if (optionalMaterializedView.isPresent() && statement instanceof Query) { @@ -2221,7 +2234,7 @@ protected Scope visitTable(Table table, Optional scope) else { // when stitching is not enabled, still check permission of each base table MaterializedViewDefinition materializedViewDefinition = optionalMaterializedView.get(); - analysis.getAccessControlReferences().addMaterializedViewDefinitionReference(name, materializedViewDefinition); + analysis.getViewDefinitionReferences().addMaterializedViewDefinitionReference(name, materializedViewDefinition); Query viewQuery = (Query) sqlParser.createStatement( materializedViewDefinition.getOriginalSql(), @@ -2410,7 +2423,7 @@ private Scope processView(Table table, Optional scope, QualifiedObjectNam } ViewDefinition view = optionalView.get(); - analysis.getAccessControlReferences().addViewDefinitionReference(name, view); + analysis.getViewDefinitionReferences().addViewDefinitionReference(name, view); Optional savedViewAccessorWhereClause = analysis.getCurrentQuerySpecification() .flatMap(QuerySpecification::getWhere); @@ -2464,7 +2477,7 @@ private Scope processMaterializedView( { MaterializedViewPlanValidator.validate((Query) sqlParser.createStatement(materializedViewDefinition.getOriginalSql(), createParsingOptions(session, warningCollector))); - analysis.getAccessControlReferences().addMaterializedViewDefinitionReference(materializedViewName, materializedViewDefinition); + analysis.getViewDefinitionReferences().addMaterializedViewDefinitionReference(materializedViewName, materializedViewDefinition); analysis.registerMaterializedViewForAnalysis(materializedViewName, materializedView, materializedViewDefinition.getOriginalSql()); @@ -2954,7 +2967,10 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional analysis.setOrderByExpressions(node, orderByExpressions); List sourceExpressions = new ArrayList<>(outputExpressions); - node.getHaving().ifPresent(sourceExpressions::add); + // Use the rewritten HAVING expression (to resolve SELECT alias references) + if (node.getHaving().isPresent()) { + sourceExpressions.add(analysis.getHaving(node)); + } analyzeGroupingOperations(node, sourceExpressions, orderByExpressions); List aggregates = analyzeAggregations(node, sourceExpressions, orderByExpressions); @@ -3807,7 +3823,12 @@ private void analyzeHaving(QuerySpecification node, Scope scope) if (node.getHaving().isPresent()) { Expression predicate = node.getHaving().get(); - ExpressionAnalysis expressionAnalysis = analyzeExpression(predicate, scope); + // Reuse OrderByExpressionRewriter to resolve SELECT aliases in HAVING + Multimap namedOutputExpressions = extractNamedOutputExpressions(node.getSelect()); + Expression rewrittenPredicate = ExpressionTreeRewriter.rewriteWith(new OrderByExpressionRewriter(namedOutputExpressions, "HAVING"), predicate); + + // Analyze the rewritten expression + ExpressionAnalysis expressionAnalysis = analyzeExpression(rewrittenPredicate, scope); expressionAnalysis.getWindowFunctions().stream() .findFirst() @@ -3817,12 +3838,12 @@ private void analyzeHaving(QuerySpecification node, Scope scope) analysis.recordSubqueries(node, expressionAnalysis); - Type predicateType = expressionAnalysis.getType(predicate); + Type predicateType = expressionAnalysis.getType(rewrittenPredicate); if (!predicateType.equals(BOOLEAN) && !predicateType.equals(UNKNOWN)) { - throw new SemanticException(TYPE_MISMATCH, predicate, "HAVING clause must evaluate to a boolean: actual type %s", predicateType); + throw new SemanticException(TYPE_MISMATCH, rewrittenPredicate, "HAVING clause must evaluate to a boolean: actual type %s", predicateType); } - analysis.setHaving(node, predicate); + analysis.setHaving(node, rewrittenPredicate); } } @@ -3873,10 +3894,17 @@ private class OrderByExpressionRewriter extends ExpressionRewriter { private final Multimap assignments; + private final String clauseName; public OrderByExpressionRewriter(Multimap assignments) + { + this(assignments, "ORDER BY"); + } + + public OrderByExpressionRewriter(Multimap assignments, String clauseName) { this.assignments = assignments; + this.clauseName = clauseName; } @Override @@ -3889,7 +3917,7 @@ public Expression rewriteIdentifier(Identifier reference, Void context, Expressi .collect(Collectors.toSet()); if (expressions.size() > 1) { - throw new SemanticException(AMBIGUOUS_ATTRIBUTE, reference, "'%s' in ORDER BY is ambiguous", name); + throw new SemanticException(AMBIGUOUS_ATTRIBUTE, reference, "'%s' in '%s' is ambiguous", name, clauseName); } if (expressions.size() == 1) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java index fc8752c3cef98..1bd9308bffb4d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java @@ -247,16 +247,19 @@ protected Void visitQuerySpecification(QuerySpecification querySpec, Context con // Wildcards are unresolved in the QuerySpecification's list of SelectItem, so we use output expressions from analysis instead List selectItems = analysis.getOutputExpressions(querySpec); - if (!context.prunable) { - // Examine all the output expressions - for (Expression expression : selectItems) { - process(expression, context); + // selectItems can be null for statements like CREATE TABLE IF NOT EXISTS when the table already exists + if (selectItems != null) { + if (!context.prunable) { + // Examine all the output expressions + for (Expression expression : selectItems) { + process(expression, context); + } } - } - else { - // Prune (Only examine output expressions that have been referenced) - for (FieldId fieldId : context.getFieldIdsToExploreInRelation(querySpec)) { - process(selectItems.get(fieldId.getFieldIndex()), context); + else { + // Prune (Only examine output expressions that have been referenced) + for (FieldId fieldId : context.getFieldIdsToExploreInRelation(querySpec)) { + process(selectItems.get(fieldId.getFieldIndex()), context); + } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java index 6f5274eb36925..31242cab48cc1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java @@ -50,6 +50,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -64,7 +65,6 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; @@ -699,6 +699,7 @@ public Optional visitTopNRowNumber(TopNRowNumberNode node, Context con new DataOrganizationSpecification( partitionBy, node.getSpecification().getOrderingScheme().map(scheme -> getCanonicalOrderingScheme(scheme, context.getExpressions()))), + node.getRankingFunction(), rowNumberVariable, node.getMaxRowCountPerPartition(), node.isPartial(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java index 19ac5a575ded5..13eb1dc8da8e4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java @@ -28,11 +28,11 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TableWriterNode.CallDistributedProcedureTarget; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.sql.planner.plan.CallDistributedProcedureNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 13cc38caef3fe..80ef55ea1139e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -182,6 +182,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -225,7 +226,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.VariableToChannelTranslator; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index 8da6e1866cad2..8f8f9715491c8 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -57,7 +57,6 @@ import com.facebook.presto.sql.planner.iterative.rule.InlineSqlFunctions; import com.facebook.presto.sql.planner.iterative.rule.LeftJoinNullFilterToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.LeftJoinWithArrayContainsToEquiJoinCondition; -import com.facebook.presto.sql.planner.iterative.rule.MaterializedViewRewrite; import com.facebook.presto.sql.planner.iterative.rule.MergeDuplicateAggregation; import com.facebook.presto.sql.planner.iterative.rule.MergeFilters; import com.facebook.presto.sql.planner.iterative.rule.MergeLimitWithDistinct; @@ -110,8 +109,10 @@ import com.facebook.presto.sql.planner.iterative.rule.PushProjectionThroughUnion; import com.facebook.presto.sql.planner.iterative.rule.PushRemoteExchangeThroughAssignUniqueId; import com.facebook.presto.sql.planner.iterative.rule.PushRemoteExchangeThroughGroupId; +import com.facebook.presto.sql.planner.iterative.rule.PushSemiJoinThroughUnion; import com.facebook.presto.sql.planner.iterative.rule.PushTableWriteThroughUnion; import com.facebook.presto.sql.planner.iterative.rule.PushTopNThroughUnion; +import com.facebook.presto.sql.planner.iterative.rule.PushdownThroughUnnest; import com.facebook.presto.sql.planner.iterative.rule.RandomizeSourceKeyInSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.RemoveCrossJoinWithConstantInput; import com.facebook.presto.sql.planner.iterative.rule.RemoveEmptyDelete; @@ -145,7 +146,9 @@ import com.facebook.presto.sql.planner.iterative.rule.RewriteSpatialPartitioningAggregation; import com.facebook.presto.sql.planner.iterative.rule.RuntimeReorderJoinSides; import com.facebook.presto.sql.planner.iterative.rule.ScaledWriterRule; +import com.facebook.presto.sql.planner.iterative.rule.SimplifyAggregationsOverConstant; import com.facebook.presto.sql.planner.iterative.rule.SimplifyCardinalityMap; +import com.facebook.presto.sql.planner.iterative.rule.SimplifyCoalesceOverJoinKeys; import com.facebook.presto.sql.planner.iterative.rule.SimplifyCountOverConstant; import com.facebook.presto.sql.planner.iterative.rule.SimplifyRowExpressions; import com.facebook.presto.sql.planner.iterative.rule.SimplifySortWithConstantInput; @@ -164,6 +167,7 @@ import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToDistinctInnerJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import com.facebook.presto.sql.planner.iterative.rule.TransformUncorrelatedLateralToJoin; +import com.facebook.presto.sql.planner.iterative.rule.materializedview.MaterializedViewRewrite; import com.facebook.presto.sql.planner.optimizations.AddExchanges; import com.facebook.presto.sql.planner.optimizations.AddExchangesForSingleNodeExecution; import com.facebook.presto.sql.planner.optimizations.AddLocalExchanges; @@ -356,7 +360,8 @@ public PlanOptimizers( estimatedExchangesCostCalculator, ImmutableSet.of( new PushProjectionThroughUnion(), - new PushProjectionThroughExchange())); + new PushProjectionThroughExchange(), + new PushdownThroughUnnest(metadata.getFunctionAndTypeManager()))); IterativeOptimizer simplifyRowExpressionOptimizer = new IterativeOptimizer( metadata, @@ -455,6 +460,7 @@ public PlanOptimizers( new ImplementBernoulliSampleAsFilter(metadata.getFunctionAndTypeManager()), new MergeLimitWithDistinct(), new PruneCountAggregationOverScalar(metadata.getFunctionAndTypeManager()), + new SimplifyAggregationsOverConstant(metadata.getFunctionAndTypeManager()), new PruneOrderByInAggregation(metadata.getFunctionAndTypeManager()), new RemoveRedundantTableFunctionProcessor(), // must run after TransformTableFunctionToTableFunctionProcessor new RewriteExcludeColumnsFunctionToProjection(), // must run after TransformTableFunctionToTableFunctionProcessor @@ -579,7 +585,8 @@ public PlanOptimizers( ruleStats, statsCalculator, estimatedExchangesCostCalculator, - ImmutableSet.of(new RemoveCrossJoinWithConstantInput(metadata.getFunctionAndTypeManager())))); + ImmutableSet.of(new RemoveCrossJoinWithConstantInput(metadata.getFunctionAndTypeManager()), + new SimplifyCoalesceOverJoinKeys()))); builder.add(new IterativeOptimizer( metadata, @@ -643,6 +650,12 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, ImmutableSet.of(new LeftJoinNullFilterToSemiJoin(metadata.getFunctionAndTypeManager()))), + new IterativeOptimizer( + metadata, + ruleStats, + statsCalculator, + estimatedExchangesCostCalculator, + ImmutableSet.of(new PushSemiJoinThroughUnion())), new IterativeOptimizer( metadata, ruleStats, @@ -1001,7 +1014,7 @@ public PlanOptimizers( featuresConfig.isNativeExecutionEnabled() && featuresConfig.isPrestoSparkExecutionEnvironment())); // Optimizers above this don't understand local exchanges, so be careful moving this. - builder.add(new AddLocalExchanges(metadata, featuresConfig.isNativeExecutionEnabled())); + builder.add(new AddLocalExchanges(metadata, statsCalculator, featuresConfig.isNativeExecutionEnabled())); // Optimizers above this do not need to care about aggregations with the type other than SINGLE // This optimizer must be run after all exchange-related optimizers diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java index 4e32a6e6918ed..fd5807999b6f4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java @@ -36,6 +36,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.ProjectNode.Locality; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; @@ -84,6 +85,7 @@ import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.callOperator; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.variable; import static com.facebook.presto.type.TypeUtils.NULL_HASH_CODE; @@ -182,6 +184,11 @@ public static Optional getHashExpression(FunctionAndTypeManager f } public static PlanNode addProjections(PlanNode source, PlanNodeIdAllocator planNodeIdAllocator, Map variableMap) + { + return addProjections(source, planNodeIdAllocator, variableMap, LOCAL); + } + + public static PlanNode addProjections(PlanNode source, PlanNodeIdAllocator planNodeIdAllocator, Map variableMap, Locality locality) { Assignments.Builder assignments = Assignments.builder(); for (VariableReferenceExpression variableReferenceExpression : source.getOutputVariables()) { @@ -194,7 +201,7 @@ public static PlanNode addProjections(PlanNode source, PlanNodeIdAllocator planN planNodeIdAllocator.getNextId(), source, assignments.build(), - LOCAL); + locality); } // Add a projection node, which assignment new value if output exists in variableMap, otherwise identity assignment @@ -577,4 +584,19 @@ public static RowExpression randomizeJoinKey(Session session, FunctionAndTypeMan } return new SpecialFormExpression(COALESCE, VARCHAR, ImmutableList.of(castToVarchar, concatExpression)); } + + public static RowExpression getVariableHash(List inputVariables, FunctionAndTypeManager functionAndTypeManager) + { + checkArgument(!inputVariables.isEmpty()); + List hashExpressionList = inputVariables.stream().map(keyVariable -> + callOperator(functionAndTypeManager.getFunctionAndTypeResolver(), OperatorType.XX_HASH_64, BIGINT, keyVariable)).collect(toImmutableList()); + RowExpression hashExpression = hashExpressionList.get(0); + if (hashExpressionList.size() > 1) { + hashExpression = orNullHashCode(hashExpression); + for (int i = 1; i < hashExpressionList.size(); ++i) { + hashExpression = call(functionAndTypeManager, "combine_hash", BIGINT, hashExpression, orNullHashCode(hashExpressionList.get(i))); + } + } + return hashExpression; + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java index 387271aa94e16..463f726cec9c5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java @@ -41,6 +41,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -63,7 +64,6 @@ import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/StatsEquivalentPlanNodeWithLimit.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/StatsEquivalentPlanNodeWithLimit.java index 7e88b2425f0ed..b0a49888c74e3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/StatsEquivalentPlanNodeWithLimit.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/StatsEquivalentPlanNodeWithLimit.java @@ -19,10 +19,10 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.plan.InternalPlanNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java index 8a4174c9cefe7..47aed56286370 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/SystemPartitioningHandle.java @@ -49,7 +49,7 @@ public final class SystemPartitioningHandle implements ConnectorPartitioningHandle { - private enum SystemPartitioning + public enum SystemPartitioning { SINGLE, FIXED, @@ -74,16 +74,31 @@ private static PartitioningHandle createSystemPartitioning(SystemPartitioning pa return new PartitioningHandle(Optional.empty(), Optional.empty(), new SystemPartitioningHandle(partitioning, function)); } + public static PartitioningHandle createSystemPartitioning(SystemPartitioning partitioning, SystemPartitionFunction function, int partitionCount) + { + return new PartitioningHandle(Optional.empty(), Optional.empty(), new SystemPartitioningHandle(partitioning, function, Optional.of(partitionCount))); + } + private final SystemPartitioning partitioning; private final SystemPartitionFunction function; + private final Optional partitionCount; @JsonCreator public SystemPartitioningHandle( @JsonProperty("partitioning") SystemPartitioning partitioning, - @JsonProperty("function") SystemPartitionFunction function) + @JsonProperty("function") SystemPartitionFunction function, + @JsonProperty("partitionCount") Optional partitionCount) { this.partitioning = requireNonNull(partitioning, "partitioning is null"); this.function = requireNonNull(function, "function is null"); + this.partitionCount = requireNonNull(partitionCount, "partitionCount is null"); + } + + public SystemPartitioningHandle( + SystemPartitioning partitioning, + SystemPartitionFunction function) + { + this(partitioning, function, Optional.empty()); } @JsonProperty @@ -98,6 +113,12 @@ public SystemPartitionFunction getFunction() return function; } + @JsonProperty + public Optional getPartitionCount() + { + return partitionCount; + } + @Override public boolean isSingleNode() { @@ -133,13 +154,14 @@ public boolean equals(Object o) } SystemPartitioningHandle that = (SystemPartitioningHandle) o; return partitioning == that.partitioning && - function == that.function; + function == that.function && + partitionCount.equals(that.partitionCount); } @Override public int hashCode() { - return Objects.hash(partitioning, function); + return Objects.hash(partitioning, function, partitionCount); } @Override @@ -162,7 +184,12 @@ else if (partitioning == SystemPartitioning.SINGLE) { nodes = nodeSelector.selectRandomNodes(1); } else if (partitioning == SystemPartitioning.FIXED) { - nodes = nodeSelector.selectRandomNodes(min(getHashPartitionCount(session), getMaxTasksPerStage(session))); + if (!partitionCount.isPresent()) { + nodes = nodeSelector.selectRandomNodes(min(getHashPartitionCount(session), getMaxTasksPerStage(session))); + } + else { + nodes = nodeSelector.selectRandomNodes(min(partitionCount.get(), min(getHashPartitionCount(session), getMaxTasksPerStage(session)))); + } } else { throw new IllegalArgumentException("Unsupported plan distribution " + partitioning); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java index a699a35133fec..01336be7a3a60 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/properties/LogicalPropertiesImpl.java @@ -190,13 +190,7 @@ private boolean keyRequirementSatisfied(Key keyRequirement) if (maxCardProperty.isAtMostOne()) { return true; } - Optional normalizedKeyRequirement = getNormalizedKey(keyRequirement, equivalenceClassProperty); - if (normalizedKeyRequirement.isPresent()) { - return keyProperty.satisfiesKeyRequirement(keyRequirement); - } - else { - return false; - } + return getNormalizedKey(keyRequirement, equivalenceClassProperty).filter(keyProperty::satisfiesKeyRequirement).isPresent(); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MinMaxByToWindowFunction.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MinMaxByToWindowFunction.java index 93447d712844d..2c2d92dfdfbcc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MinMaxByToWindowFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MinMaxByToWindowFunction.java @@ -27,11 +27,11 @@ import com.facebook.presto.spi.plan.Ordering; import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -147,6 +147,7 @@ else if (!maxByAggregations.isEmpty() && minByAggregations.isEmpty()) { node.getStatsEquivalentPlanNode(), node.getSource(), dataOrganizationSpecification, + TopNRowNumberNode.RankingFunction.ROW_NUMBER, rowNumberVariable, 1, false, diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownDereferences.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownDereferences.java index 1d09e2699b238..fbc04aa494003 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownDereferences.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushDownDereferences.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.relation.RowExpression; @@ -40,7 +41,6 @@ import com.facebook.presto.sql.planner.iterative.Rule.Context; import com.facebook.presto.sql.planner.plan.AssignUniqueId; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushSemiJoinThroughUnion.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushSemiJoinThroughUnion.java new file mode 100644 index 0000000000000..fbae8c601415a --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushSemiJoinThroughUnion.java @@ -0,0 +1,242 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SemiJoinNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.RowExpressionVariableInliner; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.isPushSemiJoinThroughUnion; +import static com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils.fromListMultimap; +import static com.facebook.presto.sql.planner.plan.Patterns.semiJoin; + +/** + * Pushes a SemiJoinNode through a UnionNode (on the probe/source side). + *

+ * Transforms: + * 

+ *     - SemiJoin (sourceJoinVar=c, output=sjOut)
+ *         - Union (output c from [a1, a2])
+ *             - source1 (outputs a1)
+ *             - source2 (outputs a2)
+ *         - filteringSource
+ *
+ * into: + * 
+ *     - Union (output sjOut from [sjOut_0, sjOut_1], c from [a1, a2])
+ *         - SemiJoin (sourceJoinVar=a1, output=sjOut_0)
+ *             - source1
+ *             - filteringSource
+ *         - SemiJoin (sourceJoinVar=a2, output=sjOut_1)
+ *             - source2
+ *             - filteringSource
+ *
+ *

+ * Also handles the case where a ProjectNode sits between the SemiJoin and Union: + * 

+ *     - SemiJoin
+ *         - Project
+ *             - Union
+ *         - filteringSource
+ *
+ * In this case, the project is pushed into each union branch before the semi join. + */ +public class PushSemiJoinThroughUnion + implements Rule +{ + private static final Pattern PATTERN = semiJoin(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isPushSemiJoinThroughUnion(session); + } + + @Override + public Result apply(SemiJoinNode semiJoinNode, Captures captures, Context context) + { + PlanNode source = context.getLookup().resolve(semiJoinNode.getSource()); + + if (source instanceof UnionNode) { + return pushThroughUnion(semiJoinNode, (UnionNode) source, Optional.empty(), context); + } + + if (source instanceof ProjectNode) { + ProjectNode projectNode = (ProjectNode) source; + PlanNode projectSource = context.getLookup().resolve(projectNode.getSource()); + if (projectSource instanceof UnionNode) { + return pushThroughUnion(semiJoinNode, (UnionNode) projectSource, Optional.of(projectNode), context); + } + } + + return Result.empty(); + } + + private Result pushThroughUnion( + SemiJoinNode semiJoinNode, + UnionNode unionNode, + Optional projectNode, + Context context) + { + ImmutableList.Builder newSources = ImmutableList.builder(); + ImmutableListMultimap.Builder outputMappings = + ImmutableListMultimap.builder(); + + for (int i = 0; i < unionNode.getSources().size(); i++) { + Map unionVarMap = unionNode.sourceVariableMap(i); + + PlanNode branchSource; + VariableReferenceExpression mappedSourceJoinVar; + Optional mappedSourceHashVar; + Map branchDynamicFilters; + + if (projectNode.isPresent()) { + // Push the project into each union branch, translating its assignments + ProjectNode project = projectNode.get(); + Assignments.Builder assignments = Assignments.builder(); + Map projectVarMapping = new HashMap<>(); + + for (Map.Entry entry : project.getAssignments().entrySet()) { + RowExpression translatedExpression = RowExpressionVariableInliner.inlineVariables(unionVarMap, entry.getValue()); + VariableReferenceExpression newVar = context.getVariableAllocator().newVariable(translatedExpression); + assignments.put(newVar, translatedExpression); + projectVarMapping.put(entry.getKey(), newVar); + } + + branchSource = new ProjectNode( + project.getSourceLocation(), + context.getIdAllocator().getNextId(), + unionNode.getSources().get(i), + assignments.build(), + project.getLocality()); + + // Map the semi-join source variables through the project variable mapping + mappedSourceJoinVar = projectVarMapping.get(semiJoinNode.getSourceJoinVariable()); + if (mappedSourceJoinVar == null) { + return Result.empty(); + } + mappedSourceHashVar = semiJoinNode.getSourceHashVariable().map(projectVarMapping::get); + if (mappedSourceHashVar.isPresent() && mappedSourceHashVar.get() == null) { + return Result.empty(); + } + + // Build output-to-input mappings for original union output variables, + // mapped through the project + for (VariableReferenceExpression semiJoinOutputVar : semiJoinNode.getOutputVariables()) { + if (semiJoinOutputVar.equals(semiJoinNode.getSemiJoinOutput())) { + continue; // handled separately below + } + // This variable comes from the project's output. Map it to the per-branch project output. + VariableReferenceExpression branchVar = projectVarMapping.get(semiJoinOutputVar); + if (branchVar != null) { + outputMappings.put(semiJoinOutputVar, branchVar); + } + } + + // Remap dynamic filter source variables through the project variable mapping + branchDynamicFilters = remapDynamicFilters(semiJoinNode.getDynamicFilters(), projectVarMapping); + } + else { + branchSource = unionNode.getSources().get(i); + + // Map the semi-join source variables through the union variable mapping + mappedSourceJoinVar = unionVarMap.get(semiJoinNode.getSourceJoinVariable()); + if (mappedSourceJoinVar == null) { + return Result.empty(); + } + mappedSourceHashVar = semiJoinNode.getSourceHashVariable().map(unionVarMap::get); + if (mappedSourceHashVar.isPresent() && mappedSourceHashVar.get() == null) { + return Result.empty(); + } + + // Build output-to-input mappings for original union output variables + for (VariableReferenceExpression unionOutputVar : unionNode.getOutputVariables()) { + outputMappings.put(unionOutputVar, unionVarMap.get(unionOutputVar)); + } + + // Remap dynamic filter source variables through the union variable mapping + branchDynamicFilters = remapDynamicFilters(semiJoinNode.getDynamicFilters(), unionVarMap); + } + + // Allocate new semiJoinOutput variable for each branch + VariableReferenceExpression newSemiJoinOutput = + context.getVariableAllocator().newVariable(semiJoinNode.getSemiJoinOutput()); + + // Build new SemiJoinNode for this branch + SemiJoinNode newSemiJoin = new SemiJoinNode( + semiJoinNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + branchSource, + semiJoinNode.getFilteringSource(), + mappedSourceJoinVar, + semiJoinNode.getFilteringSourceJoinVariable(), + newSemiJoinOutput, + mappedSourceHashVar, + semiJoinNode.getFilteringSourceHashVariable(), + semiJoinNode.getDistributionType(), + branchDynamicFilters); + + newSources.add(newSemiJoin); + + // Add the semiJoinOutput mapping + outputMappings.put(semiJoinNode.getSemiJoinOutput(), newSemiJoinOutput); + } + + ListMultimap mappings = outputMappings.build(); + + return Result.ofPlanNode(new UnionNode( + unionNode.getSourceLocation(), + context.getIdAllocator().getNextId(), + newSources.build(), + ImmutableList.copyOf(semiJoinNode.getOutputVariables()), + fromListMultimap(mappings))); + } + + private static Map remapDynamicFilters( + Map dynamicFilters, + Map variableMapping) + { + ImmutableMap.Builder remapped = ImmutableMap.builder(); + for (Map.Entry entry : dynamicFilters.entrySet()) { + VariableReferenceExpression mappedVar = variableMapping.get(entry.getValue()); + if (mappedVar != null) { + remapped.put(entry.getKey(), mappedVar); + } + } + return remapped.build(); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushdownThroughUnnest.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushdownThroughUnnest.java new file mode 100644 index 0000000000000..312353f24ca85 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushdownThroughUnnest.java @@ -0,0 +1,255 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.UnnestNode; +import com.facebook.presto.spi.relation.DeterminismEvaluator; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.VariablesExtractor; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.SystemSessionProperties.isPushdownThroughUnnest; +import static com.facebook.presto.expressions.LogicalRowExpressions.and; +import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; +import static com.facebook.presto.sql.planner.plan.Patterns.project; + +/** + * Pushes projections and filter conjuncts that don't depend on unnest output + * variables below the UnnestNode. This avoids recomputing expressions and + * filtering rows after they've been multiplied by the unnest. + * + * Handles these plan shapes: + * 
+ * 1. Project -> Unnest
+ * 2. Project -> Filter -> Unnest
+ *
+ * + * Example (shape 1): + * 
+ * - Project (x + 1 AS x_plus_1, y)       - Project (x_plus_1, y)
+ *   - Unnest (a -> y)                       - Unnest (a -> y)
+ *     - TableScan (x, a)         =>           - Project (x + 1 AS x_plus_1, a)
+ *                                                 - TableScan (x, a)
+ *
+ * + * Example (shape 2): + * 
+ * - Project (x + 1 AS x_plus_1, y)       - Project (x_plus_1, y)
+ *   - Filter (x > 10 AND y > 0)            - Filter (y > 0)
+ *     - Unnest (a -> y)            =>         - Unnest (a -> y)
+ *       - TableScan (x, a)                       - Filter (x > 10)
+ *                                                     - Project (x + 1 AS x_plus_1, a)
+ *                                                         - TableScan (x, a)
+ *
+ */ +public class PushdownThroughUnnest + implements Rule +{ + private static final Pattern PATTERN = project(); + private final DeterminismEvaluator determinismEvaluator; + + public PushdownThroughUnnest(FunctionAndTypeManager functionAndTypeManager) + { + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isPushdownThroughUnnest(session); + } + + @Override + public Result apply(ProjectNode project, Captures captures, Context context) + { + // Determine the plan shape: Project -> Unnest or Project -> Filter -> Unnest + PlanNode child = context.getLookup().resolve(project.getSource()); + + UnnestNode unnest; + Optional filterNode; + + if (child instanceof UnnestNode) { + unnest = (UnnestNode) child; + filterNode = Optional.empty(); + } + else if (child instanceof FilterNode && context.getLookup().resolve(((FilterNode) child).getSource()) instanceof UnnestNode) { + filterNode = Optional.of((FilterNode) child); + unnest = (UnnestNode) context.getLookup().resolve(((FilterNode) child).getSource()); + } + else { + return Result.empty(); + } + + // Determine which variables are produced by the unnest operation itself + Set unnestProducedVariables = getUnnestProducedVariables(unnest); + + // Partition project assignments into pushable and remaining + Map pushableAssignments = new LinkedHashMap<>(); + Map remainingAssignments = new LinkedHashMap<>(); + + for (Map.Entry entry : project.getAssignments().entrySet()) { + VariableReferenceExpression variable = entry.getKey(); + RowExpression expression = entry.getValue(); + + Set referencedVariables = VariablesExtractor.extractUnique(expression); + + if (referencedVariables.stream().noneMatch(unnestProducedVariables::contains) + && !isIdentityAssignment(variable, expression) + && determinismEvaluator.isDeterministic(expression)) { + pushableAssignments.put(variable, expression); + } + else { + remainingAssignments.put(variable, expression); + } + } + + // Partition filter conjuncts (if filter exists) into pushable and remaining + List pushableConjuncts = new ArrayList<>(); + List remainingConjuncts = new ArrayList<>(); + + if (filterNode.isPresent()) { + List conjuncts = extractConjuncts(filterNode.get().getPredicate()); + for (RowExpression conjunct : conjuncts) { + Set referencedVariables = VariablesExtractor.extractUnique(conjunct); + if (referencedVariables.stream().noneMatch(unnestProducedVariables::contains) + && determinismEvaluator.isDeterministic(conjunct)) { + pushableConjuncts.add(conjunct); + } + else { + remainingConjuncts.add(conjunct); + } + } + } + + // Nothing to push down + if (pushableAssignments.isEmpty() && pushableConjuncts.isEmpty()) { + return Result.empty(); + } + + // Build the new source below the unnest + PlanNode newUnnestSource = unnest.getSource(); + + // Add pushed-down filter below the unnest + if (filterNode.isPresent() && !pushableConjuncts.isEmpty()) { + newUnnestSource = new FilterNode( + filterNode.get().getSourceLocation(), + context.getIdAllocator().getNextId(), + newUnnestSource, + and(pushableConjuncts)); + } + + // Add pushed-down projections below the unnest + if (!pushableAssignments.isEmpty()) { + Assignments.Builder belowUnnestAssignments = Assignments.builder(); + + // Pass through all variables needed by the unnest source + for (VariableReferenceExpression variable : newUnnestSource.getOutputVariables()) { + belowUnnestAssignments.put(variable, variable); + } + + // Add the pushable expressions + for (Map.Entry entry : pushableAssignments.entrySet()) { + belowUnnestAssignments.put(entry.getKey(), entry.getValue()); + } + + newUnnestSource = new ProjectNode( + project.getSourceLocation(), + context.getIdAllocator().getNextId(), + newUnnestSource, + belowUnnestAssignments.build(), + project.getLocality()); + } + + // Build the new UnnestNode + ImmutableList.Builder newReplicateVariables = ImmutableList.builder(); + newReplicateVariables.addAll(unnest.getReplicateVariables()); + newReplicateVariables.addAll(pushableAssignments.keySet()); + + PlanNode result = new UnnestNode( + unnest.getSourceLocation(), + context.getIdAllocator().getNextId(), + newUnnestSource, + newReplicateVariables.build(), + unnest.getUnnestVariables(), + unnest.getOrdinalityVariable()); + + // Add remaining filter on top (if any conjuncts remain) + if (!remainingConjuncts.isEmpty()) { + result = new FilterNode( + filterNode.get().getSourceLocation(), + context.getIdAllocator().getNextId(), + result, + and(remainingConjuncts)); + } + + // Build the remaining project on top + Assignments.Builder topAssignments = Assignments.builder(); + for (Map.Entry entry : remainingAssignments.entrySet()) { + topAssignments.put(entry.getKey(), entry.getValue()); + } + for (VariableReferenceExpression pushedVariable : pushableAssignments.keySet()) { + topAssignments.put(pushedVariable, pushedVariable); + } + + result = new ProjectNode( + project.getSourceLocation(), + context.getIdAllocator().getNextId(), + result, + topAssignments.build(), + project.getLocality()); + + return Result.ofPlanNode(result); + } + + private static Set getUnnestProducedVariables(UnnestNode unnest) + { + ImmutableSet.Builder builder = ImmutableSet.builder(); + unnest.getUnnestVariables().values().stream() + .flatMap(List::stream) + .forEach(builder::add); + unnest.getOrdinalityVariable().ifPresent(builder::add); + return builder.build(); + } + + private static boolean isIdentityAssignment(VariableReferenceExpression variable, RowExpression expression) + { + return expression instanceof VariableReferenceExpression + && variable.equals(expression); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveCrossJoinWithConstantInput.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveCrossJoinWithConstantInput.java index a297fe0b6cd82..0b036bc75879b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveCrossJoinWithConstantInput.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveCrossJoinWithConstantInput.java @@ -36,6 +36,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.SystemSessionProperties.isRemoveCrossJoinWithConstantSingleRowInputEnabled; +import static com.facebook.presto.spi.plan.ProjectNode.Locality.UNKNOWN; import static com.facebook.presto.sql.planner.PlannerUtils.addProjections; import static com.facebook.presto.sql.planner.plan.Patterns.join; import static com.google.common.base.Preconditions.checkState; @@ -103,7 +104,7 @@ else if (isOutputSingleConstantRow(leftInput, context)) { if (!mapping.isPresent()) { return Result.empty(); } - PlanNode resultNode = addProjections(joinInput, context.getIdAllocator(), mapping.get()); + PlanNode resultNode = addProjections(joinInput, context.getIdAllocator(), mapping.get(), UNKNOWN); if (node.getFilter().isPresent()) { resultNode = new FilterNode(node.getSourceLocation(), context.getIdAllocator().getNextId(), resultNode, node.getFilter().get()); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyAggregationsOverConstant.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyAggregationsOverConstant.java new file mode 100644 index 0000000000000..4d21d1259e9d7 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyAggregationsOverConstant.java @@ -0,0 +1,315 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.RowExpressionOptimizer; +import com.facebook.presto.sql.tree.QualifiedName; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.isSimplifyAggregationsOverConstant; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; +import static com.facebook.presto.sql.relational.Expressions.constant; +import static com.facebook.presto.sql.relational.Expressions.constantNull; +import static java.util.Objects.requireNonNull; + +/** + * Folds aggregation functions to constants when the aggregation argument is + * a constant. This optimization is valid regardless of source cardinality + * for functions whose result is independent of row count: + *
    + *
  • {@code MIN(constant)} → constant (NULL if constant is NULL)
    • + *
  • {@code MAX(constant)} → constant (NULL if constant is NULL)
    • + *
  • {@code ARBITRARY(constant)} → constant (NULL if constant is NULL)
    • + *
  • {@code APPROX_DISTINCT(non-null-constant)} → 1
    • + *
  • {@code APPROX_DISTINCT(NULL)} → 0
    • + *
+ * + *

Functions like SUM and COUNT are NOT folded because their results + * depend on the number of rows (e.g., SUM(5) over N rows = 5*N). + * + *

Works with any grouping (global or GROUP BY): foldable aggregations are + * removed from the aggregation node and replaced with constant assignments in + * a ProjectNode on top. If all aggregations are folded and there are no + * grouping keys, the entire node is replaced with a ValuesNode. + * + *

The rule bails out if any aggregation has a filter, mask, distinct, or + * ordering clause, since those could affect the result. + */ +public class SimplifyAggregationsOverConstant + implements Rule +{ + private static final Pattern PATTERN = aggregation(); + private final StandardFunctionResolution functionResolution; + private final FunctionAndTypeResolver functionAndTypeResolver; + private final RowExpressionOptimizer rowExpressionOptimizer; + + public SimplifyAggregationsOverConstant(FunctionAndTypeManager functionAndTypeManager) + { + requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + this.functionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver()); + this.functionAndTypeResolver = functionAndTypeManager.getFunctionAndTypeResolver(); + this.rowExpressionOptimizer = new RowExpressionOptimizer(functionAndTypeManager); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isSimplifyAggregationsOverConstant(session); + } + + @Override + public Result apply(AggregationNode node, Captures captures, Context context) + { + if (node.getStep() != AggregationNode.Step.SINGLE) { + return Result.empty(); + } + + Map aggregations = node.getAggregations(); + if (aggregations.isEmpty()) { + return Result.empty(); + } + + // Resolve source to find constant values from ProjectNode assignments or ValuesNode rows + PlanNode resolvedSource = context.getLookup().resolve(node.getSource()); + ConnectorSession connectorSession = context.getSession().toConnectorSession(); + ConstantResolver constantResolver = buildConstantResolver(resolvedSource, connectorSession); + + // Try to fold each aggregation to a constant + Map foldedConstants = new LinkedHashMap<>(); + Map remainingAggregations = new LinkedHashMap<>(); + + for (Map.Entry entry : aggregations.entrySet()) { + Optional folded = tryFold(entry.getValue(), constantResolver); + if (folded.isPresent()) { + foldedConstants.put(entry.getKey(), folded.get()); + } + else { + remainingAggregations.put(entry.getKey(), entry.getValue()); + } + } + + if (foldedConstants.isEmpty()) { + return Result.empty(); + } + + // If all aggregations are folded and there are no grouping keys, replace with ValuesNode + if (remainingAggregations.isEmpty() && node.getGroupingKeys().isEmpty()) { + List row = new ArrayList<>(); + for (VariableReferenceExpression outputVar : node.getOutputVariables()) { + row.add(foldedConstants.get(outputVar)); + } + return Result.ofPlanNode(new ValuesNode( + node.getSourceLocation(), + node.getId(), + node.getOutputVariables(), + ImmutableList.of(row), + Optional.empty())); + } + + // Otherwise, remove folded aggregations and project their constants on top + AggregationNode newAggregation = new AggregationNode( + node.getSourceLocation(), + context.getIdAllocator().getNextId(), + node.getSource(), + remainingAggregations, + node.getGroupingSets(), + node.getPreGroupedVariables(), + node.getStep(), + node.getHashVariable(), + node.getGroupIdVariable(), + node.getAggregationId()); + + Assignments.Builder assignments = Assignments.builder(); + // Pass through grouping keys and remaining aggregations from the new aggregation node + for (VariableReferenceExpression var : newAggregation.getOutputVariables()) { + assignments.put(var, var); + } + // Add folded aggregations as constant projections + for (Map.Entry entry : foldedConstants.entrySet()) { + assignments.put(entry.getKey(), entry.getValue()); + } + + return Result.ofPlanNode(new ProjectNode( + context.getIdAllocator().getNextId(), + newAggregation, + assignments.build())); + } + + private Optional tryFold(Aggregation aggregation, ConstantResolver constantResolver) + { + // Bail out if aggregation has filter, mask, distinct, or ordering + if (aggregation.getFilter().isPresent() + || aggregation.getMask().isPresent() + || aggregation.isDistinct() + || aggregation.getOrderBy().isPresent()) { + return Optional.empty(); + } + + FunctionHandle functionHandle = aggregation.getFunctionHandle(); + List arguments = aggregation.getArguments(); + + // For functions with arguments, resolve the argument to a constant + if (arguments.size() != 1) { + return Optional.empty(); + } + + RowExpression argument = arguments.get(0); + Optional resolvedConstant = constantResolver.resolve(argument); + if (!resolvedConstant.isPresent()) { + return Optional.empty(); + } + + ConstantExpression constantArg = resolvedConstant.get(); + boolean isNull = constantArg.isNull(); + + // MIN(constant) -> constant (NULL if null) + if (functionResolution.isMinFunction(functionHandle)) { + if (isNull) { + return Optional.of(constantNull(aggregation.getCall().getType())); + } + return Optional.of(constantArg); + } + + // MAX(constant) -> constant (NULL if null) + if (functionResolution.isMaxFunction(functionHandle)) { + if (isNull) { + return Optional.of(constantNull(aggregation.getCall().getType())); + } + return Optional.of(constantArg); + } + + // ARBITRARY(constant) -> constant (NULL if null) + if (isArbitraryFunction(functionHandle)) { + if (isNull) { + return Optional.of(constantNull(aggregation.getCall().getType())); + } + return Optional.of(constantArg); + } + + // APPROX_DISTINCT(constant) -> 1 if non-null, 0 if null + if (functionResolution.isApproximateCountDistinctFunction(functionHandle)) { + return Optional.of(constant(isNull ? 0L : 1L, BIGINT)); + } + + return Optional.empty(); + } + + private ConstantResolver buildConstantResolver(PlanNode resolvedSource, ConnectorSession connectorSession) + { + if (resolvedSource instanceof ProjectNode) { + Assignments assignments = ((ProjectNode) resolvedSource).getAssignments(); + return expression -> { + if (expression instanceof ConstantExpression) { + return Optional.of((ConstantExpression) expression); + } + if (expression instanceof VariableReferenceExpression) { + RowExpression assigned = assignments.get((VariableReferenceExpression) expression); + if (assigned instanceof ConstantExpression) { + return Optional.of((ConstantExpression) assigned); + } + // Try to evaluate the expression to a constant (e.g., CAST(1 AS BIGINT)) + if (assigned != null) { + RowExpression optimized = rowExpressionOptimizer.optimize(assigned, ExpressionOptimizer.Level.OPTIMIZED, connectorSession); + if (optimized instanceof ConstantExpression) { + return Optional.of((ConstantExpression) optimized); + } + } + } + return Optional.empty(); + }; + } + + if (resolvedSource instanceof ValuesNode) { + ValuesNode valuesNode = (ValuesNode) resolvedSource; + List> rows = valuesNode.getRows(); + if (rows.size() == 1) { + List outputVars = valuesNode.getOutputVariables(); + List row = rows.get(0); + return expression -> { + if (expression instanceof ConstantExpression) { + return Optional.of((ConstantExpression) expression); + } + if (expression instanceof VariableReferenceExpression) { + int index = outputVars.indexOf(expression); + if (index >= 0 && index < row.size()) { + RowExpression value = row.get(index); + if (value instanceof ConstantExpression) { + return Optional.of((ConstantExpression) value); + } + // Try to evaluate the expression to a constant + RowExpression optimized = rowExpressionOptimizer.optimize(value, ExpressionOptimizer.Level.OPTIMIZED, connectorSession); + if (optimized instanceof ConstantExpression) { + return Optional.of((ConstantExpression) optimized); + } + } + } + return Optional.empty(); + }; + } + } + + // For other node types, we can only resolve literal constants + return expression -> { + if (expression instanceof ConstantExpression) { + return Optional.of((ConstantExpression) expression); + } + return Optional.empty(); + }; + } + + private boolean isArbitraryFunction(FunctionHandle functionHandle) + { + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName() + .equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("arbitrary"))); + } + + @FunctionalInterface + private interface ConstantResolver + { + Optional resolve(RowExpression expression); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCoalesceOverJoinKeys.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCoalesceOverJoinKeys.java new file mode 100644 index 0000000000000..d11bf1c531d32 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyCoalesceOverJoinKeys.java @@ -0,0 +1,189 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinNode; +import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.facebook.presto.SystemSessionProperties.isSimplifyCoalesceOverJoinKeys; +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; +import static com.facebook.presto.sql.planner.plan.Patterns.join; +import static com.facebook.presto.sql.planner.plan.Patterns.project; +import static com.facebook.presto.sql.planner.plan.Patterns.source; + +/** + * Simplifies redundant COALESCE expressions over equi-join keys based on join type. + *

+ * For equi-join l.x = r.y: + *

    + *
  • LEFT JOIN: COALESCE(l.x, r.y) or COALESCE(r.y, l.x) → l.x (left key is never null)
    • + *
  • RIGHT JOIN: COALESCE(l.x, r.y) or COALESCE(r.y, l.x) → r.y (right key is never null)
    • + *
  • INNER JOIN: COALESCE(l.x, r.y) → l.x, COALESCE(r.y, l.x) → r.y (both non-null, pick first arg)
    • + *
  • FULL JOIN: cannot simplify
    • + *
+ *

+ * This is important because tool-generated queries often produce patterns like + * {@code SELECT COALESCE(l.x, r.y) FROM l LEFT JOIN r ON l.x = r.y} which + * interferes with bucketed join planning. + */ +public class SimplifyCoalesceOverJoinKeys + implements Rule +{ + private static final Capture JOIN = newCapture(); + + private static final Pattern PATTERN = project() + .with(source().matching(join().capturedAs(JOIN))); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public boolean isEnabled(Session session) + { + return isSimplifyCoalesceOverJoinKeys(session); + } + + @Override + public Result apply(ProjectNode project, Captures captures, Context context) + { + JoinNode joinNode = captures.get(JOIN); + JoinType joinType = joinNode.getType(); + + // FULL JOIN: both sides may be null, cannot simplify + if (joinType == JoinType.FULL) { + return Result.empty(); + } + + List criteria = joinNode.getCriteria(); + if (criteria.isEmpty()) { + return Result.empty(); + } + + Set leftVariables = ImmutableSet.copyOf(joinNode.getLeft().getOutputVariables()); + Set rightVariables = ImmutableSet.copyOf(joinNode.getRight().getOutputVariables()); + + // Build a map of join key pairs for quick lookup + // Maps (leftVar, rightVar) pairs from equi-join criteria + Map leftToRight = new HashMap<>(); + Map rightToLeft = new HashMap<>(); + for (EquiJoinClause clause : criteria) { + leftToRight.put(clause.getLeft(), clause.getRight()); + rightToLeft.put(clause.getRight(), clause.getLeft()); + } + + Assignments assignments = project.getAssignments(); + boolean anySimplified = false; + Assignments.Builder newAssignments = Assignments.builder(); + + for (Map.Entry entry : assignments.getMap().entrySet()) { + RowExpression expression = entry.getValue(); + RowExpression simplified = trySimplifyCoalesce(expression, joinType, leftVariables, rightVariables, leftToRight, rightToLeft); + if (simplified != expression) { + anySimplified = true; + } + newAssignments.put(entry.getKey(), simplified); + } + + if (!anySimplified) { + return Result.empty(); + } + + return Result.ofPlanNode(new ProjectNode( + project.getSourceLocation(), + context.getIdAllocator().getNextId(), + joinNode, + newAssignments.build(), + project.getLocality())); + } + + private static RowExpression trySimplifyCoalesce( + RowExpression expression, + JoinType joinType, + Set leftVariables, + Set rightVariables, + Map leftToRight, + Map rightToLeft) + { + if (!(expression instanceof SpecialFormExpression)) { + return expression; + } + + SpecialFormExpression specialForm = (SpecialFormExpression) expression; + if (specialForm.getForm() != COALESCE) { + return expression; + } + + List arguments = specialForm.getArguments(); + if (arguments.size() != 2) { + return expression; + } + + // Both arguments must be variable references + if (!(arguments.get(0) instanceof VariableReferenceExpression) || + !(arguments.get(1) instanceof VariableReferenceExpression)) { + return expression; + } + + VariableReferenceExpression first = (VariableReferenceExpression) arguments.get(0); + VariableReferenceExpression second = (VariableReferenceExpression) arguments.get(1); + + // Check if these two variables form an equi-join key pair + boolean isLeftRight = leftVariables.contains(first) && rightVariables.contains(second) && + leftToRight.containsKey(first) && leftToRight.get(first).equals(second); + boolean isRightLeft = rightVariables.contains(first) && leftVariables.contains(second) && + rightToLeft.containsKey(first) && rightToLeft.get(first).equals(second); + + if (!isLeftRight && !isRightLeft) { + return expression; + } + + VariableReferenceExpression leftKey = isLeftRight ? first : second; + VariableReferenceExpression rightKey = isLeftRight ? second : first; + + switch (joinType) { + case INNER: + // Both sides non-null on join keys; pick the first argument + return first; + case LEFT: + // Left key guaranteed non-null + return leftKey; + case RIGHT: + // Right key guaranteed non-null + return rightKey; + default: + return expression; + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java index b7192926bb807..9d12d01fed842 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/SimplifyRowExpressions.java @@ -29,9 +29,13 @@ import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.google.common.annotations.VisibleForTesting; +import java.util.List; + +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.SERIALIZABLE; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.IF; import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.OR; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; @@ -47,6 +51,7 @@ public SimplifyRowExpressions(Metadata metadata, ExpressionOptimizerManager expr private static class Rewriter implements PlanRowExpressionRewriter { + private final NestedIfSimplifier nestedIfSimplifier; private final ExpressionOptimizerManager expressionOptimizerManager; private final LogicalExpressionRewriter logicalExpressionRewriter; @@ -56,6 +61,7 @@ public Rewriter(Metadata metadata, ExpressionOptimizerManager expressionOptimize requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.expressionOptimizerManager = requireNonNull(expressionOptimizerManager, "expressionOptimizerManager is null"); this.logicalExpressionRewriter = new LogicalExpressionRewriter(metadata.getFunctionAndTypeManager()); + this.nestedIfSimplifier = new NestedIfSimplifier(new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager())); } @Override @@ -69,6 +75,7 @@ private RowExpression rewrite(RowExpression expression, Session session) // Rewrite RowExpression first to reduce depth of RowExpression tree by balancing AND/OR predicates. // It doesn't matter whether we rewrite/optimize first because this will be called by IterativeOptimizer. RowExpression rewritten = RowExpressionTreeRewriter.rewriteWith(logicalExpressionRewriter, expression, true); + rewritten = RowExpressionTreeRewriter.rewriteWith(nestedIfSimplifier, rewritten); return expressionOptimizerManager.getExpressionOptimizer(session.toConnectorSession()).optimize(rewritten, SERIALIZABLE, session.toConnectorSession()); } } @@ -79,6 +86,63 @@ public static RowExpression rewrite(RowExpression expression, Metadata metadata, return new Rewriter(metadata, expressionOptimizerManager).rewrite(expression, session); } + /** + * Simplifies nested IF expressions where the outer and inner false + * branches are identical: + * 

+     *   IF(x, IF(y, v, E), E) → IF(x AND y, v, E)
+     *
+ * This covers the common case where E is null (explicit or omitted ELSE), + * as well as any other matching expression. + *

+ * The rewrite only applies when the inner condition {@code y} is + * deterministic, because the original form does not evaluate {@code y} + * when {@code x} is NULL or FALSE, whereas {@code x AND y} may evaluate + * {@code y} in those cases. + *

+ * Uses bottom-up rewriting so that deeply nested IFs are fully + * flattened in a single pass. + */ + private static class NestedIfSimplifier + extends RowExpressionRewriter + { + private final RowExpressionDeterminismEvaluator determinismEvaluator; + + NestedIfSimplifier(RowExpressionDeterminismEvaluator determinismEvaluator) + { + this.determinismEvaluator = requireNonNull(determinismEvaluator, "determinismEvaluator is null"); + } + + @Override + public RowExpression rewriteSpecialForm(SpecialFormExpression node, Void context, RowExpressionTreeRewriter treeRewriter) + { + if (node.getForm() != IF) { + return null; + } + + // Recursively simplify children first (bottom-up) + SpecialFormExpression rewritten = treeRewriter.defaultRewrite(node, context); + + List args = rewritten.getArguments(); + RowExpression condition = args.get(0); + RowExpression trueValue = args.get(1); + RowExpression falseValue = args.get(2); + + if (trueValue instanceof SpecialFormExpression + && ((SpecialFormExpression) trueValue).getForm() == IF) { + SpecialFormExpression innerIf = (SpecialFormExpression) trueValue; + List innerArgs = innerIf.getArguments(); + RowExpression innerCondition = innerArgs.get(0); + if (falseValue.equals(innerArgs.get(2)) && determinismEvaluator.isDeterministic(innerCondition)) { + RowExpression combinedCondition = new SpecialFormExpression(AND, BOOLEAN, condition, innerCondition); + return new SpecialFormExpression(rewritten.getSourceLocation(), IF, rewritten.getType(), combinedCondition, innerArgs.get(1), falseValue); + } + } + + return rewritten == node ? null : rewritten; + } + } + private static class LogicalExpressionRewriter extends RowExpressionRewriter { diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/DifferentialPlanRewriter.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/DifferentialPlanRewriter.java new file mode 100644 index 0000000000000..216589d7adbb5 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/DifferentialPlanRewriter.java @@ -0,0 +1,1061 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule.materializedview; + +import com.facebook.presto.Session; +import com.facebook.presto.common.predicate.Domain; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewDefinition.TableColumn; +import com.facebook.presto.spi.PrestoWarning; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ExceptNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinNode; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.MaterializedViewScanNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SortNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.GroupReference; +import com.facebook.presto.sql.planner.iterative.Lookup; +import com.facebook.presto.sql.planner.optimizations.SetOperationNodeUtils; +import com.facebook.presto.sql.planner.optimizations.SymbolMapper; +import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; +import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ListMultimap; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static com.facebook.presto.expressions.LogicalRowExpressions.or; +import static com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates; +import static com.facebook.presto.spi.StandardWarningCode.MATERIALIZED_VIEW_STITCHING_FALLBACK; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher.searchFrom; +import static com.facebook.presto.sql.relational.Expressions.not; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +/** + * Builds the delta plan for materialized view differential stitching. + * + *

Uses standard IVM delta algebra for monotonic operators (Join, Union, Intersect), + * and falls back to partition-level recompute for Except and Aggregation. + * + *

Terminology (matching IVM formalism): + *

    + *
  • R = unchanged rows (from non-stale partitions)
    • + *
  • R' = current state (all rows)
    • + *
  • ∆R = delta (rows from stale partitions)
    • + *
+ * + *

Identity for partition-aligned staleness: R' = R ∪ ∆R + * + *

Delta rules by operator: + *

    + *
  • Join: ∆(R ⋈ S) = (∆R ⋈ S') ∪ (R ⋈ ∆S)
    • + *
  • Union: ∆(R ∪ S) = ∆R ∪ ∆S
    • + *
  • Intersect: ∆(R ∩ S) = (∆R ∩ S') ∪ (R ∩ ∆S)
    • + *
  • Except: Uses partition replacement (see below)
    • + *
  • Selection/Projection/Aggregation: ∆(op(R)) = op(∆R)
    • + *
+ * + *

EXCEPT handling: EXCEPT is anti-monotonic in the right input, requiring + * ∆âº/∆⻠tracking per the formal rule: ∆âº(R − S) = (∆âºR − S') ∪ (R ∩ ∆â»S). + * Since we don't track deletions separately, we fall back to partition replacement: + * when S has stale partitions, we identify affected output partitions and recompute + * from the current base table state. + */ +public class DifferentialPlanRewriter +{ + private final Metadata metadata; + private final Session session; + private final PlanNodeIdAllocator idAllocator; + private final VariableAllocator variableAllocator; + private final RowExpressionDomainTranslator translator; + private final RowExpressionDeterminismEvaluator determinismEvaluator; + private final Map>> staleConstraints; + private final PassthroughColumnEquivalences columnEquivalences; + private final Lookup lookup; + private final WarningCollector warningCollector; + + public DifferentialPlanRewriter( + Metadata metadata, + Session session, + PlanNodeIdAllocator idAllocator, + VariableAllocator variableAllocator, + Map>> staleConstraints, + PassthroughColumnEquivalences columnEquivalences, + Lookup lookup, + WarningCollector warningCollector) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.session = requireNonNull(session, "session is null"); + this.idAllocator = requireNonNull(idAllocator, "idAllocator is null"); + this.variableAllocator = requireNonNull(variableAllocator, "variableAllocator is null"); + this.translator = new RowExpressionDomainTranslator(metadata); + this.determinismEvaluator = new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()); + this.staleConstraints = ImmutableMap.copyOf(requireNonNull(staleConstraints, "staleConstraints is null")); + this.columnEquivalences = requireNonNull(columnEquivalences, "columnEquivalences is null"); + this.lookup = requireNonNull(lookup, "lookup is null"); + this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); + } + + /** + * Builds a stitched query plan combining fresh MV data with recomputed delta. + * This is the main entry point for query-time stitching. + * + * @return Optional containing the stitched plan, or empty if stitching is not possible + */ + public static Optional buildStitchedPlan( + Metadata metadata, + Session session, + MaterializedViewScanNode node, + Map constraints, + MaterializedViewDefinition materializedViewDefinition, + VariableAllocator variableAllocator, + PlanNodeIdAllocator idAllocator, + Lookup lookup, + WarningCollector warningCollector) + { + SchemaTableName dataTable = new SchemaTableName(materializedViewDefinition.getSchema(), materializedViewDefinition.getTable()); + PassthroughColumnEquivalences columnEquivalences = new PassthroughColumnEquivalences(materializedViewDefinition, dataTable); + + Map>> filteredConstraints = filterPredicatesToMappedColumns(constraints, columnEquivalences); + // If any base table is stale yet has no mapped predicates, stitching is not possible + if (filteredConstraints.values().stream().anyMatch(List::isEmpty)) { + return Optional.empty(); + } + + PlanNode freshPlan = buildDataTableBranch(metadata, session, node, filteredConstraints, columnEquivalences, dataTable, idAllocator, lookup); + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + filteredConstraints, + columnEquivalences, + lookup, + warningCollector); + + NodeWithMapping deltaResult; + try { + deltaResult = builder.buildDeltaPlan(node.getViewQueryPlan(), node.getViewQueryMappings()); + } + catch (UnsupportedOperationException e) { + warningCollector.add(new PrestoWarning( + MATERIALIZED_VIEW_STITCHING_FALLBACK, + "Cannot use differential stitching for materialized view " + node.getMaterializedViewName() + + ": " + e.getMessage() + ". Falling back to full recompute.")); + return Optional.empty(); + } + + return Optional.of(buildUnionNode( + node, + freshPlan, + node.getDataTableMappings(), + deltaResult.getNode(), + deltaResult.getMapping(), + idAllocator)); + } + + /** + * Filters stale predicates from all tables to only include columns that have equivalence mappings. + */ + private static Map>> filterPredicatesToMappedColumns( + Map constraints, + PassthroughColumnEquivalences columnEquivalences) + { + return constraints.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + entry -> filterPredicatesForTable( + entry.getValue().getPredicateDisjuncts(), + entry.getKey(), + columnEquivalences))); + } + + private static List> filterPredicatesForTable( + List> stalePredicates, + SchemaTableName table, + PassthroughColumnEquivalences columnEquivalences) + { + return stalePredicates.stream() + .filter(predicate -> predicate.getDomains().isPresent()) + .map(predicate -> { + Map filteredDomains = predicate.getDomains().get().entrySet().stream() + .filter(entry -> columnEquivalences.hasEquivalence(new TableColumn(table, entry.getKey()))) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + return TupleDomain.withColumnDomains(filteredDomains); + }) + .filter(predicate -> !predicate.isAll()) + .collect(toImmutableList()); + } + + private static PlanNode buildDataTableBranch( + Metadata metadata, + Session session, + MaterializedViewScanNode node, + Map>> constraints, + PassthroughColumnEquivalences columnEquivalences, + SchemaTableName dataTable, + PlanNodeIdAllocator idAllocator, + Lookup lookup) + { + PlanNode dataTablePlan = node.getDataTablePlan(); + + // Build negated stale predicate for the data table: NOT(stale_partition_1 OR stale_partition_2 OR ...) + List> stalePredicates = collectStalePredicatesForDataTable(constraints, columnEquivalences, dataTable); + + if (stalePredicates.isEmpty()) { + return dataTablePlan; + } + + Map columnMapping = buildColumnToVariableMapping(metadata, session, dataTablePlan, lookup); + RowExpressionDomainTranslator translator = new RowExpressionDomainTranslator(metadata); + + // Transform stale partition predicates to row expressions using the data table's variables. + // After transform(), a predicate becomes isNone() if a column couldn't be mapped (should not + // happen since collectStalePredicatesForDataTable already filters out isAll() predicates). + List staleExpressions = stalePredicates.stream() + .map(predicate -> predicate.transform(col -> columnMapping.get(new TableColumn(dataTable, col)))) + .filter(pred -> !pred.isNone()) + .map(translator::toPredicate) + .collect(toImmutableList()); + + if (staleExpressions.isEmpty()) { + return dataTablePlan; + } + + RowExpression stalePredicate = or(staleExpressions); + RowExpression freshPredicate = not(metadata.getFunctionAndTypeManager(), stalePredicate); + + return new FilterNode(dataTablePlan.getSourceLocation(), idAllocator.getNextId(), dataTablePlan, freshPredicate); + } + + private static List> collectStalePredicatesForDataTable( + Map>> constraints, + PassthroughColumnEquivalences columnEquivalences, + SchemaTableName dataTable) + { + ImmutableList.Builder> result = ImmutableList.builder(); + for (Map.Entry>> entry : constraints.entrySet()) { + SchemaTableName baseTable = entry.getKey(); + for (TupleDomain stalePredicate : entry.getValue()) { + Map> equivalentPredicates = + columnEquivalences.getEquivalentPredicates(baseTable, stalePredicate); + TupleDomain dataTablePredicate = equivalentPredicates.get(dataTable); + if (dataTablePredicate != null && !dataTablePredicate.isAll()) { + result.add(dataTablePredicate); + } + } + } + return result.build(); + } + + private static Map buildColumnToVariableMapping( + Metadata metadata, + Session session, + PlanNode plan, + Lookup lookup) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + searchFrom(plan, lookup) + .where(TableScanNode.class::isInstance) + .findAll() + .stream() + .map(TableScanNode.class::cast) + .forEach(tableScan -> { + SchemaTableName tableName = metadata.getTableMetadata(session, tableScan.getTable()).getTable(); + for (Map.Entry entry : tableScan.getAssignments().entrySet()) { + ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableScan.getTable(), entry.getValue()); + builder.put(new TableColumn(tableName, columnMetadata.getName()), entry.getKey()); + } + }); + return builder.build(); + } + + private static PlanNode buildUnionNode( + MaterializedViewScanNode node, + PlanNode freshPlan, + Map freshMapping, + PlanNode deltaPlan, + Map deltaMapping, + PlanNodeIdAllocator idAllocator) + { + ImmutableListMultimap.Builder outputsToInputs = + ImmutableListMultimap.builder(); + + for (VariableReferenceExpression outputVar : node.getOutputVariables()) { + VariableReferenceExpression freshVar = freshMapping.get(outputVar); + VariableReferenceExpression deltaVar = deltaMapping.get(outputVar); + checkState(freshVar != null && deltaVar != null, + "Missing mapping for output variable %s: freshVar=%s, deltaVar=%s", outputVar, freshVar, deltaVar); + outputsToInputs.put(outputVar, freshVar); + outputsToInputs.put(outputVar, deltaVar); + } + + ListMultimap mapping = outputsToInputs.build(); + return new UnionNode( + node.getSourceLocation(), + idAllocator.getNextId(), + ImmutableList.of(freshPlan, deltaPlan), + ImmutableList.copyOf(mapping.keySet()), + SetOperationNodeUtils.fromListMultimap(mapping)); + } + + /** + * Builds a delta plan for the given view query plan. + * + * @param viewQueryPlan The view query plan to transform + * @param viewQueryMappings Mapping from MV output variables to view query variables + * @return NodeWithMapping containing the delta plan and composed mapping (MV output var → delta var) + * @throws UnsupportedOperationException if the plan contains unsupported nodes + */ + public NodeWithMapping buildDeltaPlan( + PlanNode viewQueryPlan, + Map viewQueryMappings) + { + PlanVariants result = viewQueryPlan.accept(new DeltaBuilder(), null); + Map deltaMapping = result.delta().getMapping(); + + // Compose mappings: MV output var → view query var → delta var + ImmutableMap.Builder composedMapping = ImmutableMap.builder(); + for (Map.Entry entry : viewQueryMappings.entrySet()) { + VariableReferenceExpression deltaVar = deltaMapping.get(entry.getValue()); + checkState(deltaVar != null, + "Missing delta mapping for view query variable %s", entry.getValue()); + composedMapping.put(entry.getKey(), deltaVar); + } + + return new NodeWithMapping(result.delta().getNode(), composedMapping.build()); + } + + private class DeltaBuilder + extends InternalPlanVisitor + { + @Override + public PlanVariants visitPlan(PlanNode node, Void context) + { + throw new UnsupportedOperationException("Unsupported node type: " + node.getClass().getSimpleName()); + } + + @Override + public PlanVariants visitTableScan(TableScanNode node, Void context) + { + SchemaTableName tableName = metadata.getTableMetadata(session, node.getTable()).getTable(); + List> stalePredicates = staleConstraints.getOrDefault(tableName, ImmutableList.of()); + + // Build three table scan variants with fresh variables + NodeWithMapping deltaResult = buildTableScan(node, node.getOutputVariables()); + NodeWithMapping currentResult = buildTableScan(node, node.getOutputVariables()); + NodeWithMapping unchangedResult = buildTableScan(node, node.getOutputVariables()); + + RowExpression stalePredicate = buildStalePredicate(node, stalePredicates, deltaResult.getMapping()); + RowExpression unchangedPredicate = not(metadata.getFunctionAndTypeManager(), + buildStalePredicate(node, stalePredicates, unchangedResult.getMapping())); + + // Apply stale/non-stale filters + PlanNode deltaNode = new FilterNode(node.getSourceLocation(), idAllocator.getNextId(), deltaResult.getNode(), stalePredicate); + PlanNode unchangedNode = new FilterNode(node.getSourceLocation(), idAllocator.getNextId(), unchangedResult.getNode(), unchangedPredicate); + + return new PlanVariants( + new NodeWithMapping(deltaNode, deltaResult.getMapping()), + currentResult, + new NodeWithMapping(unchangedNode, unchangedResult.getMapping())); + } + + private NodeWithMapping buildTableScan(TableScanNode original, List variables) + { + Map mapping = createFreshMapping(variables); + return new NodeWithMapping(new SymbolMapper(mapping, warningCollector).map(original, idAllocator.getNextId()), mapping); + } + + private RowExpression buildStalePredicate( + TableScanNode node, + List> stalePredicates, + Map variableMapping) + { + Map columnToVariable = node.getAssignments().entrySet().stream() + .collect(toImmutableMap( + entry -> metadata.getColumnMetadata(session, node.getTable(), entry.getValue()).getName(), + entry -> variableMapping.get(entry.getKey()))); + List predicates = stalePredicates.stream() + .map(disjunct -> disjunct.transform(columnToVariable::get)) + .map(translator::toPredicate) + .collect(toImmutableList()); + return or(predicates); + } + + @Override + public PlanVariants visitFilter(FilterNode node, Void context) + { + checkDeterministic(node.getPredicate(), "filter predicate"); + PlanVariants child = node.getSources().get(0).accept(this, context); + + // ∆(σ(R)) = σ(∆R), σ(R') = σ(R'), σ(R) = σ(R) + return new PlanVariants( + buildFilter(node, child.delta()), + buildFilter(node, child.current()), + buildFilter(node, child.unchanged())); + } + + private NodeWithMapping buildFilter(FilterNode original, NodeWithMapping source) + { + return new NodeWithMapping( + new SymbolMapper(source.getMapping(), warningCollector).map(original, source.getNode(), idAllocator.getNextId()), + source.getMapping()); + } + + @Override + public PlanVariants visitProject(ProjectNode node, Void context) + { + node.getAssignments().getExpressions().forEach(expr -> checkDeterministic(expr, "projection")); + PlanVariants child = node.getSources().get(0).accept(this, context); + + // ∆(Ï€(R)) = Ï€(∆R), Ï€(R') = Ï€(R'), Ï€(R) = Ï€(R) + return new PlanVariants( + buildProject(node, child.delta()), + buildProject(node, child.current()), + buildProject(node, child.unchanged())); + } + + private NodeWithMapping buildProject(ProjectNode original, NodeWithMapping source) + { + Map mapping = + extendMapping(source.getMapping(), original.getOutputVariables()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, source.getNode(), idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitAggregation(AggregationNode node, Void context) + { + node.getAggregations().values().forEach(agg -> agg.getCall().getArguments() + .forEach(expr -> checkDeterministic(expr, "aggregation"))); + + PlanVariants child = node.getSources().get(0).accept(this, context); + + // ∆(γ(R)) = γ(∆R), γ(R') = γ(R'), γ(R) = γ(R) + return new PlanVariants( + buildAggregation(node, child.delta()), + buildAggregation(node, child.current()), + buildAggregation(node, child.unchanged())); + } + + private NodeWithMapping buildAggregation(AggregationNode original, NodeWithMapping source) + { + Map mapping = + extendMapping(source.getMapping(), original.getOutputVariables()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, source.getNode(), idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitJoin(JoinNode node, Void context) + { + // Only inner joins are supported - outer joins have different IVM rules + if (node.getType() != INNER) { + throw new UnsupportedOperationException("Outer joins not supported: " + node.getType()); + } + node.getFilter().ifPresent(filter -> checkDeterministic(filter, "join filter")); + + // ∆(R ⋈ S) = (∆R ⋈ S') ∪ (R ⋈ ∆S) + PlanVariants leftVariants = node.getLeft().accept(this, context); + PlanVariants rightVariants = node.getRight().accept(this, context); + + // Current join: R' ⋈ S' + NodeWithMapping currentResult = buildJoin(node, leftVariants.current(), rightVariants.current()); + + // Unchanged join: R ⋈ S (for propagation through the plan) + NodeWithMapping unchangedResult = buildJoin(node, cloneNodeWithMapping(leftVariants.unchanged()), rightVariants.unchanged()); + + // First delta term: ∆R ⋈ S' (delta from left, current from right) + NodeWithMapping deltaLeftResult = buildJoin(node, leftVariants.delta(), cloneNodeWithMapping(rightVariants.current())); + + // Second delta term: R ⋈ ∆S (unchanged from left, delta from right) + // Uses the original left.unchanged() (other use was cloned above) + NodeWithMapping deltaRightResult = buildJoin(node, leftVariants.unchanged(), rightVariants.delta()); + + // Union the delta terms + NodeWithMapping deltaResult = createBinaryUnion(node, deltaLeftResult.getNode(), deltaRightResult.getNode()); + + return new PlanVariants(deltaResult, currentResult, unchangedResult); + } + + private NodeWithMapping buildJoin(JoinNode original, NodeWithMapping left, NodeWithMapping right) + { + Map mapping = + extendMapping(combineMapping(left, right), original.getOutputVariables()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, left.getNode(), right.getNode(), idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitUnion(UnionNode node, Void context) + { + // ∆(R ∪ S) = ∆R ∪ ∆S + List children = visitAllSources(node.getSources(), context); + return new PlanVariants( + buildUnion(node, children.stream().map(PlanVariants::delta).collect(toImmutableList())), + buildUnion(node, children.stream().map(PlanVariants::current).collect(toImmutableList())), + buildUnion(node, children.stream().map(PlanVariants::unchanged).collect(toImmutableList()))); + } + + private NodeWithMapping buildUnion(UnionNode original, List sources) + { + Map mapping = + extendMapping(combineMapping(sources.toArray(new NodeWithMapping[0])), original.getOutputVariables()); + List sourceNodes = sources.stream().map(NodeWithMapping::getNode).collect(toImmutableList()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, sourceNodes, idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitIntersect(IntersectNode node, Void context) + { + // ∆(R ∩ S) = (∆R ∩ S') ∪ (R ∩ ∆S) + // The delta formula assumes binary INTERSECT; n-ary INTERSECT should be + // decomposed into a binary tree before reaching this code. + List sources = node.getSources(); + checkState(sources.size() == 2, + "INTERSECT with more than 2 sources is not supported for differential stitching, found %s sources", + sources.size()); + List allVariants = visitAllSources(sources, context); + + // Current: R' ∩ S' + NodeWithMapping currentResult = buildIntersect(node, allVariants.stream().map(PlanVariants::current).collect(toImmutableList())); + + // Unchanged: R ∩ S + NodeWithMapping unchangedResult = buildIntersect(node, allVariants.stream().map(PlanVariants::unchanged).collect(toImmutableList())); + + // Delta: union of left and right delta terms + IntersectNode deltaLeft = buildIntersectDeltaLeft(node, sources, allVariants); + IntersectNode deltaRight = buildIntersectDeltaRight(node, sources, allVariants); + NodeWithMapping deltaUnionResult = createBinaryUnion(node, deltaLeft, deltaRight); + + return new PlanVariants(deltaUnionResult, currentResult, unchangedResult); + } + + private IntersectNode buildIntersectDeltaLeft( + IntersectNode original, + List originalSources, + List allVariants) + { + List sources = new ArrayList<>(); + sources.add(allVariants.get(0).delta()); + + // Filter remaining sources' current to first source's stale partitions + for (int i = 1; i < allVariants.size(); i++) { + NodeWithMapping clonedCurrent = cloneNodeWithMapping(allVariants.get(i).current()); + RowExpression stalePredicate = buildPropagatedStalePredicate(clonedCurrent.getNode(), originalSources.get(0)); + sources.add(new NodeWithMapping( + buildFilter(clonedCurrent.getNode(), stalePredicate), + clonedCurrent.getMapping())); + } + return buildIntersectFromSources(original, sources); + } + + private IntersectNode buildIntersectDeltaRight( + IntersectNode original, + List originalSources, + List allVariants) + { + // Clone and filter R's unchanged to S's stale partitions + NodeWithMapping firstUnchanged = cloneNodeWithMapping(allVariants.get(0).unchanged()); + ImmutableList.Builder stalePredicates = ImmutableList.builder(); + for (int i = 1; i < allVariants.size(); i++) { + stalePredicates.add(buildPropagatedStalePredicate(firstUnchanged.getNode(), originalSources.get(i))); + } + NodeWithMapping filteredFirst = new NodeWithMapping( + buildFilter(firstUnchanged.getNode(), or(stalePredicates.build())), + firstUnchanged.getMapping()); + + List sources = new ArrayList<>(); + sources.add(filteredFirst); + for (int i = 1; i < allVariants.size(); i++) { + sources.add(allVariants.get(i).delta()); + } + return buildIntersectFromSources(original, sources); + } + + private IntersectNode buildIntersectFromSources(IntersectNode original, List sources) + { + Map mapping = + extendMapping(combineMapping(sources.toArray(new NodeWithMapping[0])), original.getOutputVariables()); + List sourceNodes = sources.stream().map(NodeWithMapping::getNode).collect(toImmutableList()); + return new SymbolMapper(mapping, warningCollector).map(original, sourceNodes, idAllocator.getNextId()); + } + + private List visitAllSources(List sources, Void context) + { + return sources.stream() + .map(source -> source.accept(this, context)) + .collect(toImmutableList()); + } + + private PlanNode buildFilter(PlanNode source, RowExpression predicate) + { + return new FilterNode(source.getSourceLocation(), idAllocator.getNextId(), source, predicate); + } + + private NodeWithMapping buildIntersect(IntersectNode original, List sources) + { + Map mapping = + extendMapping(combineMapping(sources.toArray(new NodeWithMapping[0])), original.getOutputVariables()); + List sourceNodes = sources.stream().map(NodeWithMapping::getNode).collect(toImmutableList()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, sourceNodes, idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitExcept(ExceptNode node, Void context) + { + // EXCEPT is anti-monotonic in the right input. + // deltaLeft: (∆A - B') handles stale left side via delta algebra + // deltaRight: (A[B's stale] - B') handles stale right side via partition replacement + // The delta formula assumes binary EXCEPT; n-ary EXCEPT should be + // decomposed into a binary tree before reaching this code. + List sources = node.getSources(); + checkState(sources.size() == 2, + "EXCEPT with more than 2 sources is not supported for differential stitching, found %s sources", + sources.size()); + List allVariants = visitAllSources(sources, context); + + // Current: A' - B' + NodeWithMapping currentResult = buildExcept(node, allVariants.stream().map(PlanVariants::current).collect(toImmutableList())); + + // Unchanged: A - B' + // The right side must be B' (current) because rows added to B since the last + // refresh may cancel out rows in A. Using stale B would include rows that no + // longer survive the EXCEPT. + ImmutableList.Builder unchangedSources = ImmutableList.builder(); + unchangedSources.add(allVariants.get(0).unchanged()); + for (int i = 1; i < allVariants.size(); i++) { + unchangedSources.add(cloneNodeWithMapping(allVariants.get(i).current())); + } + NodeWithMapping unchangedResult = buildExcept(node, unchangedSources.build()); + + // Delta: union of left and right delta terms + ExceptNode deltaLeft = buildExceptDeltaLeft(node, allVariants); + ExceptNode deltaRight = buildExceptDeltaRight(node, sources, allVariants); + NodeWithMapping deltaUnionResult = createBinaryUnion(node, deltaLeft, deltaRight); + + return new PlanVariants(deltaUnionResult, currentResult, unchangedResult); + } + + private ExceptNode buildExceptDeltaLeft(ExceptNode original, List allVariants) + { + List sources = new ArrayList<>(); + sources.add(allVariants.get(0).delta()); + for (int i = 1; i < allVariants.size(); i++) { + sources.add(cloneNodeWithMapping(allVariants.get(i).current())); + } + return buildExceptFromSources(original, sources); + } + + private ExceptNode buildExceptDeltaRight( + ExceptNode original, + List originalSources, + List allVariants) + { + // Filter A's unchanged rows to only those matching B's stale partitions + NodeWithMapping firstUnchanged = cloneNodeWithMapping(allVariants.get(0).unchanged()); + RowExpression stalePredicate = buildPropagatedStalePredicate(firstUnchanged.getNode(), originalSources.get(1)); + NodeWithMapping filteredFirst = new NodeWithMapping( + buildFilter(firstUnchanged.getNode(), stalePredicate), + firstUnchanged.getMapping()); + + List sources = new ArrayList<>(); + sources.add(filteredFirst); + for (int i = 1; i < allVariants.size(); i++) { + sources.add(cloneNodeWithMapping(allVariants.get(i).current())); + } + return buildExceptFromSources(original, sources); + } + + private ExceptNode buildExceptFromSources(ExceptNode original, List sources) + { + Map mapping = + extendMapping(combineMapping(sources.toArray(new NodeWithMapping[0])), original.getOutputVariables()); + List sourceNodes = sources.stream().map(NodeWithMapping::getNode).collect(toImmutableList()); + return new SymbolMapper(mapping, warningCollector).map(original, sourceNodes, idAllocator.getNextId()); + } + + private NodeWithMapping buildExcept(ExceptNode original, List sources) + { + Map mapping = + extendMapping(combineMapping(sources.toArray(new NodeWithMapping[0])), original.getOutputVariables()); + List sourceNodes = sources.stream().map(NodeWithMapping::getNode).collect(toImmutableList()); + return new NodeWithMapping( + new SymbolMapper(mapping, warningCollector).map(original, sourceNodes, idAllocator.getNextId()), + mapping); + } + + @Override + public PlanVariants visitSort(SortNode node, Void context) + { + // Sort cannot be stitched: UNION of sorted subsets is not globally sorted + throw new UnsupportedOperationException( + "Sort cannot be differentially stitched: UNION of sorted partitions does not preserve global ordering"); + } + + @Override + public PlanVariants visitLimit(LimitNode node, Void context) + { + // Limit cannot be stitched: UNION of limited subsets may have wrong row count + throw new UnsupportedOperationException( + "Limit cannot be differentially stitched: UNION of limited partitions may return incorrect row count"); + } + + @Override + public PlanVariants visitTopN(TopNNode node, Void context) + { + // TopN cannot be stitched: UNION of top-N subsets is not global top-N + throw new UnsupportedOperationException( + "TopN cannot be differentially stitched: UNION of top-N partitions does not preserve global top-N ordering"); + } + + @Override + public PlanVariants visitGroupReference(GroupReference node, Void context) + { + return lookup.resolve(node).accept(this, context); + } + + private void checkDeterministic(RowExpression expression, String context) + { + if (!determinismEvaluator.isDeterministic(expression)) { + throw new UnsupportedOperationException("Non-deterministic expression in " + context); + } + } + + /** + * Builds a stale predicate for the left side of INTERSECT/EXCEPT by finding stale TableScans + * in the right subtree and rewriting their stale predicates to left-side columns + * using column equivalences from MV metadata. + */ + private RowExpression buildPropagatedStalePredicate(PlanNode leftSubtree, PlanNode rightSubtree) + { + // Build column-to-variable mapping for the left subtree + Map leftColumnMapping = + buildColumnToVariableMapping(metadata, session, leftSubtree, lookup); + + List predicates = searchFrom(rightSubtree, lookup) + .where(TableScanNode.class::isInstance) + .findAll() + .stream() + .map(TableScanNode.class::cast) + .flatMap(tableScan -> rewriteStalePredicatesToLeftColumns(tableScan, leftColumnMapping).stream()) + .collect(toImmutableList()); + + return or(predicates); + } + + /** + * For a given TableScan on the right side, converts its stale predicates to RowExpressions + * that reference left-side variables using column equivalences from MV metadata. + */ + private List rewriteStalePredicatesToLeftColumns( + TableScanNode tableScan, + Map leftColumnMapping) + { + SchemaTableName tableName = metadata.getTableMetadata(session, tableScan.getTable()).getTable(); + List> stalePredicates = staleConstraints.getOrDefault(tableName, ImmutableList.of()); + return columnEquivalences.translatePredicatesToVariables(tableName, stalePredicates, leftColumnMapping, translator); + } + + private Map createFreshMapping( + List variables) + { + return variables.stream() + .collect(toImmutableMap( + Function.identity(), + expression -> variableAllocator.newVariable(expression.getName(), expression.getType()))); + } + + private Map extendMapping( + Map existing, + List outputVariables) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(existing); + outputVariables.stream() + .filter(variable -> !existing.containsKey(variable)) + .forEach(variable -> builder.put(variable, variableAllocator.newVariable(variable.getName(), variable.getType()))); + return builder.buildKeepingLast(); + } + + private Map combineMapping(NodeWithMapping... sources) + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + Arrays.stream(sources).map(NodeWithMapping::getMapping).forEach(builder::putAll); + return builder.buildKeepingLast(); + } + + private NodeWithMapping createBinaryUnion(PlanNode original, PlanNode left, PlanNode right) + { + List leftOutputs = left.getOutputVariables(); + List rightOutputs = right.getOutputVariables(); + + List unionOutputs = new ArrayList<>(); + Map> variableMapping = new HashMap<>(); + ImmutableMap.Builder outputMapping = ImmutableMap.builder(); + + for (int i = 0; i < leftOutputs.size(); i++) { + VariableReferenceExpression leftVar = leftOutputs.get(i); + VariableReferenceExpression rightVar = rightOutputs.get(i); + VariableReferenceExpression outputVar = variableAllocator.newVariable(leftVar.getName(), leftVar.getType()); + unionOutputs.add(outputVar); + variableMapping.put(outputVar, ImmutableList.of(leftVar, rightVar)); + + if (i < original.getOutputVariables().size()) { + outputMapping.put(original.getOutputVariables().get(i), outputVar); + } + } + + UnionNode unionNode = new UnionNode( + original.getSourceLocation(), + idAllocator.getNextId(), + Optional.empty(), + ImmutableList.of(left, right), + unionOutputs, + variableMapping); + + return new NodeWithMapping(unionNode, outputMapping.build()); + } + + private NodeWithMapping cloneNodeWithMapping(NodeWithMapping original) + { + SubtreeRemappingVisitor visitor = new SubtreeRemappingVisitor(); + PlanNode clonedPlan = original.getNode().accept(visitor, null); + Map variableRenaming = visitor.getMapping(); + + // Compose mappings: for each (origVar, currVar) in original mapping, + // the new mapping is (origVar, renaming[currVar]) + ImmutableMap.Builder newMapping = ImmutableMap.builder(); + for (Map.Entry entry : original.getMapping().entrySet()) { + VariableReferenceExpression renamedVariable = variableRenaming.get(entry.getValue()); + if (renamedVariable != null) { + newMapping.put(entry.getKey(), renamedVariable); + } + } + + return new NodeWithMapping(clonedPlan, newMapping.build()); + } + + private class SubtreeRemappingVisitor + extends InternalPlanVisitor + { + private final Map mapping = new HashMap<>(); + + public Map getMapping() + { + return ImmutableMap.copyOf(mapping); + } + + private void ensureVariablesMapped(List variables) + { + for (VariableReferenceExpression variable : variables) { + mapping.computeIfAbsent(variable, v -> variableAllocator.newVariable(v.getName(), v.getType())); + } + } + + private SymbolMapper getMapper() + { + return new SymbolMapper(mapping, warningCollector); + } + + @Override + public PlanNode visitPlan(PlanNode node, Void context) + { + throw new UnsupportedOperationException( + "Cannot clone node type: " + node.getClass().getSimpleName() + + ". Add visitXxx method to SubtreeRemappingVisitor to support this node type."); + } + + @Override + public PlanNode visitTableScan(TableScanNode node, Void context) + { + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, idAllocator.getNextId()); + } + + @Override + public PlanNode visitFilter(FilterNode node, Void context) + { + PlanNode newSource = node.getSource().accept(this, context); + // Filter passes through source variables, no new allocations needed + return getMapper().map(node, newSource, idAllocator.getNextId()); + } + + @Override + public PlanNode visitProject(ProjectNode node, Void context) + { + PlanNode newSource = node.getSource().accept(this, context); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newSource, idAllocator.getNextId()); + } + + @Override + public PlanNode visitAggregation(AggregationNode node, Void context) + { + PlanNode newSource = node.getSource().accept(this, context); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newSource, idAllocator.getNextId()); + } + + @Override + public PlanNode visitJoin(JoinNode node, Void context) + { + PlanNode newLeft = node.getLeft().accept(this, context); + PlanNode newRight = node.getRight().accept(this, context); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newLeft, newRight, idAllocator.getNextId()); + } + + @Override + public PlanNode visitUnion(UnionNode node, Void context) + { + List newSources = node.getSources().stream() + .map(source -> source.accept(this, context)) + .collect(toImmutableList()); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newSources, idAllocator.getNextId()); + } + + @Override + public PlanNode visitIntersect(IntersectNode node, Void context) + { + List newSources = node.getSources().stream() + .map(source -> source.accept(this, context)) + .collect(toImmutableList()); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newSources, idAllocator.getNextId()); + } + + @Override + public PlanNode visitExcept(ExceptNode node, Void context) + { + List newSources = node.getSources().stream() + .map(source -> source.accept(this, context)) + .collect(toImmutableList()); + ensureVariablesMapped(node.getOutputVariables()); + return getMapper().map(node, newSources, idAllocator.getNextId()); + } + + @Override + public PlanNode visitGroupReference(GroupReference node, Void context) + { + throw new IllegalStateException( + "GroupReference should have been resolved by DeltaBuilder before cloning. " + + "This indicates the plan was not fully resolved before SubtreeRemappingVisitor was invoked."); + } + } + } + + /** + * A plan node paired with its variable mapping (original variable → new variable). + * Used for results of node building operations. + */ + public static class NodeWithMapping + { + private final PlanNode node; + private final Map mapping; + + NodeWithMapping(PlanNode node, Map mapping) + { + this.node = requireNonNull(node, "node is null"); + this.mapping = ImmutableMap.copyOf(requireNonNull(mapping, "mapping is null")); + } + + public PlanNode getNode() + { + return node; + } + + public Map getMapping() + { + return mapping; + } + } + + /** + * Three plan variants for IVM (matching the algebraic framework): + *

    + *
  • delta (∆R): rows from stale partitions — what changed
    • + *
  • current (R'): complete current state
    • + *
  • unchanged (R): rows from non-stale partitions — R'[non-stale] = R[non-stale]
    • + *
+ * + *

R' = R ∪ ∆R (for insert-only, partition-aligned staleness) + */ + private static class PlanVariants + { + private final NodeWithMapping delta; + private final NodeWithMapping current; + private final NodeWithMapping unchanged; + + PlanVariants(NodeWithMapping delta, NodeWithMapping current, NodeWithMapping unchanged) + { + this.delta = requireNonNull(delta, "delta is null"); + this.current = requireNonNull(current, "current is null"); + this.unchanged = requireNonNull(unchanged, "unchanged is null"); + } + + NodeWithMapping delta() + { + return delta; + } + + NodeWithMapping current() + { + return current; + } + + NodeWithMapping unchanged() + { + return unchanged; + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MaterializedViewRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/MaterializedViewRewrite.java similarity index 52% rename from presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MaterializedViewRewrite.java rename to presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/MaterializedViewRewrite.java index 68f3d892a360a..c3b600e079ad8 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/MaterializedViewRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/MaterializedViewRewrite.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.iterative.rule; +package com.facebook.presto.sql.planner.iterative.rule.materializedview; import com.facebook.airlift.units.Duration; import com.facebook.presto.Session; @@ -23,15 +23,19 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewStaleReadBehavior; import com.facebook.presto.spi.MaterializedViewStalenessConfig; import com.facebook.presto.spi.MaterializedViewStatus; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.analyzer.MetadataResolver; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.MaterializedViewScanNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.security.AccessControl; @@ -44,18 +48,40 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.getMaterializedViewStaleReadBehavior; +import static com.facebook.presto.SystemSessionProperties.getMaterializedViewStalenessWindow; import static com.facebook.presto.SystemSessionProperties.isLegacyMaterializedViews; -import static com.facebook.presto.spi.MaterializedViewStaleReadBehavior.USE_VIEW_QUERY; +import static com.facebook.presto.SystemSessionProperties.isMaterializedViewForceStale; +import static com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates; import static com.facebook.presto.spi.StandardErrorCode.MATERIALIZED_VIEW_STALE; +import static com.facebook.presto.spi.StandardWarningCode.MATERIALIZED_VIEW_ACCESS_CONTROL_FALLBACK; +import static com.facebook.presto.spi.StandardWarningCode.MATERIALIZED_VIEW_STALE_DATA; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; import static com.facebook.presto.spi.security.ViewSecurity.DEFINER; import static com.facebook.presto.spi.security.ViewSecurity.INVOKER; +import static com.facebook.presto.sql.planner.iterative.rule.materializedview.DifferentialPlanRewriter.buildStitchedPlan; import static com.facebook.presto.sql.planner.plan.Patterns.materializedViewScan; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.System.currentTimeMillis; import static java.util.Objects.requireNonNull; +/** + * Rewrites {@link MaterializedViewScanNode} to use pre-computed data when possible. + * + *

Controlled by the {@code materialized_view_stale_read_behavior} session property: + *

    + *
  • {@code FAIL}: Fail the query if stale.
    • + *
  • {@code USE_STITCHING}: If fully fresh, use data table; if partially stale, build + * a stitched plan combining fresh data with recomputed stale data; otherwise fall back + * to full recompute.
    • + *
  • {@code USE_VIEW_QUERY}: Always execute the original view query (full recompute).
    • + *
+ * + *

For INVOKER security mode views, the data table cannot be used if row filters or column + * masks exist on base tables, since these depend on the invoking user's identity. + * + * @see DifferentialPlanRewriter + */ public class MaterializedViewRewrite implements Rule { @@ -80,80 +106,137 @@ public Result apply(MaterializedViewScanNode node, Captures captures, Context co Session session = context.getSession(); checkState(!isLegacyMaterializedViews(session), "Materialized view rewrite rule should not fire when legacy materialized views are enabled"); + VariableAllocator variableAllocator = context.getVariableAllocator(); + PlanNodeIdAllocator idAllocator = context.getIdAllocator(); MetadataResolver metadataResolver = metadata.getMetadataResolver(session); - boolean useDataTable = isUseDataTable(node, metadataResolver, session); - PlanNode chosenPlan = useDataTable ? node.getDataTablePlan() : node.getViewQueryPlan(); - Map chosenMappings = - useDataTable ? node.getDataTableMappings() : node.getViewQueryMappings(); + Optional materializedViewDefinition = metadataResolver.getMaterializedView(node.getMaterializedViewName()); + checkState(materializedViewDefinition.isPresent(), "Materialized view definition not found for: %s", node.getMaterializedViewName()); + MaterializedViewDefinition definition = materializedViewDefinition.get(); + + MaterializedViewStatus status = metadataResolver.getMaterializedViewStatus(node.getMaterializedViewName(), TupleDomain.all()); + + MaterializedViewStaleReadBehavior staleReadBehavior = definition.getStalenessConfig() + .map(MaterializedViewStalenessConfig::getStaleReadBehavior) + .orElseGet(() -> getMaterializedViewStaleReadBehavior(session)); + + Duration stalenessWindow = definition.getStalenessConfig() + .map(MaterializedViewStalenessConfig::getStalenessWindow) + .orElseGet(() -> getMaterializedViewStalenessWindow(session).orElse(Duration.valueOf("0s"))); + + boolean canUseDataTable = canUseDataTable(session, context, node, metadataResolver, definition, status, staleReadBehavior, stalenessWindow); + boolean shouldStitch = shouldPerformStitching(status, staleReadBehavior, stalenessWindow); + if (!status.isFullyMaterialized() && !status.getPartitionsFromBaseTables().isEmpty()) { + Map constraints = status.getPartitionsFromBaseTables(); + + if (shouldStitch && canUseDataTableWithSecurityChecks(node, metadataResolver, session, definition, context)) { + Optional unionPlan = buildStitchedPlan( + metadata, + session, + node, + constraints, + definition, + variableAllocator, + idAllocator, + context.getLookup(), + context.getWarningCollector()); + + if (unionPlan.isPresent()) { + return Result.ofPlanNode(unionPlan.get()); + } + } + } + + PlanNode plan; + Map mappings; + if (canUseDataTable && !shouldStitch) { + plan = node.getDataTablePlan(); + mappings = node.getDataTableMappings(); + } + else { + plan = node.getViewQueryPlan(); + mappings = node.getViewQueryMappings(); + } Assignments.Builder assignments = Assignments.builder(); for (VariableReferenceExpression outputVariable : node.getOutputVariables()) { - VariableReferenceExpression sourceVariable = chosenMappings.get(outputVariable); + VariableReferenceExpression sourceVariable = mappings.get(outputVariable); requireNonNull(sourceVariable, "No mapping found for output variable: " + outputVariable); assignments.put(outputVariable, sourceVariable); } return Result.ofPlanNode(new ProjectNode( node.getSourceLocation(), - context.getIdAllocator().getNextId(), - chosenPlan, + idAllocator.getNextId(), + plan, assignments.build(), LOCAL)); } - private boolean isUseDataTable(MaterializedViewScanNode node, MetadataResolver metadataResolver, Session session) + private boolean canUseDataTable( + Session session, + Context context, + MaterializedViewScanNode node, + MetadataResolver metadataResolver, + MaterializedViewDefinition definition, + MaterializedViewStatus status, + MaterializedViewStaleReadBehavior staleReadBehavior, + Duration stalenessWindow) { - Optional materializedViewDefinition = metadataResolver.getMaterializedView(node.getMaterializedViewName()); - checkState(materializedViewDefinition.isPresent(), "Materialized view definition not found for: %s", node.getMaterializedViewName()); - MaterializedViewDefinition definition = materializedViewDefinition.get(); + if (isMaterializedViewForceStale(session)) { + return shouldUseDataTableWhenStale(staleReadBehavior, node.getMaterializedViewName()); + } - MaterializedViewStatus status = metadataResolver.getMaterializedViewStatus(node.getMaterializedViewName(), TupleDomain.all()); if (status.isFullyMaterialized()) { - return canUseDataTableWithSecurityChecks(node, metadataResolver, session, definition); + return canUseDataTableWithSecurityChecks(node, metadataResolver, session, definition, context); } - Optional stalenessConfig = definition.getStalenessConfig(); - if (stalenessConfig.isPresent()) { - MaterializedViewStalenessConfig config = stalenessConfig.get(); - - if (isStalenessBeyondTolerance(config, status)) { - return applyStaleReadBehavior(config, node.getMaterializedViewName()); - } - return canUseDataTableWithSecurityChecks(node, metadataResolver, session, definition); + if (isWithinStalenessWindow(status, stalenessWindow)) { + context.getWarningCollector().add(new PrestoWarning( + MATERIALIZED_VIEW_STALE_DATA, + "Materialized view " + node.getMaterializedViewName() + " is stale but within the configured staleness window; results may not reflect the latest base table data")); + return canUseDataTableWithSecurityChecks(node, metadataResolver, session, definition, context); } + return shouldUseDataTableWhenStale(staleReadBehavior, node.getMaterializedViewName()); + } - if (getMaterializedViewStaleReadBehavior(session) == USE_VIEW_QUERY) { - return false; - } - throw new PrestoException( - MATERIALIZED_VIEW_STALE, - String.format("Materialized view '%s' is stale (base tables have changed since last refresh)", node.getMaterializedViewName())); + private boolean isWithinStalenessWindow(MaterializedViewStatus status, Duration stalenessWindow) + { + return status.getLastFreshTime() + .map(time -> (currentTimeMillis() - time) <= stalenessWindow.toMillis()) + .orElse(false); } - private boolean isStalenessBeyondTolerance( - MaterializedViewStalenessConfig config, - MaterializedViewStatus status) + private boolean shouldPerformStitching( + MaterializedViewStatus status, + MaterializedViewStaleReadBehavior staleReadBehavior, + Duration stalenessWindow) { - Duration stalenessWindow = config.getStalenessWindow(); + if (status.isFullyMaterialized()) { + return false; + } + + // If within staleness window, just return stale data - don't do stitching + if (isWithinStalenessWindow(status, stalenessWindow)) { + return false; + } - Optional lastFreshTime = status.getLastFreshTime(); - return lastFreshTime - .map(time -> (currentTimeMillis() - time) > stalenessWindow.toMillis()) - .orElse(true); + return staleReadBehavior == MaterializedViewStaleReadBehavior.USE_STITCHING; } - private boolean applyStaleReadBehavior(MaterializedViewStalenessConfig config, QualifiedObjectName viewName) + private boolean shouldUseDataTableWhenStale(MaterializedViewStaleReadBehavior behavior, QualifiedObjectName viewName) { - switch (config.getStaleReadBehavior()) { + switch (behavior) { case FAIL: throw new PrestoException( MATERIALIZED_VIEW_STALE, - String.format("Materialized view '%s' is stale beyond the configured staleness window", viewName)); + String.format("Materialized view '%s' is stale", viewName)); case USE_VIEW_QUERY: return false; + case USE_STITCHING: + return true; default: - throw new IllegalStateException("Unexpected stale read behavior: " + config.getStaleReadBehavior()); + throw new IllegalStateException("Unexpected stale read behavior: " + behavior); } } @@ -161,7 +244,8 @@ private boolean canUseDataTableWithSecurityChecks( MaterializedViewScanNode node, MetadataResolver metadataResolver, Session session, - MaterializedViewDefinition definition) + MaterializedViewDefinition definition, + Context context) { // Security mode defaults to INVOKER for legacy materialized views created without explicitly specifying it ViewSecurity securityMode = definition.getSecurityMode().orElse(INVOKER); @@ -172,13 +256,10 @@ private boolean canUseDataTableWithSecurityChecks( return true; } - // Invoker rights: need to check for row filters and column masks on base tables because they may alter - // the data returned by the materialized view depending on the invoker's permissions. String catalogName = node.getMaterializedViewName().getCatalogName(); for (SchemaTableName schemaTableName : definition.getBaseTables()) { QualifiedObjectName baseTable = new QualifiedObjectName(catalogName, schemaTableName.getSchemaName(), schemaTableName.getTableName()); - // Check for row filters on this base table List rowFilters = accessControl.getRowFilters( session.getTransactionId().get(), session.getIdentity(), @@ -186,6 +267,10 @@ private boolean canUseDataTableWithSecurityChecks( baseTable); if (!rowFilters.isEmpty()) { + context.getWarningCollector().add(new PrestoWarning( + MATERIALIZED_VIEW_ACCESS_CONTROL_FALLBACK, + "Cannot use materialized view data table for " + node.getMaterializedViewName() + + ": row filters exist on base table " + baseTable + " with INVOKER security mode")); return false; } @@ -194,7 +279,6 @@ private boolean canUseDataTableWithSecurityChecks( return false; } - // Check for column masks on this base table Map columnHandles = metadata.getColumnHandles(session, tableHandle.get()); List columnsMetadata = columnHandles.values().stream() .map(handle -> metadata.getColumnMetadata(session, tableHandle.get(), handle)) @@ -208,11 +292,14 @@ private boolean canUseDataTableWithSecurityChecks( columnsMetadata); if (!columnMasks.isEmpty()) { + context.getWarningCollector().add(new PrestoWarning( + MATERIALIZED_VIEW_ACCESS_CONTROL_FALLBACK, + "Cannot use materialized view data table for " + node.getMaterializedViewName() + + ": column masks exist on base table " + baseTable + " with INVOKER security mode")); return false; } } - // No row filters or column masks found on base tables, safe to use data table return true; } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/PassthroughColumnEquivalences.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/PassthroughColumnEquivalences.java new file mode 100644 index 0000000000000..fab590e3c1656 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/PassthroughColumnEquivalences.java @@ -0,0 +1,198 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule.materializedview; + +import com.facebook.presto.common.predicate.Domain; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewDefinition.TableColumn; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.facebook.presto.util.DisjointSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +/** + * Captures column equivalence information for passthrough (direct-mapped) columns. + * + *

Only columns marked as {@code isDirectMapped=true} are included in equivalence classes. + * This is a safety constraint: passthrough columns contain exactly the same data + * as the base table columns, allowing predicates to be safely translated between them. + * + *

Columns that are transformed (e.g., {@code COALESCE(dt, '2024-01-01')}) are NOT + * included because predicate translation through transformations could produce incorrect + * results. For example, a base table row with {@code dt=NULL} would not match + * {@code dt='2024-01-01'}, but the MV row would match after the COALESCE transformation. + * + *

Each equivalence class contains columns that are equal due to: + *

    + *
  • Direct passthrough from base table to MV output
    • + *
  • Join conditions where both sides are passthrough (e.g., A.dt = B.dt both map to mv.dt)
    • + *
+ */ +public class PassthroughColumnEquivalences +{ + // Maps each column to its equivalence class (the set of all equivalent columns) + private final Map> columnToEquivalenceClass; + // All known tables (base tables + data table) for validation + private final Set knownTables; + + public PassthroughColumnEquivalences(MaterializedViewDefinition materializedViewDefinition, SchemaTableName dataTable) + { + requireNonNull(materializedViewDefinition, "materializedViewDefinition is null"); + requireNonNull(dataTable, "dataTable is null"); + + DisjointSet equivalences = new DisjointSet<>(); + + for (MaterializedViewDefinition.ColumnMapping mapping : materializedViewDefinition.getColumnMappings()) { + TableColumn dataColumn = new TableColumn(dataTable, mapping.getViewColumn().getColumnName()); + + for (TableColumn baseColumn : mapping.getBaseTableColumns()) { + if (baseColumn.isDirectMapped().orElse(true)) { + equivalences.findAndUnion(dataColumn, baseColumn); + } + } + } + + // Build the column-to-equivalence-class map from DisjointSet + ImmutableMap.Builder> builder = ImmutableMap.builder(); + for (Set equivalenceClass : equivalences.getEquivalentClasses()) { + if (equivalenceClass.size() > 1) { + ImmutableSet immutableClass = ImmutableSet.copyOf(equivalenceClass); + for (TableColumn column : equivalenceClass) { + builder.put(column, immutableClass); + } + } + } + this.columnToEquivalenceClass = builder.build(); + + this.knownTables = ImmutableSet.builder() + .add(dataTable) + .addAll(materializedViewDefinition.getBaseTables()) + .build(); + } + + public boolean hasEquivalence(TableColumn column) + { + return columnToEquivalenceClass.containsKey(column); + } + + /** + * Returns equivalent predicates for other tables based on column equivalences. + * Given a predicate on the source table, returns a map from each equivalent table + * to its corresponding predicate with column names mapped through equivalences. + */ + public Map> getEquivalentPredicates( + SchemaTableName sourceTable, + TupleDomain predicate) + { + requireNonNull(sourceTable, "sourceTable is null"); + requireNonNull(predicate, "predicate is null"); + checkState(knownTables.contains(sourceTable), + "Unknown table: %s. Expected one of: %s", sourceTable, knownTables); + + if (!predicate.getDomains().isPresent()) { + return ImmutableMap.of(); + } + + Map> domainsByTargetTable = new HashMap<>(); + + for (Map.Entry entry : predicate.getDomains().get().entrySet()) { + String columnName = entry.getKey(); + Domain domain = entry.getValue(); + + TableColumn sourceColumn = new TableColumn(sourceTable, columnName); + Set equivalents = columnToEquivalenceClass.get(sourceColumn); + if (equivalents == null) { + continue; + } + + for (TableColumn equivalent : equivalents) { + if (!equivalent.equals(sourceColumn)) { + domainsByTargetTable + .computeIfAbsent(equivalent.getTableName(), table -> new HashMap<>()) + .put(equivalent.getColumnName(), domain); + } + } + } + + return domainsByTargetTable.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + tableToDomains -> TupleDomain.withColumnDomains(ImmutableMap.copyOf(tableToDomains.getValue())))); + } + + /** + * Translates stale predicates from a source table to RowExpressions referencing target variables. + * Uses column equivalences to map predicates to equivalent tables, then binds to actual variables. + * + * @param sourceTable The table with stale predicates + * @param stalePredicates List of stale partition predicates (disjuncts) + * @param columnToVariable Mapping from (table, column) to plan variables + * @param translator Converts TupleDomain to RowExpression + * @return List of RowExpressions representing the stale predicates bound to variables + * @throws UnsupportedOperationException if predicates exist but none can be mapped + */ + public List translatePredicatesToVariables( + SchemaTableName sourceTable, + List> stalePredicates, + Map columnToVariable, + RowExpressionDomainTranslator translator) + { + if (stalePredicates.isEmpty()) { + return ImmutableList.of(); + } + + ImmutableList.Builder result = ImmutableList.builder(); + for (TupleDomain stalePredicate : stalePredicates) { + Map> equivalentPredicates = + getEquivalentPredicates(sourceTable, stalePredicate); + + for (Map.Entry> entry : equivalentPredicates.entrySet()) { + SchemaTableName targetTable = entry.getKey(); + TupleDomain targetPredicate = entry.getValue(); + + // Convert column names to variables + TupleDomain variablePredicate = targetPredicate.transform( + col -> columnToVariable.get(new TableColumn(targetTable, col))); + + // Only include if some columns could be mapped (not all dropped) + if (!variablePredicate.isAll() && !variablePredicate.isNone()) { + result.add(translator.toPredicate(variablePredicate)); + } + } + } + + List mappedPredicates = result.build(); + if (mappedPredicates.isEmpty()) { + throw new UnsupportedOperationException( + "Cannot map stale predicates from " + sourceTable + " to equivalent columns. " + + "Column equivalences may be missing or columns may not be directly mapped."); + } + + return mappedPredicates; + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index 1b6741896e2df..54ddb8c4a3430 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -56,10 +56,12 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.plan.WindowNode; +import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.statistics.TableStatistics; @@ -84,7 +86,6 @@ import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; @@ -109,6 +110,8 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Stream; import static com.facebook.presto.SystemSessionProperties.getAggregationPartitioningMergingStrategy; @@ -116,6 +119,8 @@ import static com.facebook.presto.SystemSessionProperties.getHashPartitionCount; import static com.facebook.presto.SystemSessionProperties.getPartialMergePushdownStrategy; import static com.facebook.presto.SystemSessionProperties.getPartitioningProviderCatalog; +import static com.facebook.presto.SystemSessionProperties.getRemoteFunctionFixedParallelismTaskCount; +import static com.facebook.presto.SystemSessionProperties.getRemoteFunctionNamesForFixedParallelism; import static com.facebook.presto.SystemSessionProperties.getTableScanShuffleParallelismThreshold; import static com.facebook.presto.SystemSessionProperties.getTableScanShuffleStrategy; import static com.facebook.presto.SystemSessionProperties.getTaskPartitionedWriterCount; @@ -259,8 +264,30 @@ public PlanWithProperties visitProject(ProjectNode node, PreferredProperties pre { Map identities = computeIdentityTranslations(node.getAssignments()); PreferredProperties translatedPreferred = preferredProperties.translate(symbol -> Optional.ofNullable(identities.get(symbol))); + PlanWithProperties planWithProperties = planChild(node, translatedPreferred); + + if (node.getLocality().equals(ProjectNode.Locality.REMOTE)) { + String functionNameRegex = getRemoteFunctionNamesForFixedParallelism(session); + if (!functionNameRegex.isEmpty()) { + Pattern pattern; + try { + pattern = Pattern.compile(functionNameRegex); + } + catch (PatternSyntaxException e) { + return rebaseAndDeriveProperties(node, planWithProperties); + } + if (node.getAssignments().getExpressions().stream().filter(x -> x instanceof CallExpression) + .anyMatch(x -> pattern.matcher(((CallExpression) x).getFunctionHandle().getName()).matches())) { + int taskCount = getRemoteFunctionFixedParallelismTaskCount(session); + checkState(taskCount > 0, "taskCount should be larger than 0"); + PlanNode newNode = roundRobinExchange(idAllocator.getNextId(), REMOTE_STREAMING, planWithProperties.getNode(), taskCount); + newNode = ChildReplacer.replaceChildren(node, ImmutableList.of(newNode)); + return new PlanWithProperties(newNode, derivePropertiesRecursively(newNode)); + } + } + } - return rebaseAndDeriveProperties(node, planChild(node, translatedPreferred)); + return rebaseAndDeriveProperties(node, planWithProperties); } @Override @@ -580,6 +607,7 @@ public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, PreferredPr idAllocator.getNextId(), child.getNode(), node.getSpecification(), + node.getRankingFunction(), node.getRowNumberVariable(), node.getMaxRowCountPerPartition(), true, diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java index 46d17de0d459c..92307ff34f0db 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java @@ -14,6 +14,11 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.cost.CachingStatsProvider; +import com.facebook.presto.cost.PlanNodeStatsEstimate; +import com.facebook.presto.cost.StatsCalculator; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.cost.VariableStatsEstimate; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConstantProperty; import com.facebook.presto.spi.GroupingProperty; @@ -45,6 +50,7 @@ import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -63,7 +69,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -74,6 +79,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.SystemSessionProperties.getLocalExchangeParentPreferenceStrategy; import static com.facebook.presto.SystemSessionProperties.getTaskConcurrency; import static com.facebook.presto.SystemSessionProperties.getTaskPartitionedWriterCount; import static com.facebook.presto.SystemSessionProperties.getTaskWriterCount; @@ -90,6 +96,7 @@ import static com.facebook.presto.operator.aggregation.AggregationUtils.hasSingleNodeExecutionPreference; import static com.facebook.presto.operator.aggregation.AggregationUtils.isDecomposable; import static com.facebook.presto.sql.TemporaryTableUtil.splitIntoPartialAndIntermediate; +import static com.facebook.presto.sql.analyzer.FeaturesConfig.LocalExchangeParentPreferenceStrategy; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; @@ -123,18 +130,26 @@ public class AddLocalExchanges implements PlanOptimizer { private final Metadata metadata; + private final StatsCalculator statsCalculator; private final boolean nativeExecution; - public AddLocalExchanges(Metadata metadata, boolean nativeExecution) + public AddLocalExchanges(Metadata metadata, StatsCalculator statsCalculator, boolean nativeExecution) { this.metadata = requireNonNull(metadata, "metadata is null"); + this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null"); this.nativeExecution = nativeExecution; } @Override public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider types, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, WarningCollector warningCollector) { - PlanWithProperties result = new Rewriter(variableAllocator, idAllocator, session, nativeExecution).accept(plan, any()); + LocalExchangeParentPreferenceStrategy strategy = getLocalExchangeParentPreferenceStrategy(session); + Optional statsProvider = Optional.empty(); + if (strategy == LocalExchangeParentPreferenceStrategy.AUTOMATIC) { + statsProvider = Optional.of(new CachingStatsProvider(statsCalculator, session, types)); + } + + PlanWithProperties result = new Rewriter(variableAllocator, idAllocator, session, strategy, statsProvider, nativeExecution).accept(plan, any()); boolean optimizerTriggered = PlanNodeSearcher.searchFrom(result.getNode()).where(node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope().isLocal()).findFirst().isPresent(); return PlanOptimizerResult.optimizerResult(result.getNode(), optimizerTriggered); } @@ -146,14 +161,18 @@ private class Rewriter private final PlanNodeIdAllocator idAllocator; private final Session session; private final TypeProvider types; + private final LocalExchangeParentPreferenceStrategy parentPreferenceStrategy; + private final Optional statsProvider; private final boolean nativeExecution; - public Rewriter(VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, Session session, boolean nativeExecution) + public Rewriter(VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator, Session session, LocalExchangeParentPreferenceStrategy parentPreferenceStrategy, Optional statsProvider, boolean nativeExecution) { this.variableAllocator = variableAllocator; this.types = TypeProvider.viewOf(variableAllocator.getVariables()); this.idAllocator = idAllocator; this.session = session; + this.parentPreferenceStrategy = parentPreferenceStrategy; + this.statsProvider = statsProvider; this.nativeExecution = nativeExecution; } @@ -386,7 +405,36 @@ public PlanWithProperties visitAggregation(AggregationNode node, StreamPreferred return rebaseAndDeriveProperties(node, ImmutableList.of(exchange)); } - StreamPreferredProperties childRequirements = parentPreferences + boolean useParentPreferences; + switch (parentPreferenceStrategy) { + case NEVER: + useParentPreferences = false; + break; + case AUTOMATIC: + double parentPartitionCardinality = 0; + if (parentPreferences.getPartitioningColumns().isPresent() && statsProvider.isPresent()) { + parentPartitionCardinality = 1; + PlanNodeStatsEstimate stats = statsProvider.get().getStats(node.getSource()); + for (VariableReferenceExpression partitionColumn : parentPreferences.getPartitioningColumns().get()) { + VariableStatsEstimate varStats = stats.getVariableStatistics(partitionColumn); + double distinctCount = varStats.getDistinctValuesCount(); + if (!Double.isNaN(distinctCount)) { + parentPartitionCardinality *= distinctCount; + } + else { + parentPartitionCardinality = 0; + break; + } + } + } + useParentPreferences = parentPartitionCardinality >= getTaskConcurrency(session); + break; + case ALWAYS: + default: + useParentPreferences = true; + break; + } + StreamPreferredProperties childRequirements = (useParentPreferences ? parentPreferences : StreamPreferredProperties.any()) .constrainTo(node.getSource().getOutputVariables()) .withDefaultParallelism(session) .withPartitioning(groupingKeys); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java index dcd9435290f36..e4fa7e25736db 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/ApplyConnectorOptimization.java @@ -44,6 +44,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -101,7 +102,8 @@ public class ApplyConnectorOptimization UnnestNode.class, TableWriterNode.class, TableFinishNode.class, - DeleteNode.class); + DeleteNode.class, + TopNRowNumberNode.class); // for a leaf node that does not belong to any connector (e.g., ValuesNode) private static final ConnectorId EMPTY_CONNECTOR_ID = new ConnectorId("$internal$ApplyConnectorOptimization_EMPTY_CONNECTOR"); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java index 84f42ef283d6b..3bc150814edb7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HashGenerationOptimizer.java @@ -34,6 +34,7 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SpatialJoinNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.WindowNode; @@ -49,7 +50,6 @@ import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; @@ -330,6 +330,7 @@ public PlanWithProperties visitTopNRowNumber(TopNRowNumberNode node, HashComputa node.getId(), child.getNode(), node.getSpecification(), + node.getRankingFunction(), node.getRowNumberVariable(), node.getMaxRowCountPerPartition(), node.isPartial(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java index 185bee6925035..509eeb9ff5a80 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/HistoricalStatisticsEquivalentPlanMarkingOptimizer.java @@ -26,10 +26,10 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.StatsEquivalentPlanNodeWithLimit; import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinPrefilter.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinPrefilter.java index c590bf6a6cb1f..1f3581d29233f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinPrefilter.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/JoinPrefilter.java @@ -14,7 +14,6 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; -import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.type.VarcharType; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; @@ -27,7 +26,6 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.SemiJoinNode; -import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.TypeProvider; @@ -42,7 +40,6 @@ import java.util.stream.IntStream; import static com.facebook.presto.SystemSessionProperties.isJoinPrefilterEnabled; -import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; @@ -50,13 +47,11 @@ import static com.facebook.presto.spi.plan.JoinType.LEFT; import static com.facebook.presto.sql.planner.PlannerUtils.addProjections; import static com.facebook.presto.sql.planner.PlannerUtils.clonePlanNode; +import static com.facebook.presto.sql.planner.PlannerUtils.getVariableHash; import static com.facebook.presto.sql.planner.PlannerUtils.isScanFilterProject; -import static com.facebook.presto.sql.planner.PlannerUtils.orNullHashCode; import static com.facebook.presto.sql.planner.PlannerUtils.projectExpressions; import static com.facebook.presto.sql.planner.PlannerUtils.restrictOutput; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; -import static com.facebook.presto.sql.relational.Expressions.call; -import static com.facebook.presto.sql.relational.Expressions.callOperator; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; @@ -208,7 +203,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) PlanNode leftKeys = clonePlanNode(rewrittenLeft, session, metadata, idAllocator, leftKeyList, leftVarMap); ImmutableList.Builder expressionsToProject = ImmutableList.builder(); if (hashJoinKey) { - RowExpression hashExpression = getVariableHash(leftKeyList); + RowExpression hashExpression = getVariableHash(leftKeyList, functionAndTypeManager); expressionsToProject.add(hashExpression); } else { @@ -218,7 +213,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) VariableReferenceExpression rightKeyToFilter = rightKeyList.get(0); if (hashJoinKey) { - RowExpression hashExpression = getVariableHash(rightKeyList); + RowExpression hashExpression = getVariableHash(rightKeyList, functionAndTypeManager); rightKeyToFilter = variableAllocator.newVariable(hashExpression); rewrittenRight = addProjections(rewrittenRight, idAllocator, ImmutableMap.of(rightKeyToFilter, hashExpression)); } @@ -273,19 +268,5 @@ public boolean isPlanChanged() { return planChanged; } - - private RowExpression getVariableHash(List inputVariables) - { - List hashExpressionList = inputVariables.stream().map(keyVariable -> - callOperator(functionAndTypeManager.getFunctionAndTypeResolver(), OperatorType.XX_HASH_64, BIGINT, keyVariable)).collect(toImmutableList()); - RowExpression hashExpression = hashExpressionList.get(0); - if (hashExpressionList.size() > 1) { - hashExpression = orNullHashCode(hashExpression); - for (int i = 1; i < hashExpressionList.size(); ++i) { - hashExpression = call(functionAndTypeManager, "combine_hash", BIGINT, hashExpression, orNullHashCode(hashExpressionList.get(i))); - } - } - return hashExpression; - } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java index 1c7856a62caea..e2fbc4575bc8b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java @@ -33,6 +33,7 @@ import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -40,7 +41,6 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.type.TypeUtils; import com.google.common.collect.ImmutableList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PartitioningUtils.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PartitioningUtils.java index e193cbc62f779..afe95cdcff0a4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PartitioningUtils.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PartitioningUtils.java @@ -179,6 +179,9 @@ else if (rightArgument instanceof VariableReferenceExpression) { public static boolean isPartitionedOn(Partitioning partitioning, Collection columns, Set knownConstants) { + if (partitioning.getArguments().isEmpty()) { + return partitioning.getHandle().isSingleNode() || partitioning.getHandle().isCoordinatorOnly(); + } for (RowExpression argument : partitioning.getArguments()) { // partitioned on (k_1, k_2, ..., k_n) => partitioned on (k_1, k_2, ..., k_n, k_n+1, ...) // can safely ignore all constant columns when comparing partition properties diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanNodeDecorrelator.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanNodeDecorrelator.java index 9915c495762be..590f4c507b3a4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanNodeDecorrelator.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PlanNodeDecorrelator.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -36,7 +37,6 @@ import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; @@ -310,6 +310,7 @@ public Optional visitTopN(TopNNode node, Void context) new DataOrganizationSpecification( ImmutableList.copyOf(childDecorrelationResult.variablesToPropagate), Optional.of(orderingScheme)), + TopNRowNumberNode.RankingFunction.ROW_NUMBER, variableAllocator.newVariable("row_number", BIGINT), toIntExact(node.getCount()), false, diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java index 492df9b19fc99..ebbfa738a0417 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PrefilterForLimitingAggregation.java @@ -57,8 +57,8 @@ import static com.facebook.presto.sql.planner.PlannerUtils.addProjections; import static com.facebook.presto.sql.planner.PlannerUtils.clonePlanNode; import static com.facebook.presto.sql.planner.PlannerUtils.createMapType; -import static com.facebook.presto.sql.planner.PlannerUtils.getHashExpression; import static com.facebook.presto.sql.planner.PlannerUtils.getTableScanNodeWithOnlyFilterAndProject; +import static com.facebook.presto.sql.planner.PlannerUtils.getVariableHash; import static com.facebook.presto.sql.planner.PlannerUtils.projectExpressions; import static com.facebook.presto.sql.planner.optimizations.JoinNodeUtils.typeConvert; import static com.facebook.presto.sql.planner.plan.ChildReplacer.replaceChildren; @@ -223,8 +223,8 @@ private PlanNode addPrefilter(AggregationNode aggregationNode, long count) SystemSessionProperties.getPrefilterForGroupbyLimitTimeoutMS(session)); FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager(); - RowExpression leftHashExpression = getHashExpression(functionAndTypeManager, keys).get(); - RowExpression rightHashExpression = getHashExpression(functionAndTypeManager, timedDistinctLimitNode.getOutputVariables()).get(); + RowExpression leftHashExpression = getVariableHash(keys, functionAndTypeManager); + RowExpression rightHashExpression = getVariableHash(timedDistinctLimitNode.getOutputVariables(), functionAndTypeManager); Type mapType = createMapType(functionAndTypeManager, BIGINT, BOOLEAN); PlanNode rightProjectNode = projectExpressions(timedDistinctLimitNode, idAllocator, variableAllocator, ImmutableList.of(rightHashExpression, constant(TRUE, BOOLEAN)), ImmutableList.of()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index 7dc181ca661e0..2291096606f44 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -50,6 +50,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.plan.WindowNode; @@ -77,7 +78,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.google.common.collect.ImmutableBiMap; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index 201ec219823af..032c2ad3a2406 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -47,6 +47,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -70,7 +71,6 @@ import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; @@ -785,6 +785,7 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext context) return context.defaultRewrite(node, context.get()); } + @Override + public PlanNode visitUpdate(UpdateNode node, RewriteContext context) + { + context.get().variables.addAll(node.getSource().getOutputVariables()); + return context.defaultRewrite(node, context.get()); + } + @Override public PlanNode visitTopN(TopNNode node, RewriteContext context) { @@ -820,6 +830,7 @@ private static final class SubfieldExtractor private final FunctionAndTypeManager functionAndTypeManager; private final boolean isPushDownSubfieldsFromLambdasEnabled; private final boolean isPushdownSubfieldsForMapFunctionsEnabled; + private final boolean isPushdownSubfieldsForCardinalityEnabled; private SubfieldExtractor( FunctionResolution functionResolution, @@ -835,11 +846,26 @@ private SubfieldExtractor( requireNonNull(session); this.isPushDownSubfieldsFromLambdasEnabled = isPushdownSubfieldsFromArrayLambdasEnabled(session); this.isPushdownSubfieldsForMapFunctionsEnabled = isPushSubfieldsForMapFunctionsEnabled(session); + this.isPushdownSubfieldsForCardinalityEnabled = isPushSubfieldsForCardinalityEnabled(session); } @Override public Void visitCall(CallExpression call, Context context) { + if (isPushdownSubfieldsForCardinalityEnabled && functionResolution.isCardinalityFunction(call.getFunctionHandle()) && call.getArguments().size() == 1) { + RowExpression argument = call.getArguments().get(0); + if (argument instanceof VariableReferenceExpression) { + Type argumentType = argument.getType(); + if (argumentType instanceof MapType || argumentType instanceof ArrayType) { + VariableReferenceExpression variable = (VariableReferenceExpression) argument; + Subfield cardinalitySubfield = new Subfield( + variable.getName(), + ImmutableList.of(structureOnly())); + context.subfields.add(cardinalitySubfield); + return null; + } + } + } ComplexTypeFunctionDescriptor functionDescriptor = functionAndTypeManager.getFunctionMetadata(call.getFunctionHandle()).getDescriptor(); if (isSubscriptOrElementAtFunction(call, functionResolution, functionAndTypeManager) || isMapSubSetWithConstantArray(call, functionResolution) || isMapFilterWithConstantFilterInMapKey(call, functionResolution)) { Optional> subfield = toSubfield(call, functionResolution, expressionOptimizer, connectorSession, functionAndTypeManager, isPushdownSubfieldsForMapFunctionsEnabled); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java index c6c0077747caa..e9e5d05d4df09 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SimplifyPlanWithEmptyInput.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -43,7 +44,6 @@ import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.ImmutableList; import java.util.ArrayList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java index 3effedf6638fb..fe7086c8b91d5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java @@ -40,6 +40,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -66,7 +67,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java index 824c392ce9619..d80d4f98bdac9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java @@ -16,23 +16,34 @@ import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.expressions.RowExpressionRewriter; import com.facebook.presto.expressions.RowExpressionTreeRewriter; +import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.DataOrganizationSpecification; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.ExchangeEncoding; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.IntersectNode; +import com.facebook.presto.spi.plan.JoinNode; import com.facebook.presto.spi.plan.Ordering; import com.facebook.presto.spi.plan.OrderingScheme; import com.facebook.presto.spi.plan.PartitioningScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.StatisticAggregations; import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.plan.TableFinishNode; +import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -204,7 +215,7 @@ public AggregationNode map(AggregationNode node, PlanNode source, PlanNodeIdAllo return map(node, source, idAllocator.getNextId()); } - private AggregationNode map(AggregationNode node, PlanNode source, PlanNodeId newNodeId) + public AggregationNode map(AggregationNode node, PlanNode source, PlanNodeId newNodeId) { ImmutableMap.Builder aggregations = ImmutableMap.builder(); for (Entry entry : node.getAggregations().entrySet()) { @@ -476,6 +487,128 @@ public TableFunctionProcessorNode map(TableFunctionProcessorNode node, PlanNode node.getHandle()); } + public FilterNode map(FilterNode node, PlanNode source, PlanNodeId newNodeId) + { + return new FilterNode( + node.getSourceLocation(), + newNodeId, + source, + map(node.getPredicate())); + } + + public ProjectNode map(ProjectNode node, PlanNode source, PlanNodeId newNodeId) + { + Assignments.Builder assignmentsBuilder = Assignments.builder(); + for (Entry entry : node.getAssignments().entrySet()) { + assignmentsBuilder.put(map(entry.getKey()), map(entry.getValue())); + } + + return new ProjectNode( + node.getSourceLocation(), + newNodeId, + node.getStatsEquivalentPlanNode(), + source, + assignmentsBuilder.build(), + node.getLocality()); + } + + public SortNode map(SortNode node, PlanNode source, PlanNodeId newNodeId) + { + return new SortNode( + node.getSourceLocation(), + newNodeId, + source, + map(node.getOrderingScheme()), + node.isPartial(), + map(node.getPartitionBy())); + } + + public TableScanNode map(TableScanNode node, PlanNodeId newNodeId) + { + ImmutableMap.Builder newAssignments = ImmutableMap.builder(); + for (Entry entry : node.getAssignments().entrySet()) { + newAssignments.put(map(entry.getKey()), entry.getValue()); + } + + return new TableScanNode( + node.getSourceLocation(), + newNodeId, + node.getTable(), + map(node.getOutputVariables()), + newAssignments.build(), + node.getTableConstraints(), + node.getCurrentConstraint(), + node.getEnforcedConstraint(), + node.getCteMaterializationInfo()); + } + + public JoinNode map(JoinNode node, PlanNode left, PlanNode right, PlanNodeId newNodeId) + { + List mappedCriteria = node.getCriteria().stream() + .map(clause -> new EquiJoinClause(map(clause.getLeft()), map(clause.getRight()))) + .collect(toImmutableList()); + + Map mappedDynamicFilters = node.getDynamicFilters().entrySet().stream() + .collect(toImmutableMap(Entry::getKey, entry -> map(entry.getValue()))); + + return new JoinNode( + node.getSourceLocation(), + newNodeId, + node.getType(), + left, + right, + mappedCriteria, + map(node.getOutputVariables()), + node.getFilter().map(this::map), + node.getLeftHashVariable().map(this::map), + node.getRightHashVariable().map(this::map), + node.getDistributionType(), + mappedDynamicFilters); + } + + public UnionNode map(UnionNode node, List sources, PlanNodeId newNodeId) + { + return new UnionNode( + node.getSourceLocation(), + newNodeId, + node.getStatsEquivalentPlanNode(), + sources, + map(node.getOutputVariables()), + mapSetOperationVariableMapping(node.getVariableMapping())); + } + + public IntersectNode map(IntersectNode node, List sources, PlanNodeId newNodeId) + { + return new IntersectNode( + node.getSourceLocation(), + newNodeId, + node.getStatsEquivalentPlanNode(), + sources, + map(node.getOutputVariables()), + mapSetOperationVariableMapping(node.getVariableMapping())); + } + + public ExceptNode map(ExceptNode node, List sources, PlanNodeId newNodeId) + { + return new ExceptNode( + node.getSourceLocation(), + newNodeId, + node.getStatsEquivalentPlanNode(), + sources, + map(node.getOutputVariables()), + mapSetOperationVariableMapping(node.getVariableMapping())); + } + + private Map> mapSetOperationVariableMapping( + Map> variableMapping) + { + ImmutableMap.Builder> builder = ImmutableMap.builder(); + for (Entry> entry : variableMapping.entrySet()) { + builder.put(map(entry.getKey()), map(entry.getValue())); + } + return builder.build(); + } + private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source) { return new PartitioningScheme(translateVariable(scheme.getPartitioning(), this::map), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index 1c5e2e506f282..30153f473ef0d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -51,6 +51,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -80,7 +81,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.tree.SymbolReference; @@ -593,6 +593,7 @@ public PlanNode visitTopNRowNumber(TopNRowNumberNode node, RewriteContext node.getId(), context.rewrite(node.getSource()), canonicalizeAndDistinct(node.getSpecification()), + node.getRankingFunction(), canonicalize(node.getRowNumberVariable()), node.getMaxRowCountPerPartition(), node.isPartial(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java index 337e9eb39df03..4c07777ea6e04 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/WindowFilterPushDown.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.relation.DomainTranslator.ExtractionResult; import com.facebook.presto.spi.relation.RowExpression; @@ -36,16 +37,18 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import static com.facebook.presto.SystemSessionProperties.isNativeExecutionEnabled; +import static com.facebook.presto.SystemSessionProperties.isOptimizeTopNRank; import static com.facebook.presto.SystemSessionProperties.isOptimizeTopNRowNumber; import static com.facebook.presto.common.predicate.Marker.Bound.BELOW; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -134,6 +137,12 @@ public PlanNode visitWindow(WindowNode node, RewriteContext context) return replaceChildren(node, ImmutableList.of(rewrittenSource)); } + private boolean canReplaceWithTopNRowNumber(WindowNode node) + { + return (canOptimizeRowNumberFunction(node, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) || + (isNativeExecutionEnabled(session) && canOptimizeRankFunction(node, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRank(session)); + } + @Override public PlanNode visitLimit(LimitNode node, RewriteContext context) { @@ -152,16 +161,22 @@ public PlanNode visitLimit(LimitNode node, RewriteContext context) planChanged = true; source = rowNumberNode; } - else if (source instanceof WindowNode && canOptimizeWindowFunction((WindowNode) source, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) { + else if (source instanceof WindowNode) { WindowNode windowNode = (WindowNode) source; - // verify that unordered row_number window functions are replaced by RowNumberNode - verify(windowNode.getOrderingScheme().isPresent()); - TopNRowNumberNode topNRowNumberNode = convertToTopNRowNumber(windowNode, limit); - if (windowNode.getPartitionBy().isEmpty()) { - return topNRowNumberNode; + if (canReplaceWithTopNRowNumber(windowNode)) { + // Unordered row_number window functions are replaced by RowNumberNode and + // only rank/dense_rank with ordering schema are optimized. + verify(windowNode.getOrderingScheme().isPresent()); + + TopNRowNumberNode topNRowNumberNode = convertToTopNRowNumber(windowNode, limit); + planChanged = true; + // Limit can be entirely skipped for row_number without partitioning (not for rank/dense_rank). + if (windowNode.getPartitionBy().isEmpty() && + canOptimizeRowNumberFunction(windowNode, metadata.getFunctionAndTypeManager())) { + return topNRowNumberNode; + } + source = topNRowNumberNode; } - planChanged = true; - source = topNRowNumberNode; } return replaceChildren(node, ImmutableList.of(source)); } @@ -183,15 +198,17 @@ public PlanNode visitFilter(FilterNode node, RewriteContext context) return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt()); } } - else if (source instanceof WindowNode && canOptimizeWindowFunction((WindowNode) source, metadata.getFunctionAndTypeManager()) && isOptimizeTopNRowNumber(session)) { + else if (source instanceof WindowNode) { WindowNode windowNode = (WindowNode) source; - VariableReferenceExpression rowNumberVariable = getOnlyElement(windowNode.getCreatedVariable()); - OptionalInt upperBound = extractUpperBound(tupleDomain, rowNumberVariable); - - if (upperBound.isPresent()) { - source = convertToTopNRowNumber(windowNode, upperBound.getAsInt()); - planChanged = true; - return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt()); + if (canReplaceWithTopNRowNumber(windowNode)) { + VariableReferenceExpression rowNumberVariable = getOnlyElement(windowNode.getCreatedVariable()); + OptionalInt upperBound = extractUpperBound(tupleDomain, rowNumberVariable); + + if (upperBound.isPresent()) { + source = convertToTopNRowNumber(windowNode, upperBound.getAsInt()); + planChanged = true; + return rewriteFilterSource(node, source, rowNumberVariable, upperBound.getAsInt()); + } } } return replaceChildren(node, ImmutableList.of(source)); @@ -275,11 +292,30 @@ private static RowNumberNode mergeLimit(RowNumberNode node, int newRowCountPerPa private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limit) { + String windowFunction = Iterables.getOnlyElement(windowNode.getWindowFunctions().values()).getFunctionCall().getFunctionHandle().getName(); + String[] parts = windowFunction.split("\\."); + String windowFunctionName = parts[parts.length - 1]; + TopNRowNumberNode.RankingFunction rankingFunction; + switch (windowFunctionName) { + case "row_number": + rankingFunction = TopNRowNumberNode.RankingFunction.ROW_NUMBER; + break; + case "rank": + rankingFunction = TopNRowNumberNode.RankingFunction.RANK; + break; + case "dense_rank": + rankingFunction = TopNRowNumberNode.RankingFunction.DENSE_RANK; + break; + default: + throw new IllegalArgumentException("Unsupported window function for TopNRowNumberNode: " + windowFunctionName); + } + return new TopNRowNumberNode( windowNode.getSourceLocation(), idAllocator.getNextId(), windowNode.getSource(), windowNode.getSpecification(), + rankingFunction, getOnlyElement(windowNode.getCreatedVariable()), limit, false, @@ -288,16 +324,29 @@ private TopNRowNumberNode convertToTopNRowNumber(WindowNode windowNode, int limi private static boolean canReplaceWithRowNumber(WindowNode node, FunctionAndTypeManager functionAndTypeManager) { - return canOptimizeWindowFunction(node, functionAndTypeManager) && !node.getOrderingScheme().isPresent(); + return canOptimizeRowNumberFunction(node, functionAndTypeManager) && !node.getOrderingScheme().isPresent(); } - private static boolean canOptimizeWindowFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager) + private static boolean canOptimizeRowNumberFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager) { if (node.getWindowFunctions().size() != 1) { return false; } - VariableReferenceExpression rowNumberVariable = getOnlyElement(node.getWindowFunctions().keySet()); - return isRowNumberMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(node.getWindowFunctions().get(rowNumberVariable).getFunctionHandle())); + return isRowNumberMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(getOnlyElement(node.getWindowFunctions().values()).getFunctionHandle())); + } + + private static boolean canOptimizeRankFunction(WindowNode node, FunctionAndTypeManager functionAndTypeManager) + { + if (node.getWindowFunctions().size() != 1) { + return false; + } + + // This optimization requires an ordering scheme for the rank functions. + if (!node.getOrderingScheme().isPresent()) { + return false; + } + + return isRankMetadata(functionAndTypeManager, functionAndTypeManager.getFunctionMetadata(getOnlyElement(node.getWindowFunctions().values()).getFunctionHandle())); } private static boolean isRowNumberMetadata(FunctionAndTypeManager functionAndTypeManager, FunctionMetadata functionMetadata) @@ -305,5 +354,13 @@ private static boolean isRowNumberMetadata(FunctionAndTypeManager functionAndTyp FunctionHandle rowNumberFunction = functionAndTypeManager.lookupFunction("row_number", ImmutableList.of()); return functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(rowNumberFunction)); } + + private static boolean isRankMetadata(FunctionAndTypeManager functionAndTypeManager, FunctionMetadata functionMetadata) + { + FunctionHandle rankFunction = functionAndTypeManager.lookupFunction("rank", ImmutableList.of()); + FunctionHandle denseRankFunction = functionAndTypeManager.lookupFunction("dense_rank", ImmutableList.of()); + return functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(rankFunction)) || + functionMetadata.equals(functionAndTypeManager.getFunctionMetadata(denseRankFunction)); + } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java index 7653ce684ebc5..311076e818315 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/ExchangeNode.java @@ -21,6 +21,9 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.SystemPartitioningHandle; +import com.facebook.presto.sql.planner.SystemPartitioningHandle.SystemPartitionFunction; +import com.facebook.presto.sql.planner.SystemPartitioningHandle.SystemPartitioning; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; @@ -259,6 +262,16 @@ public static ExchangeNode roundRobinExchange(PlanNodeId id, Scope scope, PlanNo new PartitioningScheme(Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()), child.getOutputVariables())); } + public static ExchangeNode roundRobinExchange(PlanNodeId id, Scope scope, PlanNode child, int partitionCount) + { + checkArgument(partitionCount > 0, "partitionCount must be positive"); + return partitionedExchange( + id, + scope, + child, + new PartitioningScheme(Partitioning.create(SystemPartitioningHandle.createSystemPartitioning(SystemPartitioning.FIXED, SystemPartitionFunction.ROUND_ROBIN, partitionCount), ImmutableList.of()), child.getOutputVariables())); + } + public static ExchangeNode mergingExchange(PlanNodeId id, Scope scope, PlanNode child, OrderingScheme orderingScheme) { PartitioningHandle partitioningHandle = scope.isLocal() ? FIXED_PASSTHROUGH_DISTRIBUTION : SINGLE_DISTRIBUTION; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java index 6bfa05d32a9a8..f2eb1076289a1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java @@ -88,11 +88,6 @@ public R visitRowNumber(RowNumberNode node, C context) return visitPlan(node, context); } - public R visitTopNRowNumber(TopNRowNumberNode node, C context) - { - return visitPlan(node, context); - } - public R visitExchange(ExchangeNode node, C context) { return visitPlan(node, context); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index 1db8a8b817eb5..6db8f5c382853 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; import com.facebook.presto.spi.plan.DeleteNode; +import com.facebook.presto.spi.plan.ExceptNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IndexSourceNode; import com.facebook.presto.spi.plan.IntersectNode; @@ -230,6 +231,11 @@ public static Pattern intersect() return typeOf(IntersectNode.class); } + public static Pattern except() + { + return typeOf(ExceptNode.class); + } + public static Pattern values() { return typeOf(ValuesNode.class); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java index 0ba5e5d4725ed..0e6ca714990d3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/OperatorInputStats.java @@ -20,16 +20,19 @@ public class OperatorInputStats { private final long totalDrivers; private final long inputPositions; + private final long inputDataSizeInBytes; private final double sumSquaredInputPositions; @JsonCreator public OperatorInputStats( @JsonProperty("totalDrivers") long totalDrivers, @JsonProperty("inputPositions") long inputPositions, + @JsonProperty("inputDataSizeInBytes") long inputDataSizeInBytes, @JsonProperty("sumSquaredInputPositions") double sumSquaredInputPositions) { this.totalDrivers = totalDrivers; this.inputPositions = inputPositions; + this.inputDataSizeInBytes = inputDataSizeInBytes; this.sumSquaredInputPositions = sumSquaredInputPositions; } @@ -45,6 +48,12 @@ public long getInputPositions() return inputPositions; } + @JsonProperty + public long getInputDataSizeInBytes() + { + return inputDataSizeInBytes; + } + @JsonProperty public double getSumSquaredInputPositions() { @@ -56,6 +65,7 @@ public static OperatorInputStats merge(OperatorInputStats first, OperatorInputSt return new OperatorInputStats( first.totalDrivers + second.totalDrivers, first.inputPositions + second.inputPositions, + first.inputDataSizeInBytes + second.inputDataSizeInBytes, first.sumSquaredInputPositions + second.sumSquaredInputPositions); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java index 17b8fce3ebf48..7c9e2a852a1bd 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanNodeStatsSummarizer.java @@ -139,6 +139,7 @@ private static List getPlanNodeStats(TaskStats taskStats) new OperatorInputStats( operatorStats.getTotalDrivers(), operatorStats.getInputPositions(), + operatorStats.getInputDataSizeInBytes(), operatorStats.getSumSquaredInputPositions())), (map1, map2) -> mergeMaps(map1, map2, OperatorInputStats::merge)); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index c35f28525bb09..ca00f8ad64d75 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -75,6 +75,7 @@ import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TableWriterNode.CallDistributedProcedureTarget; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -108,7 +109,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode.TableArgumentProperties; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java index 2809b8145831a..d20ab5765bd7a 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/planPrinter/TextRenderer.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.sql.planner.planPrinter; +import com.facebook.airlift.units.DataSize; import com.facebook.presto.cost.PlanCostEstimate; import com.facebook.presto.cost.PlanNodeStatsEstimate; import com.facebook.presto.cost.TableWriterNodeStatsEstimate; @@ -147,6 +148,7 @@ private void printDistributions(StringBuilder output, PlanNodeStats stats) { Map inputAverages = stats.getOperatorInputPositionsAverages(); Map inputStdDevs = stats.getOperatorInputPositionsStdDevs(); + Map inputStats = stats.getOperatorInputStats(); Map hashCollisionsAverages = emptyMap(); Map hashCollisionsStdDevs = emptyMap(); @@ -162,10 +164,15 @@ private void printDistributions(StringBuilder output, PlanNodeStats stats) for (String operator : translatedOperatorTypes.keySet()) { String translatedOperatorType = translatedOperatorTypes.get(operator); double inputAverage = inputAverages.get(operator); + long inputTotalRowCount = inputStats.get(operator).getInputPositions(); + long inputDataSizeInBytes = inputStats.get(operator).getInputDataSizeInBytes(); output.append(translatedOperatorType); - output.append(format(Locale.US, "Input avg.: %s rows, Input std.dev.: %s%%%n", - formatDouble(inputAverage), formatDouble(100.0d * inputStdDevs.get(operator) / inputAverage))); + output.append(format(Locale.US, "Input total: %s (%s), avg.: %s rows, std.dev.: %s%%%n", + formatPositions(inputTotalRowCount), + DataSize.succinctBytes(inputDataSizeInBytes).toString(), + formatDouble(inputAverage), + formatDouble(100.0d * inputStdDevs.get(operator) / inputAverage))); double hashCollisionsAverage = hashCollisionsAverages.getOrDefault(operator, 0.0d); double expectedHashCollisionsAverage = expectedHashCollisionsAverages.getOrDefault(operator, 0.0d); @@ -247,7 +254,7 @@ private String printEstimates(PlanRepresentation plan, NodeRepresentation node) output.append(format(formatStr, stats.getSourceInfo().getClass().getSimpleName(), formatAsLong(stats.getOutputRowCount()), - formatEstimateAsDataSize(stats.getOutputSizeInBytes(plan.getPlanNodeRoot())), + formatEstimateAsDataSize(stats.getOutputSizeInBytes(node)), formatDouble(cost.getCpuCost()), formatDouble(cost.getMaxMemory()), formatDouble(cost.getNetworkCost()), diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java index 29217e5451577..a7755d49e6148 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/CheckUnsupportedPrestissimoTypes.java @@ -41,8 +41,8 @@ import java.util.Objects; import java.util.Optional; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index 1a88f259c882e..bfd5651f37169 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -48,6 +48,7 @@ import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -77,7 +78,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode.PassThroughColumn; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index bdde9065df34b..22a4e5d54473b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -439,6 +439,11 @@ public boolean isMapFilterFunction(FunctionHandle functionHandle) return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("map_filter"))); } + public boolean isCardinalityFunction(FunctionHandle functionHandle) + { + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("cardinality"))); + } + @Override public FunctionHandle lookupBuiltInFunction(String functionName, List inputTypes) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeInputRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeInputRewrite.java index b3ee4b42a543c..5f36afa14d7e4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeInputRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeInputRewrite.java @@ -17,6 +17,8 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; @@ -50,6 +52,7 @@ import static com.facebook.presto.sql.QueryUtil.simpleQuery; import static com.facebook.presto.sql.QueryUtil.values; import static com.facebook.presto.sql.analyzer.utils.ParameterExtractor.getParameters; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForTablesAndColumns; import static com.facebook.presto.util.AnalyzerUtil.createParsingOptions; import static java.util.Objects.requireNonNull; @@ -67,9 +70,10 @@ public Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { - return (Statement) new Visitor(session, parser, metadata, queryExplainer, parameters, parameterLookup, accessControl, warningCollector, query).process(node, null); + return (Statement) new Visitor(session, parser, metadata, queryExplainer, parameters, parameterLookup, accessControl, warningCollector, query, viewDefinitionReferences).process(node, null); } private static final class Visitor @@ -84,6 +88,7 @@ private static final class Visitor private final AccessControl accessControl; private final WarningCollector warningCollector; private final String query; + private final ViewDefinitionReferences viewDefinitionReferences; public Visitor( Session session, @@ -94,7 +99,8 @@ public Visitor( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { this.session = requireNonNull(session, "session is null"); this.parser = parser; @@ -105,6 +111,7 @@ public Visitor( this.parameterLookup = parameterLookup; this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.query = requireNonNull(query, "query is null"); + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } @Override @@ -115,8 +122,10 @@ protected Node visitDescribeInput(DescribeInput node, Void context) Statement statement = parser.createStatement(sqlString, createParsingOptions(session, warningCollector)); // create analysis for the query we are describing. - Analyzer analyzer = new Analyzer(session, metadata, parser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, query); - Analysis analysis = analyzer.analyze(statement, true); + Analyzer analyzer = new Analyzer(session, metadata, parser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, query, viewDefinitionReferences); + Analysis analysis = analyzer.analyzeSemantic(statement, true); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissionsForTablesAndColumns(accessControlReferences); // get all parameters in query List parameters = getParameters(statement); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeOutputRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeOutputRewrite.java index f8fc8c19ef5ec..b8108619ceb03 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeOutputRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/DescribeOutputRewrite.java @@ -18,6 +18,8 @@ import com.facebook.presto.common.type.FixedWidthType; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; @@ -48,6 +50,7 @@ import static com.facebook.presto.sql.QueryUtil.selectList; import static com.facebook.presto.sql.QueryUtil.simpleQuery; import static com.facebook.presto.sql.QueryUtil.values; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForTablesAndColumns; import static com.facebook.presto.util.AnalyzerUtil.createParsingOptions; import static java.util.Objects.requireNonNull; @@ -65,9 +68,10 @@ public Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { - return (Statement) new Visitor(session, parser, metadata, queryExplainer, parameters, parameterLookup, accessControl, warningCollector, query).process(node, null); + return (Statement) new Visitor(session, parser, metadata, queryExplainer, parameters, parameterLookup, accessControl, warningCollector, query, viewDefinitionReferences).process(node, null); } private static final class Visitor @@ -82,6 +86,7 @@ private static final class Visitor private final AccessControl accessControl; private final WarningCollector warningCollector; private final String query; + private final ViewDefinitionReferences viewDefinitionReferences; public Visitor( Session session, @@ -92,7 +97,8 @@ public Visitor( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { this.session = requireNonNull(session, "session is null"); this.parser = parser; @@ -103,6 +109,7 @@ public Visitor( this.accessControl = accessControl; this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.query = requireNonNull(query, "query is null"); + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } @Override @@ -111,8 +118,10 @@ protected Node visitDescribeOutput(DescribeOutput node, Void context) String sqlString = session.getPreparedStatement(node.getName().getValue()); Statement statement = parser.createStatement(sqlString, createParsingOptions(session, warningCollector)); - Analyzer analyzer = new Analyzer(session, metadata, parser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, query); - Analysis analysis = analyzer.analyze(statement, true); + Analyzer analyzer = new Analyzer(session, metadata, parser, accessControl, queryExplainer, parameters, parameterLookup, warningCollector, query, viewDefinitionReferences); + Analysis analysis = analyzer.analyzeSemantic(statement, true); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissionsForTablesAndColumns(accessControlReferences); Optional limit = Optional.empty(); Row[] rows = analysis.getRootScope().getRelationType().getVisibleFields().stream().map(field -> createDescribeOutputRow(field, analysis)).toArray(Row[]::new); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java index 279d41376e999..3dcfb983b21f0 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java @@ -17,6 +17,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.AnalyzerOptions; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.procedure.ProcedureRegistry; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; @@ -62,9 +63,10 @@ public Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { - return (Statement) new Visitor(session, parser, queryExplainer, metadata.getProcedureRegistry(), warningCollector, query) + return (Statement) new Visitor(session, parser, queryExplainer, metadata.getProcedureRegistry(), warningCollector, query, viewDefinitionReferences) .process(node, null); } @@ -76,6 +78,7 @@ private static final class Visitor private final Optional queryExplainer; private final WarningCollector warningCollector; private final String query; + private final ViewDefinitionReferences viewDefinitionReferences; public Visitor( Session session, @@ -83,13 +86,15 @@ public Visitor( Optional queryExplainer, ProcedureRegistry procedureRegistry, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { this.session = requireNonNull(session, "session is null"); this.queryPreparer = new BuiltInQueryPreparer(requireNonNull(parser, "queryPreparer is null"), procedureRegistry); this.queryExplainer = requireNonNull(queryExplainer, "queryExplainer is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.query = requireNonNull(query, "query is null"); + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } @Override @@ -138,13 +143,13 @@ private Node getQueryPlan(Explain node, ExplainType.Type planType, ExplainFormat String plan; switch (planFormat) { case GRAPHVIZ: - plan = queryExplainer.get().getGraphvizPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), warningCollector, query); + plan = queryExplainer.get().getGraphvizPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), warningCollector, query, viewDefinitionReferences); break; case JSON: - plan = queryExplainer.get().getJsonPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), warningCollector, query); + plan = queryExplainer.get().getJsonPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), warningCollector, query, viewDefinitionReferences); break; case TEXT: - plan = queryExplainer.get().getPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), node.isVerbose(), warningCollector, query); + plan = queryExplainer.get().getPlan(session, preparedQuery.getStatement(), planType, preparedQuery.getParameters(), node.isVerbose(), warningCollector, query, viewDefinitionReferences); break; default: throw new IllegalArgumentException("Invalid Explain Format: " + planFormat.toString()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java index 9e72b2653447c..9221821205f04 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java @@ -17,6 +17,7 @@ import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.MaterializedViewQueryOptimizer; import com.facebook.presto.sql.analyzer.QueryExplainer; @@ -52,7 +53,8 @@ public Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { return (Statement) new MaterializedViewOptimizationRewrite .Visitor(metadata, session, parser, accessControl) diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java index e327610c98785..e0b5214a689e7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.MetadataResolver; import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.constraints.NotNullConstraint; import com.facebook.presto.spi.constraints.PrimaryKeyConstraint; import com.facebook.presto.spi.constraints.UniqueConstraint; @@ -187,7 +188,8 @@ public Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { return (Statement) new Visitor(metadata, parser, session, parameters, accessControl, queryExplainer, warningCollector).process(node, null); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java index 705c780256bb2..fbd848eff2376 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/ShowStatsRewrite.java @@ -33,6 +33,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.security.AccessControl; @@ -116,9 +117,10 @@ public Statement rewrite(Session session, Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { - return (Statement) new Visitor(metadata, session, parameters, queryExplainer, warningCollector, query).process(node, null); + return (Statement) new Visitor(metadata, session, parameters, queryExplainer, warningCollector, query, viewDefinitionReferences).process(node, null); } private static class Visitor @@ -130,8 +132,9 @@ private static class Visitor private final Optional queryExplainer; private final WarningCollector warningCollector; private final String sqlString; + private final ViewDefinitionReferences viewDefinitionReferences; - public Visitor(Metadata metadata, Session session, List parameters, Optional queryExplainer, WarningCollector warningCollector, String sqlString) + public Visitor(Metadata metadata, Session session, List parameters, Optional queryExplainer, WarningCollector warningCollector, String sqlString, ViewDefinitionReferences viewDefinitionReferences) { this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); @@ -139,6 +142,7 @@ public Visitor(Metadata metadata, Session session, List parameters, this.queryExplainer = requireNonNull(queryExplainer, "queryExplainer is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.sqlString = requireNonNull(sqlString, "sqlString is null"); + this.viewDefinitionReferences = requireNonNull(viewDefinitionReferences, "viewDefinitionReferences is null"); } @Override @@ -149,7 +153,7 @@ protected Node visitShowStats(ShowStats node, Void context) if (node.getRelation() instanceof TableSubquery) { Query query = ((TableSubquery) node.getRelation()).getQuery(); QuerySpecification specification = (QuerySpecification) query.getQueryBody(); - Plan plan = queryExplainer.get().getLogicalPlan(session, new Query(Optional.empty(), specification, Optional.empty(), Optional.empty(), Optional.empty()), parameters, warningCollector, sqlString); + Plan plan = queryExplainer.get().getLogicalPlan(session, new Query(Optional.empty(), specification, Optional.empty(), Optional.empty(), Optional.empty()), parameters, warningCollector, sqlString, viewDefinitionReferences); Set columns = validateShowStatsSubquery(node, query, specification, plan); Table table = (Table) specification.getFrom().get(); Constraint constraint = getConstraint(plan); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java index 1e3228f42fd17..f65ff4af88243 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.QueryExplainer; import com.facebook.presto.sql.parser.SqlParser; @@ -53,10 +54,11 @@ public static Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query) + String query, + ViewDefinitionReferences viewDefinitionReferences) { for (Rewrite rewrite : REWRITES) { - node = requireNonNull(rewrite.rewrite(session, metadata, parser, queryExplainer, node, parameters, parameterLookup, accessControl, warningCollector, query), "Statement rewrite returned null"); + node = requireNonNull(rewrite.rewrite(session, metadata, parser, queryExplainer, node, parameters, parameterLookup, accessControl, warningCollector, query, viewDefinitionReferences), "Statement rewrite returned null"); } return node; } @@ -73,6 +75,7 @@ Statement rewrite( Map, Expression> parameterLookup, AccessControl accessControl, WarningCollector warningCollector, - String query); + String query, + ViewDefinitionReferences viewDefinitionReferences); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 5a5c443e9baef..3a745229b7521 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -948,7 +948,7 @@ private MaterializedResultWithPlan executeExplainTypeValidate(String sql, Sessio AnalyzerContext analyzerContext = getAnalyzerContext(queryAnalyzer, metadata.getMetadataResolver(session), idAllocator, new VariableAllocator(), session, sql); QueryAnalysis queryAnalysis = queryAnalyzer.analyze(analyzerContext, preparedQuery); - checkAccessPermissions(queryAnalysis.getAccessControlReferences(), sql, session.getPreparedStatements()); + checkAccessPermissions(queryAnalysis.getAccessControlReferences(), queryAnalysis.getViewDefinitionReferences(), sql, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); MaterializedResult result = MaterializedResult.resultBuilder(session, BooleanType.BOOLEAN) .row(true) @@ -1215,7 +1215,7 @@ public Plan createPlan(Session session, @Language("SQL") String sql, List coerceTypeBase(Type sourceType, String resultTypeBase) case StandardTypes.JSON: case StandardTypes.INTERVAL_YEAR_TO_MONTH: case StandardTypes.INTERVAL_DAY_TO_SECOND: - case KHyperLogLogType.NAME: + case StandardTypes.K_HYPER_LOG_LOG: case JoniRegexpType.NAME: case LikePatternType.NAME: case JsonPathType.NAME: diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogAggregationFunction.java index 666e1d738d7be..378586a9ee5a6 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogAggregationFunction.java @@ -15,7 +15,6 @@ package com.facebook.presto.type.khyperloglog; import com.facebook.presto.common.block.BlockBuilder; -import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.spi.function.AggregationFunction; import com.facebook.presto.spi.function.AggregationState; import com.facebook.presto.spi.function.CombineFunction; @@ -26,6 +25,10 @@ import io.airlift.slice.Slice; import io.airlift.slice.XxHash64; +import static com.facebook.presto.common.type.StandardTypes.BIGINT; +import static com.facebook.presto.common.type.StandardTypes.DOUBLE; +import static com.facebook.presto.common.type.StandardTypes.K_HYPER_LOG_LOG; + @AggregationFunction("khyperloglog_agg") public final class KHyperLogLogAggregationFunction { @@ -34,7 +37,7 @@ public final class KHyperLogLogAggregationFunction private KHyperLogLogAggregationFunction() {} @InputFunction - public static void input(@AggregationState KHyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType(StandardTypes.BIGINT) long uii) + public static void input(@AggregationState KHyperLogLogState state, @SqlType(BIGINT) long value, @SqlType(BIGINT) long uii) { if (state.getKHLL() == null) { state.setKHLL(new KHyperLogLog()); @@ -44,7 +47,7 @@ public static void input(@AggregationState KHyperLogLogState state, @SqlType(Sta @InputFunction @LiteralParameters("x") - public static void input(@AggregationState KHyperLogLogState state, @SqlType("varchar(x)") Slice value, @SqlType(StandardTypes.BIGINT) long uii) + public static void input(@AggregationState KHyperLogLogState state, @SqlType("varchar(x)") Slice value, @SqlType(BIGINT) long uii) { if (state.getKHLL() == null) { state.setKHLL(new KHyperLogLog()); @@ -53,14 +56,14 @@ public static void input(@AggregationState KHyperLogLogState state, @SqlType("va } @InputFunction - public static void input(@AggregationState KHyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value, @SqlType(StandardTypes.BIGINT) long uii) + public static void input(@AggregationState KHyperLogLogState state, @SqlType(DOUBLE) double value, @SqlType(BIGINT) long uii) { input(state, Double.doubleToLongBits(value), uii); } @InputFunction @LiteralParameters("x") - public static void input(@AggregationState KHyperLogLogState state, @SqlType(StandardTypes.BIGINT) long value, @SqlType("varchar(x)") Slice uii) + public static void input(@AggregationState KHyperLogLogState state, @SqlType(BIGINT) long value, @SqlType("varchar(x)") Slice uii) { input(state, value, XxHash64.hash(uii)); } @@ -74,7 +77,7 @@ public static void input(@AggregationState KHyperLogLogState state, @SqlType("va @InputFunction @LiteralParameters("x") - public static void input(@AggregationState KHyperLogLogState state, @SqlType(StandardTypes.DOUBLE) double value, @SqlType("varchar(x)") Slice uii) + public static void input(@AggregationState KHyperLogLogState state, @SqlType(DOUBLE) double value, @SqlType("varchar(x)") Slice uii) { input(state, Double.doubleToLongBits(value), XxHash64.hash(uii)); } @@ -92,7 +95,7 @@ public static void combine(@AggregationState KHyperLogLogState state, @Aggregati } } - @OutputFunction(KHyperLogLogType.NAME) + @OutputFunction(K_HYPER_LOG_LOG) public static void output(@AggregationState KHyperLogLogState state, BlockBuilder out) { SERIALIZER.serialize(state, out); diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogFunctions.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogFunctions.java index 5d907a6a74ca5..4e3d0dbdc7b17 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogFunctions.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogFunctions.java @@ -28,6 +28,7 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.StandardTypes.K_HYPER_LOG_LOG; public final class KHyperLogLogFunctions { @@ -37,14 +38,14 @@ private KHyperLogLogFunctions() @ScalarFunction @SqlType(StandardTypes.BIGINT) - public static long cardinality(@SqlType(KHyperLogLogType.NAME) Slice khll) + public static long cardinality(@SqlType(K_HYPER_LOG_LOG) Slice khll) { return KHyperLogLog.newInstance(khll).cardinality(); } @ScalarFunction @SqlType(StandardTypes.BIGINT) - public static long intersectionCardinality(@SqlType(KHyperLogLogType.NAME) Slice slice1, @SqlType(KHyperLogLogType.NAME) Slice slice2) + public static long intersectionCardinality(@SqlType(K_HYPER_LOG_LOG) Slice slice1, @SqlType(K_HYPER_LOG_LOG) Slice slice2) { KHyperLogLog khll1 = KHyperLogLog.newInstance(slice1); KHyperLogLog khll2 = KHyperLogLog.newInstance(slice2); @@ -67,7 +68,7 @@ public static long intersectionCardinality(@SqlType(KHyperLogLogType.NAME) Slice @ScalarFunction @SqlType(StandardTypes.DOUBLE) - public static double jaccardIndex(@SqlType(KHyperLogLogType.NAME) Slice slice1, @SqlType(KHyperLogLogType.NAME) Slice slice2) + public static double jaccardIndex(@SqlType(K_HYPER_LOG_LOG) Slice slice1, @SqlType(K_HYPER_LOG_LOG) Slice slice2) { KHyperLogLog khll1 = KHyperLogLog.newInstance(slice1); KHyperLogLog khll2 = KHyperLogLog.newInstance(slice2); @@ -77,7 +78,7 @@ public static double jaccardIndex(@SqlType(KHyperLogLogType.NAME) Slice slice1, @ScalarFunction @SqlType("map(bigint,double)") - public static Block uniquenessDistribution(@TypeParameter("map") Type mapType, @SqlType(KHyperLogLogType.NAME) Slice slice) + public static Block uniquenessDistribution(@TypeParameter("map") Type mapType, @SqlType(K_HYPER_LOG_LOG) Slice slice) { KHyperLogLog khll = KHyperLogLog.newInstance(slice); return uniquenessDistribution(mapType, slice, khll.getMinhashSize()); @@ -85,7 +86,7 @@ public static Block uniquenessDistribution(@TypeParameter("map") @ScalarFunction @SqlType("map(bigint,double)") - public static Block uniquenessDistribution(@TypeParameter("map") Type mapType, @SqlType(KHyperLogLogType.NAME) Slice slice, @SqlType(StandardTypes.BIGINT) long histogramSize) + public static Block uniquenessDistribution(@TypeParameter("map") Type mapType, @SqlType(K_HYPER_LOG_LOG) Slice slice, @SqlType(StandardTypes.BIGINT) long histogramSize) { KHyperLogLog khll = KHyperLogLog.newInstance(slice); @@ -102,15 +103,15 @@ public static Block uniquenessDistribution(@TypeParameter("map") @ScalarFunction @SqlType(StandardTypes.DOUBLE) - public static double reidentificationPotential(@SqlType(KHyperLogLogType.NAME) Slice khll, @SqlType(StandardTypes.BIGINT) long threshold) + public static double reidentificationPotential(@SqlType(K_HYPER_LOG_LOG) Slice khll, @SqlType(StandardTypes.BIGINT) long threshold) { return KHyperLogLog.newInstance(khll).reidentificationPotential(threshold); } @ScalarFunction - @SqlType(KHyperLogLogType.NAME) + @SqlType(K_HYPER_LOG_LOG) @SqlNullable - public static Slice mergeKhll(@SqlType("array(KHyperLogLog)") Block block) + public static Slice mergeKhll(@SqlType("array(" + K_HYPER_LOG_LOG + ")") Block block) { if (block.getPositionCount() == 0) { return null; diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogOperators.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogOperators.java index fd647d5f61f56..335da17837c2d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogOperators.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/KHyperLogLogOperators.java @@ -14,12 +14,13 @@ package com.facebook.presto.type.khyperloglog; -import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.spi.function.ScalarOperator; import com.facebook.presto.spi.function.SqlType; import io.airlift.slice.Slice; import static com.facebook.presto.common.function.OperatorType.CAST; +import static com.facebook.presto.common.type.StandardTypes.K_HYPER_LOG_LOG; +import static com.facebook.presto.common.type.StandardTypes.VARBINARY; public final class KHyperLogLogOperators { @@ -28,15 +29,15 @@ private KHyperLogLogOperators() } @ScalarOperator(CAST) - @SqlType(StandardTypes.VARBINARY) - public static Slice castToBinary(@SqlType(KHyperLogLogType.NAME) Slice slice) + @SqlType(VARBINARY) + public static Slice castToBinary(@SqlType(K_HYPER_LOG_LOG) Slice slice) { return slice; } @ScalarOperator(CAST) - @SqlType(KHyperLogLogType.NAME) - public static Slice castFromBinary(@SqlType(StandardTypes.VARBINARY) Slice slice) + @SqlType(K_HYPER_LOG_LOG) + public static Slice castFromBinary(@SqlType(VARBINARY) Slice slice) { return slice; } diff --git a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogAggregationFunction.java index c45e0bb0918b1..9c1211c73191a 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/type/khyperloglog/MergeKHyperLogLogAggregationFunction.java @@ -15,6 +15,7 @@ package com.facebook.presto.type.khyperloglog; import com.facebook.presto.common.block.BlockBuilder; +import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.spi.function.AggregationFunction; import com.facebook.presto.spi.function.AggregationState; import com.facebook.presto.spi.function.CombineFunction; @@ -31,7 +32,7 @@ public final class MergeKHyperLogLogAggregationFunction private MergeKHyperLogLogAggregationFunction() {} @InputFunction - public static void input(@AggregationState KHyperLogLogState state, @SqlType(KHyperLogLogType.NAME) Slice value) + public static void input(@AggregationState KHyperLogLogState state, @SqlType(StandardTypes.K_HYPER_LOG_LOG) Slice value) { KHyperLogLog instance = KHyperLogLog.newInstance(value); merge(state, instance); @@ -53,7 +54,7 @@ private static void merge(@AggregationState KHyperLogLogState state, KHyperLogLo } } - @OutputFunction(KHyperLogLogType.NAME) + @OutputFunction(StandardTypes.K_HYPER_LOG_LOG) public static void output(@AggregationState KHyperLogLogState state, BlockBuilder out) { if (state.getKHLL() == null) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java b/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java index a91c39abb22f1..8c795658e8360 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java +++ b/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java @@ -16,7 +16,6 @@ import com.facebook.presto.Session; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.transaction.TransactionId; -import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; @@ -26,7 +25,7 @@ import com.facebook.presto.spi.analyzer.AnalyzerOptions; import com.facebook.presto.spi.analyzer.MetadataResolver; import com.facebook.presto.spi.analyzer.QueryAnalyzer; -import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.spi.security.AccessControlContext; @@ -101,32 +100,22 @@ public static AnalyzerContext getAnalyzerContext( return new AnalyzerContext(metadataResolver, idAllocator, variableAllocator, query); } - public static void checkAccessPermissions(AccessControlReferences accessControlReferences, String query, Map preparedStatements) + public static void checkAccessPermissions(AccessControlReferences accessControlReferences, ViewDefinitionReferences viewDefinitionReferences, String query, Map preparedStatements, Identity identity, AccessControl accessControl, AccessControlContext accessControlContext) { // Query check - checkQueryIntegrity(accessControlReferences, query, preparedStatements); - // Table checks - checkAccessPermissionsForTable(accessControlReferences); - // Table Column checks - checkAccessPermissionsForColumns(accessControlReferences); + checkQueryIntegrity(identity, accessControl, accessControlContext, query, preparedStatements, viewDefinitionReferences); + + //Table and column checks + checkAccessPermissionsForTablesAndColumns(accessControlReferences); } - public static void checkQueryIntegrity(AccessControlReferences accessControlReferences, String query, Map preparedStatements) + public static void checkAccessPermissionsForTablesAndColumns(AccessControlReferences accessControlReferences) { - AccessControlInfo queryAccessControlInfo = accessControlReferences.getQueryAccessControlInfo(); - // Only check access if query gets analyzed - if (queryAccessControlInfo != null) { - AccessControl queryAccessControl = queryAccessControlInfo.getAccessControl(); - Identity identity = queryAccessControlInfo.getIdentity(); - AccessControlContext queryAccessControlContext = queryAccessControlInfo.getAccessControlContext(); - Map viewDefinitionMap = accessControlReferences.getViewDefinitions(); - Map materializedViewDefinitionMap = accessControlReferences.getMaterializedViewDefinitions(); - - queryAccessControl.checkQueryIntegrity(identity, queryAccessControlContext, query, preparedStatements, viewDefinitionMap, materializedViewDefinitionMap); - } + checkAccessPermissionsForTable(accessControlReferences); + checkAccessPermissionsForColumns(accessControlReferences); } - public static void checkAccessPermissionsForColumns(AccessControlReferences accessControlReferences) + private static void checkAccessPermissionsForColumns(AccessControlReferences accessControlReferences) { accessControlReferences.getTableColumnAndSubfieldReferencesForAccessControl() .forEach((accessControlInfo, tableColumnReferences) -> @@ -142,7 +131,7 @@ public static void checkAccessPermissionsForColumns(AccessControlReferences acce })); } - public static void checkAccessPermissionsForTable(AccessControlReferences accessControlReferences) + private static void checkAccessPermissionsForTable(AccessControlReferences accessControlReferences) { accessControlReferences.getTableReferences().forEach((accessControlRole, accessControlInfoForTables) -> accessControlInfoForTables.forEach(accessControlInfoForTable -> { AccessControlInfo accessControlInfo = accessControlInfoForTable.getAccessControlInfo(); @@ -168,4 +157,9 @@ public static void checkAccessPermissionsForTable(AccessControlReferences access } })); } + + private static void checkQueryIntegrity(Identity identity, AccessControl accessControl, AccessControlContext accessControlContext, String query, Map preparedStatements, ViewDefinitionReferences viewDefinitionReferences) + { + accessControl.checkQueryIntegrity(identity, accessControlContext, query, preparedStatements, viewDefinitionReferences.getViewDefinitions(), viewDefinitionReferences.getMaterializedViewDefinitions()); + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main-base/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index 5368f400f4646..e227f7ec2d578 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main-base/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -45,6 +45,7 @@ import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TableWriterNode.CallDistributedProcedureTarget; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.UnnestNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -73,7 +74,6 @@ import com.facebook.presto.sql.planner.plan.TableFunctionNode; import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; import com.facebook.presto.sql.tree.ComparisonExpression; @@ -448,9 +448,11 @@ public Void visitTopNRowNumber(TopNRowNumberNode node, Void context) { printNode(node, "TopNRowNumber", - format("partition by = %s|order by = %s|n = %s", + format("function = %s|partition by = %s|order by = %s|n = %s", + node.getRankingFunction(), Joiner.on(", ").join(node.getPartitionBy()), - Joiner.on(", ").join(node.getOrderingScheme().getOrderByVariables()), node.getMaxRowCountPerPartition()), + Joiner.on(", ").join(node.getOrderingScheme().getOrderByVariables()), + node.getMaxRowCountPerPartition()), NODE_COLORS.get(NodeType.WINDOW)); return node.getSource().accept(this, context); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/util/PrestoDataDefBindingHelper.java b/presto-main-base/src/main/java/com/facebook/presto/util/PrestoDataDefBindingHelper.java index 5091a7e3b285a..efcdc6e9d16ed 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/util/PrestoDataDefBindingHelper.java +++ b/presto-main-base/src/main/java/com/facebook/presto/util/PrestoDataDefBindingHelper.java @@ -19,11 +19,13 @@ import com.facebook.presto.execution.AlterFunctionTask; import com.facebook.presto.execution.CallTask; import com.facebook.presto.execution.CommitTask; +import com.facebook.presto.execution.CreateBranchTask; import com.facebook.presto.execution.CreateFunctionTask; import com.facebook.presto.execution.CreateMaterializedViewTask; import com.facebook.presto.execution.CreateRoleTask; import com.facebook.presto.execution.CreateSchemaTask; import com.facebook.presto.execution.CreateTableTask; +import com.facebook.presto.execution.CreateTagTask; import com.facebook.presto.execution.CreateTypeTask; import com.facebook.presto.execution.CreateViewTask; import com.facebook.presto.execution.DataDefinitionTask; @@ -61,11 +63,13 @@ import com.facebook.presto.sql.tree.AlterFunction; import com.facebook.presto.sql.tree.Call; import com.facebook.presto.sql.tree.Commit; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; +import com.facebook.presto.sql.tree.CreateTag; import com.facebook.presto.sql.tree.CreateType; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; @@ -128,6 +132,8 @@ private PrestoDataDefBindingHelper() {} dataDefBuilder.put(CreateTable.class, CreateTableTask.class); dataDefBuilder.put(RenameTable.class, RenameTableTask.class); dataDefBuilder.put(RenameColumn.class, RenameColumnTask.class); + dataDefBuilder.put(CreateBranch.class, CreateBranchTask.class); + dataDefBuilder.put(CreateTag.class, CreateTagTask.class); dataDefBuilder.put(DropBranch.class, DropBranchTask.class); dataDefBuilder.put(DropTag.class, DropTagTask.class); dataDefBuilder.put(DropColumn.class, DropColumnTask.class); diff --git a/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java b/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java index aef0e39f5629d..4efe15d47bdf4 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java +++ b/presto-main-base/src/test/java/com/facebook/presto/cost/AbstractTestFilterStatsCalculator.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.Optional; @@ -37,6 +38,7 @@ import static java.lang.Double.POSITIVE_INFINITY; import static java.lang.String.format; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; public abstract class AbstractTestFilterStatsCalculator { @@ -63,6 +65,36 @@ public AbstractTestFilterStatsCalculator(boolean withHistograms) .build(); } + /** + * Returns expressions on a variable with NDV 1 and the expected row count after applying the filter + * Row count for the input plan node is assumed to be 100 + * @return + */ + @DataProvider + public static Object[][] ndv1Expressions() + { + return new Object[][] { + {"name <> 'bar'", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + {"name <> 'name' AND name <> 'bar'", 81D}, // 100 * UNKNOWN_FILTER_COEFFICIENT * UNKNOWN_FILTER_COEFFICIENT + {"name <> 'foo' OR name is NULL", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + {"name is NULL OR name <> 'foo'", 90D}, // 100 * UNKNOWN_FILTER_COEFFICIENT + }; + } + + @DataProvider + public static Object[][] inList() + { + return new Object[][] { + {"'one'"}, + {"'one','two'"}, + {"'one','two','three'"}, + {"'one','two','three','four'"}, + {"'one','two','three','four','five'"}, + {"'one','two','three','four','five','six'"}, + {"'one','two','three','four','five','six', 'seven'"} + }; + } + @BeforeClass public void setUp() throws Exception @@ -500,6 +532,50 @@ public void testSymbolEqualsSameSymbolFilter() .build()); } + @Test(dataProvider = "inList") + public void testInPredicateWithoutNDV(String inList) + { + Expression exp = expression("status IN (" + inList + ")"); + TypeProvider customTypes = TypeProvider.fromVariables(ImmutableList.builder() + .add(new VariableReferenceExpression(Optional.empty(), "status", MEDIUM_VARCHAR_TYPE)) + .build()); + + RowExpression rowExpression = translator.translateAndOptimize(exp, customTypes); + + VariableStatsEstimate nameStats = VariableStatsEstimate.builder() + // Nulls fraction is known, but NDV is not. Stats propagation should work + .setNullsFraction(0.0D) + .build(); + + PlanNodeStatsEstimate inputStats = PlanNodeStatsEstimate.builder() + .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "status", MEDIUM_VARCHAR_TYPE), nameStats) + .setOutputRowCount(100D) + .build(); + + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(inputStats, rowExpression, session); + + // The IN filter should always apply a filter factor between (0,1) (never NaN/0/1) + int inListLength = inList.split(",").length; + if (inListLength == 1) { + // A single entry IN list is equivalent to an infinite range intersect; we use StatisticRange#INFINITE_TO_INFINITE_RANGE_INTERSECT_OVERLAP_HEURISTIC_FACTOR (0.5) + // as our filter factor, resulting in : non-null-inputRowCount * 0.5 = 50 + assertEquals(rowExpressionStatsEstimate.getOutputRowCount(), 50D); + } + else { + // Multiple values in the IN list - We sum up the estimates, but cap it to non-null-inputRowCount * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT = 80 in this case + assertEquals(rowExpressionStatsEstimate.getOutputRowCount(), 80D); + } + } + + @Test(dataProvider = "inList") + public void testNotInPredicateEstimateIsNeverZero(String inList) + { + RowExpression rowExpression = translator.translateAndOptimize(expression("mediumVarchar NOT IN (" + inList + ")"), standardTypes); + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(standardInputStatistics, rowExpression, session); + + assertNotEquals(rowExpressionStatsEstimate.getOutputRowCount(), 0D, 0.0001D); + } + @Test public void testInPredicateFilter() { @@ -588,7 +664,8 @@ public void testInPredicateFilter() // More values in range than distinct values assertExpression("z IN (DOUBLE '-1', 3.14e0, 0e0, 1e0, 2e0, 3e0, 4e0, 5e0, 6e0, 7e0, 8e0, DOUBLE '-2')") - .outputRowsCount(900.0) + // Range estimate is never the full-range, it's non-null count * CEIL_IN_PREDICATE_UPPER_BOUND_COEFFICIENT + .outputRowsCount(720.0) .variableStats(new VariableReferenceExpression(Optional.empty(), "z", DOUBLE), variableStats -> variableStats.distinctValuesCount(5.0) .lowValue(-2.0) @@ -605,6 +682,34 @@ public void testInPredicateFilter() .nullsFraction(0.0)); } + @Test(dataProvider = "ndv1Expressions") + public void testNotEqualsOnVariablesWithNDV1(String expressionStr, double expectedOutputRowsCount) + { + Expression exp = expression(expressionStr); + + VariableReferenceExpression name = new VariableReferenceExpression(Optional.empty(), "name", MEDIUM_VARCHAR_TYPE); + TypeProvider customTypes = TypeProvider.fromVariables(ImmutableList.builder() + .add(name) + .build()); + + RowExpression rowExpression = translator.translateAndOptimize(exp, customTypes); + + VariableStatsEstimate nameStats = VariableStatsEstimate.builder() + .setNullsFraction(0D) + .setDistinctValuesCount(1D) + .build(); + + PlanNodeStatsEstimate rowExpressionStatsEstimate = statsCalculator.filterStats(PlanNodeStatsEstimate.builder() + .addVariableStatistics(name, nameStats) + .setOutputRowCount(100D) + .build(), rowExpression, session); + + PlanNodeStatsAssertion.assertThat(rowExpressionStatsEstimate) + .outputRowsCount(expectedOutputRowsCount) + // Variable Stats remains unchanged + .variableStats(name, variableStats -> variableStats.distinctValuesCount(1D).nullsFraction(0D)); + } + protected PlanNodeStatsAssertion assertExpression(String expression) { return assertExpression(expression(expression)); diff --git a/presto-main-base/src/test/java/com/facebook/presto/cost/TestAggregationStatsRule.java b/presto-main-base/src/test/java/com/facebook/presto/cost/TestAggregationStatsRule.java index 2656d9777a549..128d271a993e6 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/cost/TestAggregationStatsRule.java +++ b/presto-main-base/src/test/java/com/facebook/presto/cost/TestAggregationStatsRule.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.cost; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import org.testng.annotations.Test; @@ -20,21 +21,26 @@ import java.util.function.Consumer; import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.statistics.SourceInfo.ConfidenceLevel.FACT; public class TestAggregationStatsRule extends BaseStatsCalculatorTest { + private static final VariableReferenceExpression VARIABLE_X = new VariableReferenceExpression(Optional.empty(), "x", BIGINT); + private static final VariableReferenceExpression VARIABLE_Y = new VariableReferenceExpression(Optional.empty(), "y", BIGINT); + private static final VariableReferenceExpression VARIABLE_Z = new VariableReferenceExpression(Optional.empty(), "z", BIGINT); + @Test public void testAggregationWhenAllStatisticsAreKnown() { Consumer outputRowCountAndZStatsAreCalculated = check -> check .outputRowsCount(15) - .variableStats(new VariableReferenceExpression(Optional.empty(), "z", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .variableStats(VARIABLE_Z, symbolStatsAssertion -> symbolStatsAssertion .lowValue(10) .highValue(15) .distinctValuesCount(4) .nullsFraction(0.2)) - .variableStats(new VariableReferenceExpression(Optional.empty(), "y", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion .lowValue(0) .highValue(3) .distinctValuesCount(3) @@ -59,11 +65,11 @@ public void testAggregationWhenAllStatisticsAreKnown() Consumer outputRowsCountAndZStatsAreNotFullyCalculated = check -> check .outputRowsCountUnknown() - .variableStats(new VariableReferenceExpression(Optional.empty(), "z", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .variableStats(VARIABLE_Z, symbolStatsAssertion -> symbolStatsAssertion .unknownRange() .distinctValuesCountUnknown() .nullsFractionUnknown()) - .variableStats(new VariableReferenceExpression(Optional.empty(), "y", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion .unknownRange() .nullsFractionUnknown() .distinctValuesCountUnknown()); @@ -96,19 +102,19 @@ private StatsCalculatorAssertion testAggregation(VariableStatsEstimate zStats) .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT), pb.variable("z", BIGINT))))) .withSourceStats(PlanNodeStatsEstimate.builder() .setOutputRowCount(100) - .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "x", BIGINT), VariableStatsEstimate.builder() + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() .setLowValue(1) .setHighValue(10) .setDistinctValuesCount(5) .setNullsFraction(0.3) .build()) - .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "y", BIGINT), VariableStatsEstimate.builder() + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() .setLowValue(0) .setHighValue(3) .setDistinctValuesCount(3) .setNullsFraction(0) .build()) - .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "z", BIGINT), zStats) + .addVariableStatistics(VARIABLE_Z, zStats) .build()) .check(check -> check .variableStats(new VariableReferenceExpression(Optional.empty(), "sum", BIGINT), symbolStatsAssertion -> symbolStatsAssertion @@ -126,7 +132,7 @@ private StatsCalculatorAssertion testAggregation(VariableStatsEstimate zStats) .highValueUnknown() .distinctValuesCountUnknown() .nullsFractionUnknown()) - .variableStats(new VariableReferenceExpression(Optional.empty(), "x", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .variableStats(VARIABLE_X, symbolStatsAssertion -> symbolStatsAssertion .lowValueUnknown() .highValueUnknown() .distinctValuesCountUnknown() @@ -144,9 +150,459 @@ public void testAggregationStatsCappedToInputRows() .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT), pb.variable("z", BIGINT))))) .withSourceStats(PlanNodeStatsEstimate.builder() .setOutputRowCount(100) - .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "y", BIGINT), VariableStatsEstimate.builder().setDistinctValuesCount(50).build()) - .addVariableStatistics(new VariableReferenceExpression(Optional.empty(), "z", BIGINT), VariableStatsEstimate.builder().setDistinctValuesCount(50).build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder().setDistinctValuesCount(50).build()) + .addVariableStatistics(VARIABLE_Z, VariableStatsEstimate.builder().setDistinctValuesCount(50).build()) .build()) .check(check -> check.outputRowsCount(100)); } + + /** + * Verifies that a global aggregation (no grouping keys) always produces + * exactly one output row with FACT confidence level, regardless of the + * input statistics. + */ + @Test + public void testGlobalAggregationReturnsOneRow() + { + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("sum", BIGINT), pb.rowExpression("sum(x)")) + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .globalGrouping() + .source(pb.values(pb.variable("x", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(1000) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(100) + .setDistinctValuesCount(50) + .setNullsFraction(0.1) + .build()) + .build()) + .check(check -> check + .outputRowsCount(1) + .confident(FACT) + .variableStats(new VariableReferenceExpression(Optional.empty(), "sum", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .lowValueUnknown() + .highValueUnknown() + .distinctValuesCountUnknown() + .nullsFractionUnknown()) + .variableStats(new VariableReferenceExpression(Optional.empty(), "count", BIGINT), symbolStatsAssertion -> symbolStatsAssertion + .lowValueUnknown() + .highValueUnknown() + .distinctValuesCountUnknown() + .nullsFractionUnknown())); + } + + /** + * Verifies that a global aggregation with zero input rows still produces + * exactly one output row with FACT confidence. This is the expected behavior + * for queries like {@code SELECT count(*) FROM empty_table}. + */ + @Test + public void testGlobalAggregationWithZeroInputRows() + { + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .globalGrouping() + .source(pb.values(pb.variable("x", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(0) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setDistinctValuesCount(0) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(1) + .confident(FACT)); + } + + /** + * Verifies that a PARTIAL aggregation step does not reduce the estimated + * row count. The rule pessimistically assumes no reduction for partial + * aggregations and forwards the source row count directly. + */ + @Test + public void testPartialAggregationPreservesSourceRowCount() + { + double sourceRowCount = 500; + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("sum", BIGINT), pb.rowExpression("sum(x)")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .step(AggregationNode.Step.PARTIAL) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(sourceRowCount) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(5) + .setNullsFraction(0.1) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(3) + .setDistinctValuesCount(3) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(sourceRowCount) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(3) + .distinctValuesCount(3) + .nullsFraction(0))); + } + + /** + * Verifies that an INTERMEDIATE aggregation step behaves identically to a + * PARTIAL step: no reduction in estimated row count, source stats forwarded. + */ + @Test + public void testIntermediateAggregationPreservesSourceRowCount() + { + double sourceRowCount = 500; + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("sum", BIGINT), pb.rowExpression("sum(x)")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .step(AggregationNode.Step.INTERMEDIATE) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(sourceRowCount) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(5) + .setNullsFraction(0.1) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(3) + .setDistinctValuesCount(3) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(sourceRowCount) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(3) + .distinctValuesCount(3) + .nullsFraction(0))); + } + + /** + * Verifies that for a SINGLE-step aggregation with a single grouping key, + * the output row count equals the distinct value count of the grouping key. + * Also verifies that the grouping key's nulls fraction is set to zero when + * the source has no nulls in that column. + */ + @Test + public void testSingleGroupingKeyNoNulls() + { + // y has 10 distinct values with no nulls, source has 200 rows + // Expected output: 10 rows (= NDV of y) + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(200) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(50) + .setDistinctValuesCount(50) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(9) + .setDistinctValuesCount(10) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(10) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(9) + .distinctValuesCount(10) + .nullsFraction(0))); + } + + /** + * Verifies that when a grouping key has a non-zero nulls fraction, the + * output row count accounts for the null group (NDV + 1 for the null row). + * Also checks that the resulting nulls fraction for the grouping key is + * adjusted to {@code 1 / (NDV + 1)}. + */ + @Test + public void testSingleGroupingKeyWithNulls() + { + // y has 10 distinct values with 20% nulls, source has 200 rows + // Expected output: 10 + 1 = 11 rows (NDV + 1 for null group) + // Expected nulls fraction: 1 / (10 + 1) = 1/11 + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(200) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(50) + .setDistinctValuesCount(50) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(9) + .setDistinctValuesCount(10) + .setNullsFraction(0.2) + .build()) + .build()) + .check(check -> check + .outputRowsCount(11) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(9) + .distinctValuesCount(10) + .nullsFraction(1.0 / 11))); + } + + /** + * Verifies the row count estimate for multiple grouping keys with nulls. + * The output row count is the product of (NDV + null_row) for each key, + * capped at the source row count. The nulls fractions of grouping keys + * are each adjusted to {@code 1 / (NDV + 1)}. + */ + @Test + public void testMultipleGroupingKeysWithNulls() + { + // y: NDV=3, nullsFraction=0.1 -> contributes 3+1=4 + // z: NDV=5, nullsFraction=0.2 -> contributes 5+1=6 + // Product = 4 * 6 = 24, source has 200 rows, so 24 is used + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT), pb.variable("z", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT), pb.variable("z", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(200) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(10) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(2) + .setDistinctValuesCount(3) + .setNullsFraction(0.1) + .build()) + .addVariableStatistics(VARIABLE_Z, VariableStatsEstimate.builder() + .setLowValue(10) + .setHighValue(14) + .setDistinctValuesCount(5) + .setNullsFraction(0.2) + .build()) + .build()) + .check(check -> check + .outputRowsCount(24) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(2) + .distinctValuesCount(3) + .nullsFraction(1.0 / 4)) + .variableStats(VARIABLE_Z, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(10) + .highValue(14) + .distinctValuesCount(5) + .nullsFraction(1.0 / 6))); + } + + /** + * Verifies that when grouping key statistics are completely unknown + * (all NaN), the output row count estimate is also unknown. This mirrors + * how join stats handle missing column statistics. + */ + @Test + public void testAggregationWithUnknownGroupingKeyStats() + { + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(100) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(5) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.unknown()) + .build()) + .check(check -> check + .outputRowsCountUnknown() + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .unknownRange() + .distinctValuesCountUnknown() + .nullsFractionUnknown())); + } + + /** + * Verifies that a FINAL-step aggregation with a single grouping key + * produces the same estimates as a SINGLE-step aggregation, since both + * are handled by the same {@code groupBy} code path. + */ + @Test + public void testFinalAggregationMatchesSingleStep() + { + double sourceRowCount = 500; + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("sum", BIGINT), pb.rowExpression("sum(x)")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .step(AggregationNode.Step.FINAL) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(sourceRowCount) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(5) + .setNullsFraction(0.1) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(19) + .setDistinctValuesCount(20) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(20) + .variableStats(VARIABLE_Y, symbolStatsAssertion -> symbolStatsAssertion + .lowValue(0) + .highValue(19) + .distinctValuesCount(20) + .nullsFraction(0))); + } + + /** + * Verifies that a partial aggregation with a global grouping (no grouping keys) + * preserves the full source row count, since partial aggregations assume + * pessimistic (no) reduction. + */ + @Test + public void testPartialGlobalAggregationPreservesSourceRows() + { + double sourceRowCount = 300; + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .globalGrouping() + .step(AggregationNode.Step.PARTIAL) + .source(pb.values(pb.variable("x", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(sourceRowCount) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(100) + .setDistinctValuesCount(50) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check + .outputRowsCount(sourceRowCount)); + } + + /** + * Verifies that the aggregation output row count is correctly capped at the + * source row count when multiple grouping keys with no nulls produce a + * product of NDVs that exceeds the number of input rows. + */ + @Test + public void testMultipleGroupingKeysCappedToInputRows() + { + // y: NDV=50, no nulls -> contributes 50 + // z: NDV=50, no nulls -> contributes 50 + // Product = 50 * 50 = 2500, but source has only 100 rows => capped to 100 + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT), pb.variable("z", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT), pb.variable("z", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(100) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(49) + .setDistinctValuesCount(50) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Z, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(49) + .setDistinctValuesCount(50) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check.outputRowsCount(100)); + } + + /** + * Verifies that aggregation statistics are correctly computed for a + * SINGLE-step aggregation when the grouping key has a high NDV but the + * source row count is low. The output row count should equal the source + * row count since NDV cannot exceed it. + */ + @Test + public void testGroupingKeyNdvExceedsSourceRows() + { + // y: NDV=200, no nulls, but source only has 50 rows => capped to 50 + tester().assertStatsFor(pb -> pb + .registerVariable(pb.variable("x")) + .aggregation(ab -> ab + .addAggregation(pb.variable("count", BIGINT), pb.rowExpression("count()")) + .singleGroupingSet(pb.variable("y", BIGINT)) + .source(pb.values(pb.variable("x", BIGINT), pb.variable("y", BIGINT))))) + .withSourceStats(PlanNodeStatsEstimate.builder() + .setOutputRowCount(50) + .addVariableStatistics(VARIABLE_X, VariableStatsEstimate.builder() + .setLowValue(1) + .setHighValue(10) + .setDistinctValuesCount(10) + .setNullsFraction(0) + .build()) + .addVariableStatistics(VARIABLE_Y, VariableStatsEstimate.builder() + .setLowValue(0) + .setHighValue(199) + .setDistinctValuesCount(200) + .setNullsFraction(0) + .build()) + .build()) + .check(check -> check.outputRowsCount(50)); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java b/presto-main-base/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java index afe69f74e8fe7..c5248a3d8ae3b 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java +++ b/presto-main-base/src/test/java/com/facebook/presto/cost/TestScalarStatsCalculator.java @@ -114,10 +114,12 @@ public void testLiteral() .highValue(75.5) .nullsFraction(0.0); - assertCalculate(new StringLiteral("blah")) + VariableStatsAssertion blah = assertCalculate(new StringLiteral("blah")); + blah .distinctValuesCount(1.0) .lowValueUnknown() .highValueUnknown() + .averageRowSize(4.0) .nullsFraction(0.0); assertCalculate(new NullLiteral()) @@ -162,6 +164,7 @@ public void testVarbinaryConstant() .distinctValuesCount(1.0) .lowValueUnknown() .highValueUnknown() + .averageRowSize(11.0) .nullsFraction(0.0); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java b/presto-main-base/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java index dd2527f0435fd..3015c08993a43 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/TestQueryManagerConfig.java @@ -85,7 +85,9 @@ public void testDefaults() .setRateLimiterCacheLimit(1000) .setRateLimiterCacheWindowMinutes(5) .setEnableWorkerIsolation(false) - .setMinColumnarEncodingChannelsToPreferRowWiseEncoding(1000)); + .setMinColumnarEncodingChannelsToPreferRowWiseEncoding(1000) + .setMaxQueryAdmissionsPerSecond(Integer.MAX_VALUE) + .setMinRunningQueriesForPacing(30)); } @Test @@ -100,7 +102,7 @@ public void testExplicitPropertyMappings() .put("query.stage-count-warning-threshold", "12300") .put("max-total-running-task-count-to-kill-query", "60000") .put("max-query-running-task-count", "10000") - .put("experimental.max-total-running-task-count-to-not-execute-new-query", "50000") + .put("max-total-running-task-count-to-not-execute-new-query", "50000") .put("concurrency-threshold-to-enable-resource-group-refresh", "2") .put("resource-group-runtimeinfo-refresh-interval", "10ms") .put("query.schedule-split-batch-size", "99") @@ -141,6 +143,8 @@ public void testExplicitPropertyMappings() .put("query.cte-partitioning-provider-catalog", "hive") .put("query-manager.enable-worker-isolation", "true") .put("min-columnar-encoding-channels-to-prefer-row-wise-encoding", "123") + .put("query-manager.query-pacing.max-queries-per-second", "10") + .put("query-manager.query-pacing.min-running-queries", "5") .build(); QueryManagerConfig expected = new QueryManagerConfig() @@ -193,7 +197,9 @@ public void testExplicitPropertyMappings() .setRateLimiterCacheWindowMinutes(60) .setCtePartitioningProviderCatalog("hive") .setEnableWorkerIsolation(true) - .setMinColumnarEncodingChannelsToPreferRowWiseEncoding(123); + .setMinColumnarEncodingChannelsToPreferRowWiseEncoding(123) + .setMaxQueryAdmissionsPerSecond(10) + .setMinRunningQueriesForPacing(5); ConfigAssertions.assertFullMapping(properties, expected); } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/BenchmarkResourceGroup.java b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/BenchmarkResourceGroup.java index 1ab8ba0bc30e7..7fba90f5fdc63 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/BenchmarkResourceGroup.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/BenchmarkResourceGroup.java @@ -77,7 +77,7 @@ public static class BenchmarkData @Setup public void setup() { - root = new RootInternalResourceGroup("root", (group, export) -> {}, executor, ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker()); + root = new RootInternalResourceGroup("root", (group, export) -> {}, executor, ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(queries); root.setHardConcurrencyLimit(queries); diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestInternalResourceGroupManager.java b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestInternalResourceGroupManager.java index 257d957f8e578..b54e1984fcec2 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestInternalResourceGroupManager.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestInternalResourceGroupManager.java @@ -30,6 +30,11 @@ import org.weakref.jmx.MBeanExporter; import org.weakref.jmx.testing.TestingMBeanServer; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + public class TestInternalResourceGroupManager { @Test(expectedExceptions = PrestoException.class, expectedExceptionsMessageRegExp = ".*Presto server is still initializing.*") @@ -48,4 +53,218 @@ public void testQuerySucceedsWhenConfigurationManagerLoaded() internalResourceGroupManager.loadConfigurationManager(); internalResourceGroupManager.submit(new MockManagedQueryExecution(0), new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), command -> {}); } + + // Tests that admission always succeeds when pacing is disabled (default config) + @Test + public void testAdmissionPacingUnlimited() + { + // When maxQueryAdmissionsPerSecond is Integer.MAX_VALUE (default), admission should always succeed + QueryManagerConfig config = new QueryManagerConfig(); + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + // Multiple consecutive calls should all succeed + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + } + + // Tests that admission respects 1 query/second rate limit + @Test + public void testAdmissionPacingOnePerSecond() + throws InterruptedException + { + // When maxQueryAdmissionsPerSecond is 1, verify admission succeeds after waiting + QueryManagerConfig config = new QueryManagerConfig().setMaxQueryAdmissionsPerSecond(1); + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + // First admission should succeed + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Wait for 1 second (required interval) and verify next admission succeeds + Thread.sleep(1100); + assertTrue(manager.tryAcquireAdmissionSlot()); + } + + // Tests that admission respects 10 queries/second rate limit + @Test + public void testAdmissionPacingMultiplePerSecond() + throws InterruptedException + { + // When maxQueryAdmissionsPerSecond is 10, verify admission succeeds after waiting appropriate interval + QueryManagerConfig config = new QueryManagerConfig().setMaxQueryAdmissionsPerSecond(10); + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + // First admission should succeed + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Wait for 150ms (more than the 100ms interval required for 10 queries/sec) and verify next admission succeeds + Thread.sleep(150); + assertTrue(manager.tryAcquireAdmissionSlot()); + } + + // Tests that pacing is bypassed when running queries are below threshold + @Test + public void testAdmissionPacingBypassedBelowRunningQueryThreshold() + throws Exception + { + // Configure pacing with a threshold of 5 running queries + // When running queries are below threshold, pacing should be bypassed + QueryManagerConfig config = new QueryManagerConfig() + .setMaxQueryAdmissionsPerSecond(1) // Very slow pacing: 1 per second + .setMinRunningQueriesForPacing(5); // Threshold of 5 running queries + + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + manager.loadConfigurationManager(); + + // Create a resource group with some running queries (but below threshold) + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + MockManagedQueryExecution query2 = new MockManagedQueryExecution(0); + manager.submit(query1, new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), directExecutor()); + manager.submit(query2, new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), directExecutor()); + + // With only 2 running queries (below threshold of 5), pacing should be bypassed + // Multiple rapid admissions should all succeed without waiting + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Verify metrics are NOT tracked when pacing is bypassed + assertEquals(manager.getTotalAdmissionAttempts(), 0); + assertEquals(manager.getTotalAdmissionsGranted(), 0); + assertEquals(manager.getTotalAdmissionsDenied(), 0); + } + + // Tests that pacing is enforced when running queries exceed threshold + @Test + public void testAdmissionPacingAppliedAboveRunningQueryThreshold() + throws Exception + { + // Configure pacing with a threshold of 2 running queries + QueryManagerConfig config = new QueryManagerConfig() + .setMaxQueryAdmissionsPerSecond(1) // 1 per second + .setMinRunningQueriesForPacing(2); // Threshold of 2 running queries + + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + manager.loadConfigurationManager(); + + // Create resource groups with enough running queries to exceed threshold + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + MockManagedQueryExecution query2 = new MockManagedQueryExecution(0); + MockManagedQueryExecution query3 = new MockManagedQueryExecution(0); + manager.submit(query1, new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), directExecutor()); + manager.submit(query2, new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), directExecutor()); + manager.submit(query3, new SelectionContext<>(new ResourceGroupId("global"), ImmutableMap.of()), directExecutor()); + + // Wait for rate limit window to expire after query submissions (which internally call tryAcquireAdmissionSlot) + Thread.sleep(1100); + + // With 3 running queries (above threshold of 2), pacing should be applied + // First admission should succeed + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Immediate second attempt should be denied (need to wait 1 second) + assertFalse(manager.tryAcquireAdmissionSlot()); + + // Verify metrics ARE tracked when pacing is applied + // Note: Query 3's submission also triggered pacing (running queries = 2 at submission time), + // so we have 3 total attempts: 1 from query3 submission + 2 from explicit calls + assertEquals(manager.getTotalAdmissionAttempts(), 3); + assertEquals(manager.getTotalAdmissionsGranted(), 2); + assertEquals(manager.getTotalAdmissionsDenied(), 1); + } + + // Tests that pacing turns off when running queries drop below the threshold + @Test + public void testAdmissionPacingTurnsOffWhenRunningQueriesDropBelowThreshold() + throws Exception + { + // Configure pacing with a threshold of 2 running queries and a slow rate + QueryManagerConfig config = new QueryManagerConfig() + .setMaxQueryAdmissionsPerSecond(1) // 1 per second, so pacing should be visible + .setMinRunningQueriesForPacing(2); // Threshold of 2 running queries + + InternalResourceGroupManager> manager = new InternalResourceGroupManager<>( + (poolId, listener) -> {}, + config, + new NodeInfo("test"), + new MBeanExporter(new TestingMBeanServer()), + () -> null, + new ServerConfig(), + new InMemoryNodeManager(), + new ClusterResourceChecker(new CpuMemoryOverloadPolicy(new ClusterOverloadConfig()), new ClusterOverloadConfig(), new InMemoryNodeManager())); + + // Simulate being above the threshold by incrementing running queries counter + manager.incrementRunningQueries(); + manager.incrementRunningQueries(); + + // With 2 running queries (at threshold), pacing should be applied + // First admission should succeed and set the lastAdmittedQueryNanos timestamp + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Immediate second attempt should be denied (need to wait 1 second) + assertFalse(manager.tryAcquireAdmissionSlot()); + + // Verify metrics are tracked when pacing is applied + assertEquals(manager.getTotalAdmissionAttempts(), 2); + assertEquals(manager.getTotalAdmissionsGranted(), 1); + assertEquals(manager.getTotalAdmissionsDenied(), 1); + + // Now simulate queries finishing so that we drop below the threshold + manager.decrementRunningQueries(); + manager.decrementRunningQueries(); + + // With 0 running queries (below threshold of 2), pacing should be bypassed + // Multiple rapid admissions should all succeed without waiting + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + assertTrue(manager.tryAcquireAdmissionSlot()); + + // Verify metrics did NOT increase when pacing was bypassed + // (should still be the same as before the decrement) + assertEquals(manager.getTotalAdmissionAttempts(), 2); + assertEquals(manager.getTotalAdmissionsGranted(), 1); + assertEquals(manager.getTotalAdmissionsDenied(), 1); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java index 5890a210c44e1..79b035e279d39 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/resourceGroups/TestResourceGroups.java @@ -72,7 +72,7 @@ public class TestResourceGroups @Test(timeOut = 10_000) public void testQueueFull() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(1); root.setHardConcurrencyLimit(1); @@ -94,7 +94,7 @@ public void testQueueFull() @Test(timeOut = 10_000) public void testFairEligibility() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(4); root.setHardConcurrencyLimit(1); @@ -154,7 +154,7 @@ public void testFairEligibility() @Test public void testSetSchedulingPolicy() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(4); root.setHardConcurrencyLimit(1); @@ -200,7 +200,7 @@ public void testSetSchedulingPolicy() @Test(timeOut = 10_000) public void testFairQueuing() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(4); root.setHardConcurrencyLimit(1); @@ -246,7 +246,7 @@ public void testFairQueuing() @Test(timeOut = 10_000) public void testMemoryLimit() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, BYTE)); root.setMaxQueuedQueries(4); root.setHardConcurrencyLimit(3); @@ -274,7 +274,7 @@ public void testMemoryLimit() @Test public void testSubgroupMemoryLimit() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(10, BYTE)); root.setMaxQueuedQueries(4); root.setHardConcurrencyLimit(3); @@ -307,7 +307,7 @@ public void testSubgroupMemoryLimit() @Test(timeOut = 10_000) public void testSoftCpuLimit() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, BYTE)); root.setSoftCpuLimit(new Duration(1, SECONDS)); root.setHardCpuLimit(new Duration(2, SECONDS)); @@ -344,7 +344,7 @@ public void testSoftCpuLimit() @Test(timeOut = 10_000) public void testPerWorkerQueryLimit() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setWorkersPerQueryLimit(5); root.setMaxQueuedQueries(2); root.setHardConcurrencyLimit(2); @@ -377,7 +377,7 @@ public void testPerWorkerQueryLimit() @Test(timeOut = 10_000) public void testPerWorkerQueryLimitMultipleGroups() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setWorkersPerQueryLimit(5); root.setMaxQueuedQueries(5); root.setHardConcurrencyLimit(2); @@ -420,7 +420,7 @@ public void testPerWorkerQueryLimitMultipleGroups() @Test(timeOut = 10_000) public void testHardCpuLimit() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, BYTE)); root.setHardCpuLimit(new Duration(1, SECONDS)); root.setCpuQuotaGenerationMillisPerSecond(2000); @@ -447,7 +447,7 @@ public void testHardCpuLimit() @Test(timeOut = 10_000) public void testPriorityScheduling() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(100); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -497,7 +497,7 @@ public void testPriorityScheduling() @Test(timeOut = 20_000) public void testWeightedScheduling() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(4); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -546,7 +546,7 @@ public void testWeightedScheduling() @Test(timeOut = 30_000) public void testWeightedFairScheduling() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(50); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -589,7 +589,7 @@ public void testWeightedFairScheduling() @Test(timeOut = 10_000) public void testWeightedFairSchedulingEqualWeights() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(50); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -648,7 +648,7 @@ public void testWeightedFairSchedulingEqualWeights() @Test(timeOut = 20_000) public void testWeightedFairSchedulingNoStarvation() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(50); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -689,7 +689,7 @@ public void testWeightedFairSchedulingNoStarvation() @Test public void testGetInfo() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(40); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -779,7 +779,7 @@ public void testGetInfo() @Test public void testGetResourceGroupStateInfo() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, GIGABYTE)); root.setMaxQueuedQueries(40); root.setHardConcurrencyLimit(10); @@ -847,7 +847,7 @@ public void testGetResourceGroupStateInfo() @Test public void testGetStaticResourceGroupInfo() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, GIGABYTE)); root.setMaxQueuedQueries(100); root.setHardConcurrencyLimit(10); @@ -924,7 +924,7 @@ private Optional getResourceGroupInfoForId(InternalResourceGr @Test public void testGetBlockedQueuedQueries() { - RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker()); + RootInternalResourceGroup root = new RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, createNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(40); // Start with zero capacity, so that nothing starts running until we've added all the queries @@ -1098,4 +1098,215 @@ public String getName() return new ClusterResourceChecker(mockPolicy, config, createNodeManager()); } + + // Tests that when task limit is exceeded, new queries are queued instead of starting immediately + @Test(timeOut = 10_000) + public void testTaskLimitExceededQueuesQuery() + { + RootInternalResourceGroup root = new RootInternalResourceGroup( + "root", + (group, export) -> {}, + directExecutor(), + ignored -> Optional.empty(), + rg -> false, + createNodeManager(), + createClusterResourceChecker(), + QueryPacingContext.NOOP); + root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + root.setMaxQueuedQueries(10); + root.setHardConcurrencyLimit(10); + + // Set task limit exceeded + root.setTaskLimitExceeded(true); + + // Submit a query - it should be queued because task limit is exceeded + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + query1.startWaitingForPrerequisites(); + root.run(query1); + + // Query should be queued, not running + assertEquals(query1.getState(), QUEUED); + assertEquals(root.getQueuedQueries(), 1); + assertEquals(root.getRunningQueries(), 0); + } + + // Tests that queued queries start when task limit is no longer exceeded + @Test(timeOut = 10_000) + public void testQueryStartsWhenTaskLimitClears() + { + RootInternalResourceGroup root = new RootInternalResourceGroup( + "root", + (group, export) -> {}, + directExecutor(), + ignored -> Optional.empty(), + rg -> false, + createNodeManager(), + createClusterResourceChecker(), + QueryPacingContext.NOOP); + root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + root.setMaxQueuedQueries(10); + root.setHardConcurrencyLimit(10); + + // Set task limit exceeded + root.setTaskLimitExceeded(true); + + // Submit queries - they should be queued + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + query1.startWaitingForPrerequisites(); + root.run(query1); + MockManagedQueryExecution query2 = new MockManagedQueryExecution(0); + query2.startWaitingForPrerequisites(); + root.run(query2); + + assertEquals(query1.getState(), QUEUED); + assertEquals(query2.getState(), QUEUED); + assertEquals(root.getQueuedQueries(), 2); + assertEquals(root.getRunningQueries(), 0); + + // Clear task limit + root.setTaskLimitExceeded(false); + + // Process queued queries - they should now start + root.processQueuedQueries(); + + assertEquals(query1.getState(), RUNNING); + assertEquals(query2.getState(), RUNNING); + assertEquals(root.getQueuedQueries(), 0); + assertEquals(root.getRunningQueries(), 2); + } + + // Tests that queries in a subgroup hierarchy are properly queued and started when task limit changes + @Test(timeOut = 10_000) + public void testTaskLimitExceededWithSubgroups() + { + RootInternalResourceGroup root = new RootInternalResourceGroup( + "root", + (group, export) -> {}, + directExecutor(), + ignored -> Optional.empty(), + rg -> false, + createNodeManager(), + createClusterResourceChecker(), + QueryPacingContext.NOOP); + root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + root.setMaxQueuedQueries(10); + root.setHardConcurrencyLimit(10); + + InternalResourceGroup groupA = root.getOrCreateSubGroup("A", true); + groupA.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + groupA.setMaxQueuedQueries(10); + groupA.setHardConcurrencyLimit(10); + + InternalResourceGroup groupG = groupA.getOrCreateSubGroup("G", true); + groupG.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + groupG.setMaxQueuedQueries(10); + groupG.setHardConcurrencyLimit(10); + + // Set task limit exceeded + root.setTaskLimitExceeded(true); + + // Submit a query to leaf group G - it should be queued + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + query1.startWaitingForPrerequisites(); + groupG.run(query1); + + assertEquals(query1.getState(), QUEUED); + assertEquals(groupG.getQueuedQueries(), 1); + assertEquals(groupG.getRunningQueries(), 0); + + // Clear task limit and process queued queries + root.setTaskLimitExceeded(false); + root.processQueuedQueries(); + + // Query should now be running + assertEquals(query1.getState(), RUNNING); + assertEquals(groupG.getQueuedQueries(), 0); + assertEquals(groupG.getRunningQueries(), 1); + } + + // Tests that when task limit is exceeded, queries already running continue, but new ones are queued + @Test(timeOut = 10_000) + public void testTaskLimitExceededDoesNotAffectRunningQueries() + { + RootInternalResourceGroup root = new RootInternalResourceGroup( + "root", + (group, export) -> {}, + directExecutor(), + ignored -> Optional.empty(), + rg -> false, + createNodeManager(), + createClusterResourceChecker(), + QueryPacingContext.NOOP); + root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + root.setMaxQueuedQueries(10); + root.setHardConcurrencyLimit(10); + + // Submit a query before task limit is exceeded - it should run + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + query1.startWaitingForPrerequisites(); + root.run(query1); + assertEquals(query1.getState(), RUNNING); + + // Now set task limit exceeded + root.setTaskLimitExceeded(true); + + // Submit another query - it should be queued + MockManagedQueryExecution query2 = new MockManagedQueryExecution(0); + query2.startWaitingForPrerequisites(); + root.run(query2); + assertEquals(query2.getState(), QUEUED); + + // The first query should still be running + assertEquals(query1.getState(), RUNNING); + assertEquals(root.getRunningQueries(), 1); + assertEquals(root.getQueuedQueries(), 1); + } + + // Tests that task limit transitions work correctly with multiple cycles + @Test(timeOut = 10_000) + public void testTaskLimitExceededMultipleCycles() + { + RootInternalResourceGroup root = new RootInternalResourceGroup( + "root", + (group, export) -> {}, + directExecutor(), + ignored -> Optional.empty(), + rg -> false, + createNodeManager(), + createClusterResourceChecker(), + QueryPacingContext.NOOP); + root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); + root.setMaxQueuedQueries(10); + root.setHardConcurrencyLimit(10); + + // Cycle 1: Task limit exceeded, query queued + root.setTaskLimitExceeded(true); + MockManagedQueryExecution query1 = new MockManagedQueryExecution(0); + query1.startWaitingForPrerequisites(); + root.run(query1); + assertEquals(query1.getState(), QUEUED); + + // Clear task limit, query starts + root.setTaskLimitExceeded(false); + root.processQueuedQueries(); + assertEquals(query1.getState(), RUNNING); + + // Cycle 2: Task limit exceeded again, new query queued + root.setTaskLimitExceeded(true); + MockManagedQueryExecution query2 = new MockManagedQueryExecution(0); + query2.startWaitingForPrerequisites(); + root.run(query2); + assertEquals(query2.getState(), QUEUED); + assertEquals(query1.getState(), RUNNING); // query1 still running + + // Complete query1, processQueuedQueries should not start query2 (task limit still exceeded) + query1.complete(); + root.processQueuedQueries(); + assertEquals(query2.getState(), QUEUED); // Still queued because task limit exceeded + + // Clear task limit, query2 starts + root.setTaskLimitExceeded(false); + root.processQueuedQueries(); + assertEquals(query2.getState(), RUNNING); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java b/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java index 92e2e380a1bfb..aa0fa9a60d577 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java +++ b/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestExtractSpatialInnerJoin.java @@ -261,7 +261,7 @@ public void testSphericalGeographiesDoesFire() .on(p -> p.filter( sqlToRowExpression( - "ST_Distance(a, b) < 5000", + "ST_Distance(a, b) < BIGINT '5000'", ImmutableMap.of("a", SPHERICAL_GEOGRAPHY, "b", SPHERICAL_GEOGRAPHY)), p.join(INNER, p.values(p.variable("a", SPHERICAL_GEOGRAPHY)), diff --git a/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestGeoFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestGeoFunctions.java index b651f9188161b..28a20d1be8a63 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestGeoFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/geospatial/TestGeoFunctions.java @@ -457,12 +457,12 @@ public void testGeometryInvalidReason() assertInvalidReason("POLYGON ((0 0, 1 1, 0 1, 1 0, 0 0))", "Error constructing Polygon: shell is empty but holes are not"); assertInvalidReason("POLYGON ((0 0, 0 1, 0 1, 1 1, 1 0, 0 0), (2 2, 2 3, 3 3, 3 2, 2 2))", "Hole lies outside shell"); assertInvalidReason("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0), (2 2, 2 3, 3 3, 3 2, 2 2))", "Hole lies outside shell"); - assertInvalidReason("POLYGON ((0 0, 0 1, 2 1, 1 1, 1 0, 0 0))", "Ring Self-intersection"); + assertInvalidReason("POLYGON ((0 0, 0 1, 2 1, 1 1, 1 0, 0 0))", "Self-intersection"); assertInvalidReason("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0), (0 1, 1 1, 0.5 0.5, 0 1))", "Self-intersection"); assertInvalidReason("POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0), (0 0, 0.5 0.7, 1 1, 0.5 0.4, 0 0))", "Interior is disconnected"); assertInvalidReason("POLYGON ((0 0, -1 0.5, 0 1, 1 1, 1 0, 0 1, 0 0))", "Ring Self-intersection"); assertInvalidReason("MULTIPOLYGON (((0 0, 0 1, 1 1, 1 0, 0 0)), ((0.5 0.5, 0.5 2, 2 2, 2 0.5, 0.5 0.5)))", "Self-intersection"); - assertInvalidReason("GEOMETRYCOLLECTION (POINT (1 2), POLYGON ((0 0, 0 1, 2 1, 1 1, 1 0, 0 0)))", "Ring Self-intersection"); + assertInvalidReason("GEOMETRYCOLLECTION (POINT (1 2), POLYGON ((0 0, 0 1, 2 1, 1 1, 1 0, 0 0)))", "Self-intersection"); // non-simple geometries assertInvalidReason("MULTIPOINT (1 2, 2 4, 3 6, 1 2)", "[MultiPoint] Repeated point: (1.0 2.0)"); diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index 7ee833b6c9529..db0d91d897684 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -753,6 +753,18 @@ public Set getConnectorCapabilities(Session session, Conn throw new UnsupportedOperationException(); } + @Override + public void createBranch(Session session, TableHandle tableHandle, String branchName, boolean replace, boolean ifNotExists, Optional tableVersion, Optional retainDays, Optional minSnapshotsToKeep, Optional maxSnapshotAgeDays) + { + throw new UnsupportedOperationException(); + } + + @Override + public void createTag(Session session, TableHandle tableHandle, String branchName, boolean replace, boolean ifNotExists, Optional tableVersion, Optional retainDays) + { + throw new UnsupportedOperationException(); + } + @Override public void dropBranch(Session session, TableHandle tableHandle, String branchName, boolean branchExists) { diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java index 9e9a8b629124a..33757fb9d57d8 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.common.type.NamedTypeSignature; +import com.facebook.presto.common.type.RowFieldName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.common.type.TypeSignatureParameter; import org.testng.annotations.Test; @@ -265,4 +266,71 @@ public void testConvertApplicableTypeToVariableGenericArrayParamType() convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); } + + @Test + public void testConvertApplicableTypeToVariableNamedRowFieldsWithVarchar() + { + TypeSignature actualTypeSignature = parseTypeSignature("row(format_type varchar, num_vectors bigint)"); + TypeSignature expectedTypeSignature = new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.of(new RowFieldName("format_type", false)), + parseTypeSignature("varchar"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.of(new RowFieldName("num_vectors", false)), + parseTypeSignature("bigint")))); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableNamedRowFieldsWithMap() + { + TypeSignature actualTypeSignature = parseTypeSignature("row(metadata map(varchar, varchar), count bigint)"); + TypeSignature expectedTypeSignature = new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.of(new RowFieldName("metadata", false)), + new TypeSignature( + "map", + TypeSignatureParameter.of(parseTypeSignature("varchar")), + TypeSignatureParameter.of(parseTypeSignature("varchar"))))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.of(new RowFieldName("count", false)), + parseTypeSignature("bigint")))); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableSignatureWithVarcharMapBigint() + { + TypeSignature actualTypeSignature = parseTypeSignature( + "row(format_type varchar, num_vectors bigint, dimension integer, " + + "index_type varchar, distance_metric varchar, id_type varchar, " + + "metadata map(varchar, varchar))"); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + + List params = resolvedTypeSignature.getParameters(); + assertEquals(params.size(), 7); + + assertNamedField(params.get(0), "format_type", "varchar"); + assertNamedField(params.get(1), "num_vectors", "bigint"); + assertNamedField(params.get(2), "dimension", "integer"); + assertNamedField(params.get(3), "index_type", "varchar"); + assertNamedField(params.get(4), "distance_metric", "varchar"); + assertNamedField(params.get(5), "id_type", "varchar"); + assertNamedField(params.get(6), "metadata", "map"); + } + + private static void assertNamedField(TypeSignatureParameter typeSignatureParameter, String expectedFieldName, String expectedTypeBase) + { + assertTrue(typeSignatureParameter.isNamedTypeSignature()); + assertEquals(typeSignatureParameter.getNamedTypeSignature().getFieldName(), Optional.of(new RowFieldName(expectedFieldName, false))); + assertEquals(typeSignatureParameter.getNamedTypeSignature().getTypeSignature().getBase(), expectedTypeBase); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestMetadataManagerStats.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestMetadataManagerStats.java new file mode 100644 index 0000000000000..4f4532c95386b --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestMetadataManagerStats.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class TestMetadataManagerStats +{ + @Test + public void testInitialState() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Verify all counters start at 0 + assertEquals(stats.getListSchemaNamesCalls(), 0); + assertEquals(stats.getListTablesCalls(), 0); + assertEquals(stats.getGetTableMetadataCalls(), 0); + assertEquals(stats.getGetColumnHandlesCalls(), 0); + + // Verify timing stats are initialized + assertNotNull(stats.getListSchemaNamesTime()); + assertNotNull(stats.getListTablesTime()); + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 0.0); + assertEquals(stats.getListTablesTime().getAllTime().getCount(), 0.0); + } + + @Test + public void testRecordCalls() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Record some calls + stats.recordListSchemaNamesCall(1000000); // 1ms in nanoseconds + stats.recordListSchemaNamesCall(2000000); // 2ms + stats.recordListTablesCall(3000000); // 3ms + + // Verify counters incremented + assertEquals(stats.getListSchemaNamesCalls(), 2); + assertEquals(stats.getListTablesCalls(), 1); + + // Verify timing recorded + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 2.0); + assertEquals(stats.getListTablesTime().getAllTime().getCount(), 1.0); + } + + @Test + public void testTimingStatistics() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Record calls with different durations + stats.recordListSchemaNamesCall(1000000); // 1ms + stats.recordListSchemaNamesCall(5000000); // 5ms + stats.recordListSchemaNamesCall(3000000); // 3ms + + // Verify count + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 3.0); + + // Verify min/max are reasonable + assertTrue(stats.getListSchemaNamesTime().getAllTime().getMin() > 0); + assertTrue(stats.getListSchemaNamesTime().getAllTime().getMax() > 0); + assertTrue(stats.getListSchemaNamesTime().getAllTime().getMax() >= stats.getListSchemaNamesTime().getAllTime().getMin()); + } + + @Test + public void testMultipleOperations() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Record various operations + stats.recordListSchemaNamesCall(1000000); + stats.recordListTablesCall(2000000); + stats.recordGetTableMetadataCall(3000000); + stats.recordGetColumnHandlesCall(4000000); + + // Verify all were recorded independently + assertEquals(stats.getListSchemaNamesCalls(), 1); + assertEquals(stats.getListTablesCalls(), 1); + assertEquals(stats.getGetTableMetadataCalls(), 1); + assertEquals(stats.getGetColumnHandlesCalls(), 1); + + // Verify timing for each + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 1.0); + assertEquals(stats.getListTablesTime().getAllTime().getCount(), 1.0); + assertEquals(stats.getGetTableMetadataTime().getAllTime().getCount(), 1.0); + assertEquals(stats.getGetColumnHandlesTime().getAllTime().getCount(), 1.0); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestStatsRecordingMetadataManager.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestStatsRecordingMetadataManager.java new file mode 100644 index 0000000000000..f6cd66631009d --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestStatsRecordingMetadataManager.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import org.testng.annotations.Test; + +import static com.facebook.presto.spi.testing.InterfaceTestUtils.assertAllMethodsOverridden; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestStatsRecordingMetadataManager +{ + @Test + public void testEverythingDelegated() + { + assertAllMethodsOverridden(Metadata.class, StatsRecordingMetadataManager.class); + } + + @Test + public void testStatsRecording() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Verify initial state - no calls recorded + assertEquals(stats.getListSchemaNamesCalls(), 0); + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 0.0); + + // Record a call directly (without session to avoid transaction requirement) + stats.recordListSchemaNamesCall(1000000); + + // Verify stats were recorded + assertEquals(stats.getListSchemaNamesCalls(), 1); + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 1.0); + } + + @Test + public void testMultipleCallsRecorded() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Verify initial state + assertEquals(stats.getListSchemaNamesCalls(), 0); + assertEquals(stats.getListTablesCalls(), 0); + + // Record different metadata operations directly + stats.recordListSchemaNamesCall(1000000); + stats.recordListSchemaNamesCall(2000000); + stats.recordListTablesCall(3000000); + + // Verify all calls were recorded + assertEquals(stats.getListSchemaNamesCalls(), 2); + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 2.0); + assertEquals(stats.getListTablesCalls(), 1); + assertEquals(stats.getListTablesTime().getAllTime().getCount(), 1.0); + } + + @Test + public void testTimingRecorded() + { + MetadataManagerStats stats = new MetadataManagerStats(); + + // Record operation with timing + stats.recordListSchemaNamesCall(5000000); + + // Verify timing was recorded - count should be 1 + assertEquals(stats.getListSchemaNamesTime().getAllTime().getCount(), 1.0); + // Verify max and min are greater than 0 (some time was recorded) + assertTrue(stats.getListSchemaNamesTime().getAllTime().getMax() > 0, "Max time should be greater than 0"); + assertTrue(stats.getListSchemaNamesTime().getAllTime().getMin() > 0, "Min time should be greater than 0"); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java index 64bc0afd92f01..c837923ea9911 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/TestApproximateCountDistinctIpAddress.java @@ -20,8 +20,8 @@ import java.util.concurrent.ThreadLocalRandom; import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; public class TestApproximateCountDistinctIpAddress extends AbstractTestApproximateCountDistinct diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java index e27fc7f2e57d1..349b2a190fe43 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianAggregation.java @@ -43,6 +43,8 @@ import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.P4HyperLogLogType.P4_HYPER_LOG_LOG; import static com.facebook.presto.common.type.QuantileDigestParametricType.QDIGEST; @@ -68,8 +70,6 @@ import static com.facebook.presto.type.ArrayParametricType.ARRAY; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.MapParametricType.MAP; import static com.facebook.presto.type.RowParametricType.ROW; import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java index 34a84c7c8a3bc..815a396bd27d7 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/aggregation/noisyaggregation/TestNoisyCountGaussianRandomSeedAggregation.java @@ -43,6 +43,8 @@ import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.P4HyperLogLogType.P4_HYPER_LOG_LOG; import static com.facebook.presto.common.type.QuantileDigestParametricType.QDIGEST; @@ -68,8 +70,6 @@ import static com.facebook.presto.type.ArrayParametricType.ARRAY; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.type.MapParametricType.MAP; import static com.facebook.presto.type.RowParametricType.ROW; import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index e9b3e8641c885..dc8cf971df256 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -168,7 +168,8 @@ protected void assertInvalidFunction(String projection, StandardErrorCode errorC functionAssertions.assertInvalidFunction(projection, errorCode, messagePattern); } - protected void assertInvalidFunction(String projection, String messagePattern) + @Override + public void assertInvalidFunction(String projection, String messagePattern) { functionAssertions.assertInvalidFunction(projection, INVALID_FUNCTION_ARGUMENT, messagePattern); } @@ -208,7 +209,8 @@ protected void assertInvalidCast(String projection) functionAssertions.assertInvalidCast(projection); } - protected void assertInvalidCast(@Language("SQL") String projection, String message) + @Override + public void assertInvalidCast(@Language("SQL") String projection, String message) { functionAssertions.assertInvalidCast(projection, message); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java index 21f2b615887c9..61f80a92e5170 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestIpPrefixFunctions.java @@ -13,311 +13,10 @@ */ package com.facebook.presto.operator.scalar; -import com.facebook.presto.common.type.ArrayType; -import com.google.common.collect.ImmutableList; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import com.facebook.presto.tests.operator.scalar.AbstractTestIpPrefix; public class TestIpPrefixFunctions extends AbstractTestFunctions + implements AbstractTestIpPrefix { - @DataProvider(name = "public-ip-provider") - public Object[] publicIpProvider() - { - return new Object[] { - "6.7.8.9", - "157.240.200.99", - "8.8.8.8", - "128.1.2.8", - "2a03:2880:f031:12:face:b00c:0:2", - "2600:1406:6c00::173c:ad43", - "2607:f8b0:4007:818::2004" - }; - } - - @DataProvider(name = "private-ip-provider") - public Object[][] privateIpProvider() - { - return new Object[][] { - // The first and last IP address in each private range - {"0.0.0.0"}, {"0.255.255.255"}, // 0.0.0.0/8 RFC1122: "This host on this network" - {"10.0.0.0"}, {"10.255.255.255"}, // 10.0.0.0/8 RFC1918: Private-Use - {"100.64.0.0"}, {"100.127.255.255"}, // 100.64.0.0/10 RFC6598: Shared Address Space - {"127.0.0.0"}, {"127.255.255.255"}, // 127.0.0.0/8 RFC1122: Loopback - {"169.254.0.0"}, {"169.254.255.255"}, // 169.254.0.0/16 RFC3927: Link Local - {"172.16.0.0"}, {"172.31.255.255"}, // 172.16.0.0/12 RFC1918: Private-Use - {"192.0.0.0"}, {"192.0.0.255"}, // 192.0.0.0/24 RFC6890: IETF Protocol Assignments - {"192.0.2.0"}, {"192.0.2.255"}, // 192.0.2.0/24 RFC5737: Documentation (TEST-NET-1) - {"192.88.99.0"}, {"192.88.99.255"}, // 192.88.99.0/24 RFC3068: 6to4 Relay anycast - {"192.168.0.0"}, {"192.168.255.255"}, // 192.168.0.0/16 RFC1918: Private-Use - {"198.18.0.0"}, {"198.19.255.255"}, // 198.18.0.0/15 RFC2544: Benchmarking - {"198.51.100.0"}, {"198.51.100.255"}, // 198.51.100.0/24 RFC5737: Documentation (TEST-NET-2) - {"203.0.113.0"}, {"203.0.113.255"}, // 203.0.113.0/24 RFC5737: Documentation (TEST-NET-3) - {"240.0.0.0"}, {"255.255.255.255"}, // 240.0.0.0/4 RFC1112: Reserved - {"::"}, {"::"}, // ::/128 RFC4291: Unspecified address - {"::1"}, {"::1"}, // ::1/128 RFC4291: Loopback address - {"100::"}, {"100::ffff:ffff:ffff:ffff"}, // 100::/64 RFC6666: Discard-Only Address Block - {"64:ff9b:1::"}, {"64:ff9b:1:ffff:ffff:ffff:ffff:ffff"}, // 64:ff9b:1::/48 RFC8215: IPv4-IPv6 Translation - {"2001:2::"}, {"2001:2:0:ffff:ffff:ffff:ffff:ffff"}, // 2001:2::/48 RFC5180,RFC Errata 1752: Benchmarking - {"2001:db8::"}, {"2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001:db8::/32 RFC3849: Documentation - {"2001::"}, {"2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001::/23 RFC2928: IETF Protocol Assignments - {"5f00::"}, {"5f00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 5f00::/16 RFC-ietf-6man-sids-06: Segment Routing (SRv6) - {"fe80::"}, {"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fe80::/10 RFC4291: Link-Local Unicast - {"fc00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fc00::/7 RFC4193, RFC8190: Unique Local - // some IPs in the middle of ranges - {"10.1.2.3"}, - {"100.64.3.2"}, - {"192.168.55.99"}, - {"2001:0DB8:0000:0000:face:b00c:0000:0000"}, - {"0100:0000:0000:0000:ffff:ffff:0000:0000"} - }; - } - - @Test - public void testIpAddressIpPrefix() - { - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); - assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); - assertFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); - assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 0)", IPPREFIX, "::/0"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); - } - - @Test - public void testStringIpPrefix() - { - assertFunction("IP_PREFIX('1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX('1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); - assertFunction("IP_PREFIX('1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); - assertFunction("IP_PREFIX('::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); - assertFunction("IP_PREFIX('64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); - assertFunction("IP_PREFIX('64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); - assertFunction("IP_PREFIX('64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); - assertFunction("IP_PREFIX('64:ff9b::17', 0)", IPPREFIX, "::/0"); - assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); - assertInvalidFunction("IP_PREFIX('64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidFunction("IP_PREFIX('64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); - assertInvalidCast("IP_PREFIX('localhost', 24)", "Cannot cast value to IPADDRESS: localhost"); - assertInvalidCast("IP_PREFIX('64::ff9b::10', 24)", "Cannot cast value to IPADDRESS: 64::ff9b::10"); - assertInvalidCast("IP_PREFIX('64:face:book::10', 24)", "Cannot cast value to IPADDRESS: 64:face:book::10"); - assertInvalidCast("IP_PREFIX('123.456.789.012', 24)", "Cannot cast value to IPADDRESS: 123.456.789.012"); - } - - @Test - public void testIpSubnetMin() - { - assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/24')", IPADDRESS, "1.2.3.0"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/32')", IPADDRESS, "1.2.3.4"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/127')", IPADDRESS, "64:ff9b::16"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/128')", IPADDRESS, "64:ff9b::17"); - assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "::"); - } - - @Test - public void testIpSubnetMax() - { - assertFunction("IP_SUBNET_MAX(IPPREFIX '1.2.3.128/26')", IPADDRESS, "1.2.3.191"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '192.168.128.4/32')", IPADDRESS, "192.168.128.4"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '10.1.16.3/9')", IPADDRESS, "10.127.255.255"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/127')", IPADDRESS, "2001:db8::17"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/128')", IPADDRESS, "2001:db8::16"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::ffff:ffff:ffff:ffff"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/72')", IPADDRESS, "64:ff9b::ff:ffff:ffff:ffff"); - assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); - } - - @Test - public void testIpSubnetRange() - { - assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.160/24')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.0", "1.2.3.255")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.128/31')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.128", "1.2.3.129")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/32')", new ArrayType(IPADDRESS), ImmutableList.of("10.1.6.46", "10.1.6.46")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/0')", new ArrayType(IPADDRESS), ImmutableList.of("0.0.0.0", "255.255.255.255")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/64')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::", "64:ff9b::ffff:ffff:ffff:ffff")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::52f4/120')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::5200", "64:ff9b::52ff")); - assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/128')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::17", "64:ff9b::17")); - } - - @Test - public void testIsSubnetOf() - { - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.3.129')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.5.1')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/32', IPADDRESS '1.2.3.128')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/0', IPADDRESS '192.168.5.1')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ff9b::ffff:ff')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ffff::17')", BOOLEAN, false); - - assertFunction("IS_SUBNET_OF(IPPREFIX '192.168.3.131/26', IPPREFIX '192.168.3.144/30')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.5.1/30')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.3.128/26')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ff9b::ff:25/80')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ffff::17/64')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/37', IPPREFIX '2804:431:b000::/38')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/38', IPPREFIX '2804:431:b000::/37')", BOOLEAN, false); - assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/22', IPPREFIX '170.0.52.0/24')", BOOLEAN, true); - assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/24', IPPREFIX '170.0.52.0/22')", BOOLEAN, false); - } - - @Test - public void testIpv4PrefixCollapse() - { - // simple - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.1.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/23")); - - // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.1.0/24', IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.2.0/24', IPPREFIX '192.168.9.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/23", "192.168.2.0/24", "192.168.9.0/24")); - } - - @Test - public void testIpv6PrefixCollapse() - { - // simple - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090::/48', IPPREFIX '2620:10d:c091::/48'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2620:10d:c090::/47")); - - // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2804:13c:4d6:e200::/56', IPPREFIX '2804:13c:4d6:dd00::/56', IPPREFIX '2804:13c:4d6:dc00::/56', IPPREFIX '2804:13c:4d6:de00::/56'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2804:13c:4d6:dc00::/55", "2804:13c:4d6:de00::/56", "2804:13c:4d6:e200::/56")); - } - - @Test - public void testIpPrefixCollapseIpv4SingleIPs() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.1/32', IPPREFIX '192.168.33.1/32'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.1/32", "192.168.33.1/32")); - } - - @Test - public void testIpPrefixCollapseIpv6SingleIPs() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090:400::5:a869/128', IPPREFIX '2620:10d:c091:400::5:a869/128'])", - new ArrayType(IPPREFIX), - ImmutableList.of("2620:10d:c090:400::5:a869/128", "2620:10d:c091:400::5:a869/128")); - } - - @Test - public void testIpPrefixCollapseSinglePrefixReturnsSamePrefix() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - } - - @Test - public void testIpPrefixCollapseOverlappingPrefixes() - { - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.0.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.2.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.3.0/24'])", - new ArrayType(IPPREFIX), - ImmutableList.of("192.168.0.0/22")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.64.0/18', IPPREFIX '10.2.0.0/15', IPPREFIX '10.0.0.0/8', IPPREFIX '11.0.0.0/8', IPPREFIX '172.168.32.0/20', IPPREFIX '172.168.0.0/18'])", - new ArrayType(IPPREFIX), - ImmutableList.of("10.0.0.0/7", "172.168.0.0/18")); - assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.0.0/8', IPPREFIX '10.0.0.0/7'])", - new ArrayType(IPPREFIX), - ImmutableList.of("10.0.0.0/7")); - } - - @Test - public void testIpPrefixCollapseEmptyArrayInput() - { - assertFunction("IP_PREFIX_COLLAPSE(CAST(ARRAY[] AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), ImmutableList.of()); - } - - @Test - public void testIpPrefixCollapseNullInput() - { - assertFunction("IP_PREFIX_COLLAPSE(CAST(NULL AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), null); - } - - @Test - public void testIpPrefixCollapseNoNullPrefixesError() - { - assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', CAST(NULL AS IPPREFIX)])", - "ip_prefix_collapse does not support null elements"); - } - - @Test - public void testIpPrefixCollapseMixedIpVersionError() - { - assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '2409:4043:251a:d200::/56'])", - "All IPPREFIX elements must be the same IP version."); - } - - @Test (dataProvider = "private-ip-provider") - public void testIsPrivateTrue(String ipAddress) - { - assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, true); - } - - @Test (dataProvider = "public-ip-provider") - public void testIsPrivateIpFalse(String ipAddress) - { - assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, false); - } - - @Test - public void testIsPrivateIpNull() - { - assertFunction("IS_PRIVATE_IP(NULL)", BOOLEAN, null); - } - - @Test - public void testIpPrefixSubnets() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 25)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/25", "192.168.1.128/25")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 26)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.0.0/26", "192.168.0.64/26", "192.168.0.128/26", "192.168.0.192/26")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2A03:2880:C000::/34', 37)", - new ArrayType(IPPREFIX), - ImmutableList.of("2a03:2880:c000::/37", "2a03:2880:c800::/37", "2a03:2880:d000::/37", "2a03:2880:d800::/37", "2a03:2880:e000::/37", "2a03:2880:e800::/37", "2a03:2880:f000::/37", "2a03:2880:f800::/37")); - } - - @Test - public void testIpPrefixSubnetsReturnSelf() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 24)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/24")); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2804:431:b000::/38', 38)", new ArrayType(IPPREFIX), ImmutableList.of("2804:431:b000::/38")); - } - - @Test - public void testIpPrefixSubnetsNewPrefixLengthLongerReturnsEmpty() - { - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 23)", new ArrayType(IPPREFIX), ImmutableList.of()); - assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 48)", new ArrayType(IPPREFIX), ImmutableList.of()); - } - - @Test - public void testIpPrefixSubnetsInvalidPrefixLengths() - { - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', -1)", "Invalid prefix length for IPv4: -1"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 33)", "Invalid prefix length for IPv4: 33"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', -1)", "Invalid prefix length for IPv6: -1"); - assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 129)", "Invalid prefix length for IPv6: 129"); - } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestTryFunction.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestTryFunction.java index ed82543100a47..555a1ec61d466 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestTryFunction.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestTryFunction.java @@ -13,14 +13,21 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.ErrorType; +import com.facebook.presto.common.function.SqlFunctionProperties; import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.common.type.SqlDecimal; +import com.facebook.presto.common.type.TimeZoneKey; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.SqlType; +import com.google.common.collect.ImmutableSet; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.Locale; + import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.DecimalType.createDecimalType; @@ -31,6 +38,9 @@ import static com.facebook.presto.common.type.VarcharType.createVarcharType; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static java.util.Arrays.asList; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.fail; public class TestTryFunction extends AbstractTestFunctions @@ -73,4 +83,112 @@ public void testExceptions() // Exceptions that should not be suppressed assertInvalidFunction("\"$internal$try\"(() -> throw_error())", GENERIC_INTERNAL_ERROR); } + + @Test + public void testErrorNotCatchableByDefault() + { + // Custom error codes should NOT be catchable by TRY by default + ErrorCode customError = new ErrorCode(0x0005_0001, "CUSTOM_ERROR", ErrorType.EXTERNAL); + PrestoException customException = new PrestoException(() -> customError, "Custom error"); + + SqlFunctionProperties propertiesEmpty = SqlFunctionProperties.builder() + .setTimeZoneKey(TimeZoneKey.UTC_KEY) + .setSessionStartTime(System.currentTimeMillis()) + .setSessionLocale(Locale.ENGLISH) + .setSessionUser("test") + .setTryCatchableErrorCodes(ImmutableSet.of()) + .build(); + + try { + TryFunction.tryLong(propertiesEmpty, () -> { + throw customException; + }); + fail("Expected PrestoException to be thrown when error code is not in catchable list"); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), customError); + } + } + + @Test + public void testErrorCatchableWithSessionProperty() + { + // Errors should be catchable by TRY when error code name is in the session property list + ErrorCode customError = new ErrorCode(0x0005_0001, "CUSTOM_ERROR", ErrorType.EXTERNAL); + PrestoException customException = new PrestoException(() -> customError, "Custom error"); + + SqlFunctionProperties propertiesWithError = SqlFunctionProperties.builder() + .setTimeZoneKey(TimeZoneKey.UTC_KEY) + .setSessionStartTime(System.currentTimeMillis()) + .setSessionLocale(Locale.ENGLISH) + .setSessionUser("test") + .setTryCatchableErrorCodes(ImmutableSet.of("CUSTOM_ERROR")) + .build(); + + Long result = TryFunction.tryLong(propertiesWithError, () -> { + throw customException; + }); + assertNull(result, "Custom error should be caught when error code is in catchable list"); + } + + @Test + public void testMultipleErrorsCatchableWithSessionProperty() + { + // Multiple error codes can be specified in the session property + ErrorCode error1 = new ErrorCode(0x0005_0001, "ERROR_ONE", ErrorType.EXTERNAL); + ErrorCode error2 = new ErrorCode(0x0005_0002, "ERROR_TWO", ErrorType.EXTERNAL); + ErrorCode error3 = new ErrorCode(0x0005_0003, "ERROR_THREE", ErrorType.EXTERNAL); + + SqlFunctionProperties propertiesWithMultiple = SqlFunctionProperties.builder() + .setTimeZoneKey(TimeZoneKey.UTC_KEY) + .setSessionStartTime(System.currentTimeMillis()) + .setSessionLocale(Locale.ENGLISH) + .setSessionUser("test") + .setTryCatchableErrorCodes(ImmutableSet.of("ERROR_ONE", "ERROR_TWO")) + .build(); + + // ERROR_ONE should be caught + PrestoException exception1 = new PrestoException(() -> error1, "Error one"); + Long result1 = TryFunction.tryLong(propertiesWithMultiple, () -> { + throw exception1; + }); + assertNull(result1, "ERROR_ONE should be caught when in catchable list"); + + // ERROR_TWO should be caught + PrestoException exception2 = new PrestoException(() -> error2, "Error two"); + Long result2 = TryFunction.tryLong(propertiesWithMultiple, () -> { + throw exception2; + }); + assertNull(result2, "ERROR_TWO should be caught when in catchable list"); + + // ERROR_THREE should NOT be caught (not in list) + PrestoException exception3 = new PrestoException(() -> error3, "Error three"); + try { + TryFunction.tryLong(propertiesWithMultiple, () -> { + throw exception3; + }); + fail("Expected PrestoException for ERROR_THREE which is not in catchable list"); + } + catch (PrestoException e) { + assertEquals(e.getErrorCode(), error3); + } + } + + @Test + public void testDefaultCatchableByTryStillWorks() + { + // Errors marked with catchableByTry=true in their definition should still be caught + // even without session property (e.g., DIVISION_BY_ZERO) + SqlFunctionProperties propertiesEmpty = SqlFunctionProperties.builder() + .setTimeZoneKey(TimeZoneKey.UTC_KEY) + .setSessionStartTime(System.currentTimeMillis()) + .setSessionLocale(Locale.ENGLISH) + .setSessionUser("test") + .setTryCatchableErrorCodes(ImmutableSet.of()) + .build(); + + // This test uses the actual TRY function behavior through assertFunction + // DIVISION_BY_ZERO is marked catchableByTry=true in StandardErrorCode + assertFunction("\"$internal$try\"(() -> 1/0)", INTEGER, null); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/security/TestAccessControlManager.java b/presto-main-base/src/test/java/com/facebook/presto/security/TestAccessControlManager.java index 480c7fb2f44f0..d1cc72af54ad9 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/security/TestAccessControlManager.java +++ b/presto-main-base/src/test/java/com/facebook/presto/security/TestAccessControlManager.java @@ -604,6 +604,18 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con throw new UnsupportedOperationException(); } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + throw new UnsupportedOperationException(); + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + throw new UnsupportedOperationException(); + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-main-base/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java b/presto-main-base/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java index f38af02192ce5..f14a66b5a2629 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java +++ b/presto-main-base/src/test/java/com/facebook/presto/server/TestQueryStateInfo.java @@ -22,6 +22,7 @@ import com.facebook.presto.execution.QueryState; import com.facebook.presto.execution.QueryStats; import com.facebook.presto.execution.resourceGroups.InternalResourceGroup; +import com.facebook.presto.execution.resourceGroups.QueryPacingContext; import com.facebook.presto.execution.scheduler.clusterOverload.ClusterOverloadPolicy; import com.facebook.presto.execution.scheduler.clusterOverload.ClusterResourceChecker; import com.facebook.presto.metadata.InMemoryNodeManager; @@ -62,7 +63,7 @@ public class TestQueryStateInfo @Test public void testQueryStateInfo() { - InternalResourceGroup.RootInternalResourceGroup root = new InternalResourceGroup.RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker()); + InternalResourceGroup.RootInternalResourceGroup root = new InternalResourceGroup.RootInternalResourceGroup("root", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); root.setSoftMemoryLimit(new DataSize(1, MEGABYTE)); root.setMaxQueuedQueries(40); root.setHardConcurrencyLimit(0); diff --git a/presto-main-base/src/test/java/com/facebook/presto/server/TestServerConfig.java b/presto-main-base/src/test/java/com/facebook/presto/server/TestServerConfig.java index b570f157a36f4..d572a34c47e67 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/server/TestServerConfig.java +++ b/presto-main-base/src/test/java/com/facebook/presto/server/TestServerConfig.java @@ -35,6 +35,7 @@ public void testDefaults() { assertRecordedDefaults(ConfigAssertions.recordDefaults(ServerConfig.class) .setCoordinator(true) + .setWebUIEnabled(true) .setPrestoVersion(null) .setDataSources(null) .setIncludeExceptionInResponse(true) @@ -58,6 +59,7 @@ public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() .put("coordinator", "false") + .put("webui-enabled", "false") .put("presto.version", "test") .put("datasources", "jmx") .put("http.include-exception-in-response", "false") @@ -78,6 +80,7 @@ public void testExplicitPropertyMappings() ServerConfig expected = new ServerConfig() .setCoordinator(false) + .setWebUIEnabled(false) .setPrestoVersion("test") .setDataSources("jmx") .setIncludeExceptionInResponse(false) diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java index af25f7ca02b49..9ea7a163cec3c 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java @@ -33,6 +33,13 @@ import com.facebook.presto.connector.tvf.TestingTableFunctions.TableArgumentRowSemanticsFunction; import com.facebook.presto.connector.tvf.TestingTableFunctions.TwoScalarArgumentsFunction; import com.facebook.presto.connector.tvf.TestingTableFunctions.TwoTableArgumentsFunction; +import com.facebook.presto.cost.CostCalculator; +import com.facebook.presto.cost.CostCalculatorUsingExchanges; +import com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges; +import com.facebook.presto.cost.CostComparator; +import com.facebook.presto.cost.TaskCountEstimator; +import com.facebook.presto.execution.QueryManagerConfig; +import com.facebook.presto.execution.TaskManagerConfig; import com.facebook.presto.execution.warnings.WarningCollectorConfig; import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutor; @@ -43,13 +50,16 @@ import com.facebook.presto.metadata.InMemoryNodeManager; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.connector.Connector; import com.facebook.presto.spi.connector.ConnectorMetadata; import com.facebook.presto.spi.connector.ConnectorSplitManager; @@ -63,13 +73,22 @@ import com.facebook.presto.spi.procedure.Procedure; import com.facebook.presto.spi.procedure.Procedure.Argument; import com.facebook.presto.spi.procedure.TableDataRewriteDistributedProcedure; +import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.spi.security.AllowAllAccessControl; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spi.transaction.IsolationLevel; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.expressions.JsonCodecRowExpressionSerde; import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.PartitioningProviderManager; +import com.facebook.presto.sql.planner.PlanFragmenter; +import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; +import com.facebook.presto.sql.planner.sanity.PlanChecker; import com.facebook.presto.sql.tree.NodeLocation; import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.testing.TestProcedureRegistry; import com.facebook.presto.testing.TestingAccessControlManager; import com.facebook.presto.testing.TestingMetadata; @@ -80,6 +99,8 @@ import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; import org.testng.annotations.BeforeClass; +import org.weakref.jmx.MBeanExporter; +import org.weakref.jmx.testing.TestingMBeanServer; import java.util.ArrayList; import java.util.Collections; @@ -87,6 +108,7 @@ import java.util.Optional; import java.util.function.Consumer; +import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.presto.SystemSessionProperties.CHECK_ACCESS_CONTROL_ON_UTILIZED_COLUMNS_ONLY; import static com.facebook.presto.SystemSessionProperties.CHECK_ACCESS_CONTROL_WITH_SUBFIELDS; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -108,6 +130,7 @@ import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -350,15 +373,15 @@ public void setup() new MaterializedViewDefinition.ColumnMapping(materializedViewTableColumn, Collections.singletonList(baseTableColumns))); MaterializedViewDefinition materializedViewData1 = new MaterializedViewDefinition( - "select a from t2", - "s1", - "mv1", - baseTables, - Optional.of("user"), - Optional.empty(), - columnMappings, - new ArrayList<>(), - Optional.of(new ArrayList<>(Collections.singletonList("a")))); + "select a from t2", + "s1", + "mv1", + baseTables, + Optional.of("user"), + Optional.empty(), + columnMappings, + new ArrayList<>(), + Optional.of(new ArrayList<>(Collections.singletonList("a")))); ConnectorTableMetadata materializedViewMetadata1 = new ConnectorTableMetadata( materializedTable, ImmutableList.of(ColumnMetadata.builder().setName("a").setType(BIGINT).build())); @@ -566,9 +589,11 @@ private void analyze(Session clientSession, WarningCollector warningCollector, @ .readUncommitted() .readOnly() .execute(clientSession, session -> { - Analyzer analyzer = AbstractAnalyzerTest.createAnalyzer(session, metadata, warningCollector, query); + Analyzer analyzer = AbstractAnalyzerTest.createAnalyzer(session, metadata, warningCollector, Optional.empty(), query); Statement statement = SQL_PARSER.createStatement(query); - analyzer.analyze(statement); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissions(accessControlReferences, analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); }); } @@ -642,18 +667,61 @@ protected void assertFails(Session session, SemanticErrorCode error, String mess } } - protected static Analyzer createAnalyzer(Session session, Metadata metadata, WarningCollector warningCollector, String query) + protected static Analyzer createAnalyzer(Session session, Metadata metadata, WarningCollector warningCollector, Optional queryExplainer, String query) { return new Analyzer( session, metadata, SQL_PARSER, new AllowAllAccessControl(), - Optional.empty(), + queryExplainer, emptyList(), emptyMap(), warningCollector, - query); + query, + new ViewDefinitionReferences()); + } + + protected static QueryExplainer createTestingQueryExplainer(Session session, AccessControl accessControl, Metadata metadata) + { + try (LocalQueryRunner localQueryRunner = new LocalQueryRunner(session)) { + SqlParser sqlParser = new SqlParser(); + FeaturesConfig featuresConfig = new FeaturesConfig(); + TaskCountEstimator taskCountEstimator = new TaskCountEstimator(localQueryRunner::getNodeCount); + CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); + List optimizers = new PlanOptimizers( + metadata, + sqlParser, + localQueryRunner.getNodeCount() == 1, + new MBeanExporter(new TestingMBeanServer()), + localQueryRunner.getSplitManager(), + localQueryRunner.getPlanOptimizerManager(), + localQueryRunner.getPageSourceManager(), + localQueryRunner.getStatsCalculator(), + costCalculator, + new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), + new CostComparator(featuresConfig), + taskCountEstimator, + new PartitioningProviderManager(), + featuresConfig, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + localQueryRunner.getMetadata().getFunctionAndTypeManager(), + new JsonCodecRowExpressionSerde(jsonCodec(RowExpression.class))), + new TaskManagerConfig(), + localQueryRunner.getAccessControl()) + .getPlanningTimeOptimizers(); + return new QueryExplainer( + optimizers, + new PlanFragmenter(metadata, localQueryRunner.getNodePartitioningManager(), new QueryManagerConfig(), featuresConfig, localQueryRunner.getPlanCheckerProviderManager()), + metadata, + accessControl, + sqlParser, + localQueryRunner.getStatsCalculator(), + costCalculator, + ImmutableMap.of(), + new PlanChecker(featuresConfig, false, localQueryRunner.getPlanCheckerProviderManager())); + } } private Catalog createTestingCatalog(String catalogName, ConnectorId connectorId) diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java index c3236d3e6627a..f6bfcc5d75e58 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java @@ -125,6 +125,13 @@ private static void assertNoWarning(WarningCollector warningCollector) assertTrue(warnings.isEmpty()); } + @Test + public void testCTASIfNotExistsWhenExists() + { + assertHasWarning(analyzeWithWarnings("CREATE TABLE IF NOT EXISTS t1 AS SELECT a, b FROM t1"), + SEMANTIC_WARNING, "Table 'tpch.s1.t1' already exists, skipping table creation"); + } + @Test public void testNonComparableGroupBy() { @@ -301,7 +308,32 @@ public void testReferenceToOutputColumnFromOrderByAggregation() @Test public void testHavingReferencesOutputAlias() { - assertFails(MISSING_ATTRIBUTE, "SELECT sum(a) x FROM t1 HAVING x > 5"); + // HAVING now support referencing SELECT aliases for improved SQL compatibility + analyze("SELECT sum(a) x FROM t1 HAVING x > 5"); + analyze("SELECT sum(a) AS total FROM t1 GROUP BY b HAVING total > 10"); + analyze("SELECT count(*) AS cnt, sum(a) AS total FROM t1 GROUP BY b HAVING cnt > 5 AND total > 100"); + analyze("SELECT sum(a) as sum_a FROM t1 GROUP BY b HAVING sum_a > 1"); + } + + @Test + public void testHavingAmbiguousAlias() + { + // Ambiguous alias referenced in HAVING should throw appropriate error + assertFails(AMBIGUOUS_ATTRIBUTE, "SELECT sum(a) AS x, count(b) AS x FROM t1 GROUP BY c HAVING x > 5"); + } + + @Test + public void testHavingNonExistentAlias() + { + // Non-existent alias in HAVING should fail with MISSING_ATTRIBUTE + assertFails(MISSING_ATTRIBUTE, "SELECT sum(a) AS total FROM t1 GROUP BY b HAVING unknown_alias > 5"); + } + + @Test + public void testHavingWindowFunctionViaAlias() + { + // Window functions are not allowed in HAVING, even when referenced via alias + assertFails(NESTED_WINDOW, "SELECT row_number() OVER () AS rn FROM t1 GROUP BY b HAVING rn > 1"); } @Test diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestColumnAndSubfieldAnalyzer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestColumnAndSubfieldAnalyzer.java index df194fc6b4783..e8a8e5a942a00 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestColumnAndSubfieldAnalyzer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestColumnAndSubfieldAnalyzer.java @@ -17,6 +17,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.Subfield; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.sql.tree.Statement; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -24,12 +25,14 @@ import org.testng.annotations.Test; import java.util.Map; +import java.util.Optional; import java.util.Set; import static com.facebook.presto.SystemSessionProperties.CHECK_ACCESS_CONTROL_ON_UTILIZED_COLUMNS_ONLY; import static com.facebook.presto.SystemSessionProperties.CHECK_ACCESS_CONTROL_WITH_SUBFIELDS; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static org.testng.Assert.assertEquals; @@ -214,9 +217,12 @@ private void assertTableColumns(@Language("SQL") String query, Map { - Analyzer analyzer = createAnalyzer(s, metadata, WarningCollector.NOOP, query); + Analyzer analyzer = createAnalyzer(s, metadata, WarningCollector.NOOP, Optional.empty(), query); Statement statement = SQL_PARSER.createStatement(query); - Analysis analysis = analyzer.analyze(statement); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissions(accessControlReferences, analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + assertEquals( analysis.getAccessControlReferences().getTableColumnAndSubfieldReferencesForAccessControl() .values().stream().findFirst().get().entrySet().stream() diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 257dc1ff5c148..0c7010f0ef9a2 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.LeftJoinArrayContainsToInnerJoinStrategy; +import com.facebook.presto.sql.analyzer.FeaturesConfig.LocalExchangeParentPreferenceStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartitioningPrecisionStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PushDownFilterThroughCrossJoinStrategy; @@ -135,6 +136,11 @@ public void testDefaults() .setExchangeChecksumEnabled(false) .setEnableIntermediateAggregations(false) .setPushAggregationThroughJoin(true) + .setPushPartialAggregationThroughJoin(false) + .setPushSemiJoinThroughUnion(false) + .setSimplifyCoalesceOverJoinKeys(false) + .setPushdownThroughUnnest(false) + .setSimplifyAggregationsOverConstant(false) .setForceSingleNodeOutput(true) .setPagesIndexEagerCompactionEnabled(false) .setFilterAndProjectMinOutputPageSize(new DataSize(500, KILOBYTE)) @@ -146,7 +152,9 @@ public void testDefaults() .setPartialAggregationByteReductionThreshold(0.5) .setAdaptivePartialAggregationEnabled(false) .setAdaptivePartialAggregationRowsReductionRatioThreshold(0.8) + .setLocalExchangeParentPreferenceStrategy(LocalExchangeParentPreferenceStrategy.ALWAYS) .setOptimizeTopNRowNumber(true) + .setOptimizeTopNRank(false) .setOptimizeCaseExpressionPredicate(false) .setDistributedSortEnabled(true) .setMaxGroupingSets(2048) @@ -271,7 +279,9 @@ public void testDefaults() .setExcludeInvalidWorkerSessionProperties(false) .setAddExchangeBelowPartialAggregationOverGroupId(false) .setAddDistinctBelowSemiJoinBuild(false) + .setTryFunctionCatchableErrors("") .setPushdownSubfieldForMapFunctions(true) + .setPushdownSubfieldForCardinality(false) .setUtilizeUniquePropertyInQueryPlanning(true) .setExpressionOptimizerUsedInRowExpressionRewrite("") .setInnerJoinPushdownEnabled(false) @@ -283,7 +293,9 @@ public void testDefaults() .setTableScanShuffleParallelismThreshold(0.1) .setTableScanShuffleStrategy(FeaturesConfig.ShuffleForTableScanStrategy.DISABLED) .setSkipPushdownThroughExchangeForRemoteProjection(false) - .setUseConnectorProvidedSerializationCodecs(false)); + .setUseConnectorProvidedSerializationCodecs(false) + .setRemoteFunctionNamesForFixedParallelism("") + .setRemoteFunctionFixedParallelismTaskCount(10)); } @Test @@ -349,7 +361,13 @@ public void testExplicitPropertyMappings() .put("optimizer.retry-query-with-history-based-optimization", "true") .put("optimizer.treat-low-confidence-zero-estimation-as-unknown", "true") .put("optimizer.push-aggregation-through-join", "false") + .put("optimizer.push-partial-aggregation-through-join", "true") + .put("optimizer.push-semi-join-through-union", "true") + .put("optimizer.simplify-coalesce-over-join-keys", "true") + .put("optimizer.pushdown-through-unnest", "true") + .put("optimizer.simplify-aggregations-over-constant", "true") .put("optimizer.aggregation-partition-merging", "top_down") + .put("optimizer.local-exchange-parent-preference-strategy", "automatic") .put("experimental.spill-enabled", "true") .put("experimental.join-spill-enabled", "false") .put("experimental.spiller-spill-path", "/tmp/custom/spill/path1,/tmp/custom/spill/path2") @@ -377,6 +395,7 @@ public void testExplicitPropertyMappings() .put("experimental.adaptive-partial-aggregation", "true") .put("experimental.adaptive-partial-aggregation-rows-reduction-ratio-threshold", "0.9") .put("optimizer.optimize-top-n-row-number", "false") + .put("optimizer.optimize-top-n-rank", "true") .put("optimizer.optimize-case-expression-predicate", "true") .put("distributed-sort", "false") .put("analyzer.max-grouping-sets", "2047") @@ -504,7 +523,9 @@ public void testExplicitPropertyMappings() .put("expression-optimizer-name", "custom") .put("exclude-invalid-worker-session-properties", "true") .put("optimizer.add-distinct-below-semi-join-build", "true") + .put("try-function-catchable-errors", "GENERIC_INTERNAL_ERROR,INVALID_ARGUMENTS") .put("optimizer.pushdown-subfield-for-map-functions", "false") + .put("optimizer.pushdown-subfield-for-cardinality", "true") .put("optimizer.utilize-unique-property-in-query-planning", "false") .put("optimizer.expression-optimizer-used-in-expression-rewrite", "custom") .put("optimizer.add-exchange-below-partial-aggregation-over-group-id", "true") @@ -513,6 +534,8 @@ public void testExplicitPropertyMappings() .put("optimizer.table-scan-shuffle-strategy", "ALWAYS_ENABLED") .put("optimizer.skip-pushdown-through-exchange-for-remote-projection", "true") .put("use-connector-provided-serialization-codecs", "true") + .put("optimizer.remote-function-names-for-fixed-parallelism", "remote_.*") + .put("optimizer.remote-function-fixed-parallelism-task-count", "100") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -575,6 +598,10 @@ public void testExplicitPropertyMappings() .setTreatLowConfidenceZeroEstimationAsUnknownEnabled(true) .setAggregationPartitioningMergingStrategy(TOP_DOWN) .setPushAggregationThroughJoin(false) + .setPushSemiJoinThroughUnion(true) + .setSimplifyCoalesceOverJoinKeys(true) + .setPushdownThroughUnnest(true) + .setSimplifyAggregationsOverConstant(true) .setSpillEnabled(true) .setJoinSpillingEnabled(false) .setSpillerSpillPaths("/tmp/custom/spill/path1,/tmp/custom/spill/path2") @@ -601,7 +628,9 @@ public void testExplicitPropertyMappings() .setPartialAggregationByteReductionThreshold(0.8) .setAdaptivePartialAggregationEnabled(true) .setAdaptivePartialAggregationRowsReductionRatioThreshold(0.9) + .setLocalExchangeParentPreferenceStrategy(LocalExchangeParentPreferenceStrategy.AUTOMATIC) .setOptimizeTopNRowNumber(false) + .setOptimizeTopNRank(true) .setOptimizeCaseExpressionPredicate(true) .setDistributedSortEnabled(false) .setMaxGroupingSets(2047) @@ -728,7 +757,9 @@ public void testExplicitPropertyMappings() .setExcludeInvalidWorkerSessionProperties(true) .setAddExchangeBelowPartialAggregationOverGroupId(true) .setAddDistinctBelowSemiJoinBuild(true) + .setTryFunctionCatchableErrors("GENERIC_INTERNAL_ERROR,INVALID_ARGUMENTS") .setPushdownSubfieldForMapFunctions(false) + .setPushdownSubfieldForCardinality(true) .setUtilizeUniquePropertyInQueryPlanning(false) .setExpressionOptimizerUsedInRowExpressionRewrite("custom") .setInEqualityJoinPushdownEnabled(true) @@ -740,7 +771,10 @@ public void testExplicitPropertyMappings() .setTableScanShuffleParallelismThreshold(0.3) .setTableScanShuffleStrategy(FeaturesConfig.ShuffleForTableScanStrategy.ALWAYS_ENABLED) .setSkipPushdownThroughExchangeForRemoteProjection(true) - .setUseConnectorProvidedSerializationCodecs(true); + .setUseConnectorProvidedSerializationCodecs(true) + .setRemoteFunctionNamesForFixedParallelism("remote_.*") + .setPushPartialAggregationThroughJoin(true) + .setRemoteFunctionFixedParallelismTaskCount(100); assertFullMapping(properties, expected); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java index 108ebb4da54e3..bf70110ea0eb5 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestMaterializedViewQueryOptimizer.java @@ -450,6 +450,42 @@ public void testWithTableAlias() assertOptimizedQuery(baseQuerySqlWithTablePrefix, expectedRewrittenSql, originalViewSqlWithTablePrefix, BASE_TABLE_1, VIEW_1); } + @Test + public void testWithSchemaQualifiedTableName() + { + String schemaQualifiedTable = SESSION_SCHEMA + "." + BASE_TABLE_1; + + String originalViewSql = format("SELECT a, b FROM %s", BASE_TABLE_1); + String baseQuerySql = format("SELECT a, b FROM %s", schemaQualifiedTable); + String expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b FROM %s", schemaQualifiedTable); + baseQuerySql = format("SELECT a, b FROM %s", BASE_TABLE_1); + expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b FROM %s", schemaQualifiedTable); + baseQuerySql = format("SELECT a, b FROM %s", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT a, b FROM %s", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT a, b, c FROM %s", BASE_TABLE_1); + baseQuerySql = format("SELECT a, b FROM %s WHERE c > 10", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT a, b FROM %s WHERE c > 10", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + + originalViewSql = format("SELECT SUM(a) as sum_a, b FROM %s GROUP BY b", BASE_TABLE_1); + baseQuerySql = format("SELECT SUM(a), b FROM %s GROUP BY b", schemaQualifiedTable); + expectedRewrittenSql = format("SELECT SUM(sum_a), b FROM %s GROUP BY b", VIEW_1); + + assertOptimizedQuery(baseQuerySql, expectedRewrittenSql, originalViewSql, BASE_TABLE_1, VIEW_1); + } + @Test public void testAggregationWithTableAlias() { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestUtilizedColumnsAnalyzer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestUtilizedColumnsAnalyzer.java index 0bff1b7961cd7..5d64d7eb66d42 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestUtilizedColumnsAnalyzer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestUtilizedColumnsAnalyzer.java @@ -16,6 +16,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.AccessControlInfo; +import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.sql.tree.Statement; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -23,10 +24,12 @@ import org.testng.annotations.Test; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static org.testng.Assert.assertEquals; @Test(singleThreaded = true) @@ -656,9 +659,12 @@ private void assertUtilizedTableColumns(@Language("SQL") String query, Map { - Analyzer analyzer = createAnalyzer(session, metadata, WarningCollector.NOOP, query); + Analyzer analyzer = createAnalyzer(session, metadata, WarningCollector.NOOP, Optional.empty(), query); Statement statement = SQL_PARSER.createStatement(query); - Analysis analysis = analyzer.analyze(statement); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissions(accessControlReferences, analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + assertEquals(analysis.getUtilizedTableColumnReferences().entrySet().stream().collect(Collectors.toMap(entry -> extractAccessControlInfo(entry.getKey()), Map.Entry::getValue)), expected); }); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestViewDefinitionCollector.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestViewDefinitionCollector.java new file mode 100644 index 0000000000000..a0407e9d61f46 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestViewDefinitionCollector.java @@ -0,0 +1,221 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; +import com.facebook.presto.sql.tree.Statement; +import com.google.common.collect.ImmutableMap; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; +import static org.testng.Assert.assertEquals; + +@Test(singleThreaded = true) +public class TestViewDefinitionCollector + extends AbstractAnalyzerTest +{ + public void testSelectLeftJoinViews() + { + @Language("SQL") String query = "SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testCreateViewWithNestedViews() + { + @Language("SQL") String query = "CREATE VIEW top_level_view1 AS SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testCreateTableAsSelectWithViews() + { + @Language("SQL") String query = "CREATE TABLE top_level_view1 AS SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainWithViews() + { + @Language("SQL") String query = "EXPLAIN SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainTypeIoWithViews() + { + @Language("SQL") String query = "EXPLAIN (TYPE IO) SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainTypeValidateWithViews() + { + @Language("SQL") String query = "EXPLAIN (TYPE VALIDATE) SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainAnalyzeWithViews() + { + @Language("SQL") String query = "EXPLAIN ANALYZE SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainExplainWithViews() + { + @Language("SQL") String query = "EXPLAIN EXPLAIN SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainExplainTypeValidateWithViews() + { + @Language("SQL") String query = "EXPLAIN EXPLAIN (TYPE VALIDATE) SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainTypeValidateExplainWithViews() + { + @Language("SQL") String query = "EXPLAIN (TYPE VALIDATE) EXPLAIN SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainTypeValidateExplainTypeValidateWithViews() + { + @Language("SQL") String query = "EXPLAIN (TYPE VALIDATE) EXPLAIN (TYPE VALIDATE) SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainAnalyzeExplainWithViews() + { + @Language("SQL") String query = "EXPLAIN ANALYZE EXPLAIN SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainAnalyzeExplainAnalyzeWithViews() + { + @Language("SQL") String query = "EXPLAIN ANALYZE EXPLAIN ANALYZE SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainExplainAnalyzeWithViews() + { + @Language("SQL") String query = "EXPLAIN EXPLAIN ANALYZE SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainAnalyzeExplainTypeValidateWithViews() + { + @Language("SQL") String query = "EXPLAIN ANALYZE EXPLAIN (TYPE VALIDATE) SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + public void testExplainTypeValidateExplainAnalyzeWithViews() + { + @Language("SQL") String query = "EXPLAIN (TYPE VALIDATE) EXPLAIN ANALYZE SELECT view_definer1.a, view_definer1.c, view_invoker2.y FROM view_definer1 left join view_invoker2 on view_invoker2.y = view_definer1.c"; + + assertViewDefinitions(query, ImmutableMap.of( + "tpch.s1.view_invoker2", "select x, y, z from t13", + "tpch.s1.view_definer1", "select a,b,c from t1" + ), ImmutableMap.of()); + } + + private void assertViewDefinitions(@Language("SQL") String query, Map expectedViewDefinitions, Map expectedMaterializedViewDefinitions) + { + transaction(transactionManager, accessControl) + .singleStatement() + .readUncommitted() + .readOnly() + .execute(CLIENT_SESSION, session -> { + Analyzer analyzer = createAnalyzer(session, metadata, WarningCollector.NOOP, Optional.of(createTestingQueryExplainer(session, accessControl, metadata)), query); + Statement statement = SQL_PARSER.createStatement(query); + Analysis analysis = analyzer.analyzeSemantic(statement, false); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissions(accessControlReferences, analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + + Map viewDefinitionsMap = analysis.getViewDefinitionReferences().getViewDefinitions().entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey().toString(), + entry -> entry.getValue().getOriginalSql())); + Map materializedDefinitionsMap = analysis.getViewDefinitionReferences().getMaterializedViewDefinitions().entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey().toString(), + entry -> entry.getValue().getOriginalSql())); + + assertEquals(viewDefinitionsMap, expectedViewDefinitions); + assertEquals(materializedDefinitionsMap, expectedMaterializedViewDefinitions); + }); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 1ff85caca45a9..1f10a5a6c7570 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -70,6 +70,7 @@ import com.facebook.presto.util.MorePredicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -90,6 +91,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.SystemSessionProperties.LEAF_NODE_LIMIT_ENABLED; +import static com.facebook.presto.SystemSessionProperties.LOCAL_EXCHANGE_PARENT_PREFERENCE_STRATEGY; import static com.facebook.presto.SystemSessionProperties.MAX_LEAF_NODES_IN_PLAN; import static com.facebook.presto.SystemSessionProperties.NATIVE_EXECUTION_ENABLED; import static com.facebook.presto.SystemSessionProperties.OFFSET_CLAUSE_ENABLED; @@ -98,6 +100,7 @@ import static com.facebook.presto.SystemSessionProperties.PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID; import static com.facebook.presto.SystemSessionProperties.REMOVE_CROSS_JOIN_WITH_CONSTANT_SINGLE_ROW_INPUT; import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_PLAN_WITH_EMPTY_INPUT; +import static com.facebook.presto.SystemSessionProperties.SINGLE_NODE_EXECUTION_ENABLED; import static com.facebook.presto.SystemSessionProperties.TASK_CONCURRENCY; import static com.facebook.presto.SystemSessionProperties.getMaxLeafNodesInPlan; import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_LAST; @@ -1813,6 +1816,30 @@ public void testRewriteExcludeColumnsFunctionToProjection() output(tableScan("orders"))); } + @Test + public void testInfinityAndNaNExpression() + { + assertPlan("select nan(), infinity(), cast (nan() as real), cast(infinity() as real)", + anyTree(strictProject( + ImmutableMap.of( + "col_1", expression("double 'NaN'"), + "col_2", expression("double 'Infinity'"), + "col_3", expression("real 'NaN'"), + "col_4", expression("real 'Infinity'")), + values()))); + } + + @Test + public void testBigintAndIntegerExpression() + { + assertPlan("select cast(123 as integer), cast(123 as bigint)", + anyTree(strictProject( + ImmutableMap.of( + "col_4", expression("bigint '123'"), + "col_3", expression("integer '123'")), + values()))); + } + private Session noJoinReordering() { return Session.builder(this.getQueryRunner().getDefaultSession()) @@ -2137,4 +2164,86 @@ public void testSubselectQualifiedObjectNameContainsDot() String query = "SELECT min((SELECT totalprice FROM orders WHERE orderstatus = \"Outer.Table\".\"orderstatus\")) as min FROM orders AS \"Outer.Table\""; assertPlanSucceeded(query, this.getQueryRunner().getDefaultSession()); } + + @Test + public void testLocalExchangeWithParentPreference() + { + // Query with two nested aggregations on the orders table: + // First aggregation: GROUP BY orderstatus, orderpriority (cardinality = 3 * 5 = 15) + // Second aggregation: GROUP BY orderstatus (cardinality = 3) + String query = "SELECT sum(cnt) FROM (SELECT orderstatus, orderpriority, count(*) cnt FROM orders GROUP BY orderstatus, orderpriority) GROUP BY orderstatus"; + + // Test ALWAYS strategy: always use parent preferences regardless of concurrency. + // First aggregation partitions by orderstatus (parent preference), second aggregation becomes SINGLE. + assertLocalExchangeWithParentPreference(query, "ALWAYS", "4", true); + assertLocalExchangeWithParentPreference(query, "ALWAYS", "2", true); + + // Test NEVER strategy: never use parent preferences regardless of concurrency. + // Both aggregations partition by their own grouping keys. + assertLocalExchangeWithParentPreference(query, "NEVER", "4", false); + assertLocalExchangeWithParentPreference(query, "NEVER", "2", false); + + // Test AUTOMATIC strategy: cost-based decision. + // When task concurrency (4) > parent cardinality (3), don't use parent preferences. + assertLocalExchangeWithParentPreference(query, "AUTOMATIC", "4", false); + // When task concurrency (2) <= parent cardinality (3), use parent preferences. + assertLocalExchangeWithParentPreference(query, "AUTOMATIC", "2", true); + } + + private void assertLocalExchangeWithParentPreference(String query, String strategy, String taskConcurrency, boolean expectParentPreference) + { + Session session = Session.builder(this.getQueryRunner().getDefaultSession()) + .setSystemProperty(SINGLE_NODE_EXECUTION_ENABLED, "true") + .setSystemProperty(TASK_CONCURRENCY, taskConcurrency) + .setSystemProperty(LOCAL_EXCHANGE_PARENT_PREFERENCE_STRATEGY, strategy) + .build(); + + if (expectParentPreference) { + // When using parent preferences, first aggregation partitions by orderstatus (parent preference), + // and there is no local exchange at the second aggregation which becomes a SINGLE aggregation + assertDistributedPlan( + query, + session, + anyTree( + project( + aggregation( + ImmutableMap.of("outer_sum", functionCall("sum", ImmutableList.of("final_count"))), + SINGLE, + project( + aggregation( + ImmutableMap.of("final_count", functionCall("count", ImmutableList.of("partial_count"))), + FINAL, + exchange(LOCAL, REPARTITION, ImmutableList.of(), ImmutableSet.of("orderstatus"), + project( + aggregation( + ImmutableMap.of("partial_count", functionCall("count", ImmutableList.of())), + PARTIAL, + anyTree( + tableScan("orders", ImmutableMap.of("orderstatus", "orderstatus", "orderpriority", "orderpriority")))))))))))); + } + else { + // When not using parent preferences, local exchanges partition by each aggregation's own grouping keys + assertDistributedPlan( + query, + session, + anyTree( + aggregation( + ImmutableMap.of("final_sum", functionCall("sum", ImmutableList.of("partial_sum"))), + FINAL, + exchange(LOCAL, REPARTITION, ImmutableList.of(), ImmutableSet.of("orderstatus"), + aggregation( + ImmutableMap.of("partial_sum", functionCall("sum", ImmutableList.of("final_count"))), + PARTIAL, + project( + aggregation( + ImmutableMap.of("final_count", functionCall("count", ImmutableList.of("partial_count"))), + FINAL, + exchange(LOCAL, REPARTITION, ImmutableList.of(), ImmutableSet.of("orderstatus", "orderpriority"), + aggregation( + ImmutableMap.of("partial_count", functionCall("count", ImmutableList.of())), + PARTIAL, + anyTree( + tableScan("orders", ImmutableMap.of("orderstatus", "orderstatus", "orderpriority", "orderpriority")))))))))))); + } + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java index 8901ea70d9d31..5800b57dab0e9 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/RowExpressionVerifier.java @@ -424,6 +424,9 @@ private static OperatorType getOperatorType(ArithmeticBinaryExpression.Operator @Override protected Boolean visitGenericLiteral(GenericLiteral expected, RowExpression actual) { + if (!expected.getType().equalsIgnoreCase(actual.getType().getTypeSignature().getBase())) { + return false; + } return compareLiteral(expected, actual); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/TopNRowNumberMatcher.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/TopNRowNumberMatcher.java index 590625570cbd7..728940e930ce2 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/TopNRowNumberMatcher.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/assertions/TopNRowNumberMatcher.java @@ -19,9 +19,9 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.DataOrganizationSpecification; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import java.util.List; import java.util.Map; diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java index 3759e5018adef..44a7aa7ffe9fd 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestLogicalPropertyPropagation.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.constraints.PrimaryKeyConstraint; import com.facebook.presto.spi.constraints.TableConstraint; import com.facebook.presto.spi.constraints.UniqueConstraint; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; @@ -71,6 +72,7 @@ import static com.facebook.presto.sql.relational.Expressions.constant; import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Collections.emptyList; +import static org.testng.Assert.assertTrue; public class TestLogicalPropertyPropagation extends BaseRuleTest @@ -200,6 +202,58 @@ void testValuesNodeLogicalProperties() .matches(expectedLogicalProperties); } + @Test + public void testKeyNormalization() + { + tester().assertThat(new NoOpRule(), logicalPropertiesProvider) + .on(p -> { + TableScanNode customerTableScan = p.tableScan( + customerTableHandle, + ImmutableList.of(customerCustKeyVariable), + ImmutableMap.of(customerCustKeyVariable, customerCustKeyColumn), + TupleDomain.none(), + TupleDomain.none(), + tester().getTableConstraints(customerTableHandle)); + + TableScanNode ordersTableScan = p.tableScan( + ordersTableHandle, + ImmutableList.of(ordersCustKeyVariable), + ImmutableMap.of(ordersCustKeyVariable, ordersCustKeyColumn), + TupleDomain.none(), + TupleDomain.none(), + tester().getTableConstraints(ordersTableHandle)); + + TableScanNode lineitemTableScan = p.tableScan( + lineitemTableHandle, + ImmutableList.of(lineitemOrderkeyVariable), + ImmutableMap.of(lineitemOrderkeyVariable, lineitemOrderkeyColumn), + TupleDomain.none(), + TupleDomain.none(), + tester().getTableConstraints(lineitemTableHandle)); + + JoinNode ordersCustomerJoin = p.join(JoinType.INNER, + ordersTableScan, + customerTableScan, + new EquiJoinClause(ordersCustKeyVariable, customerCustKeyVariable)); + + AggregationNode aggregation = p.aggregation(builder -> builder + .singleGroupingSet(ordersCustKeyVariable) + .source(p.join(JoinType.INNER, + ordersCustomerJoin, + lineitemTableScan, + new EquiJoinClause(customerCustKeyVariable, lineitemOrderkeyVariable)))); + return aggregation; + }).assertLogicalProperties(groupProperties -> { + // SINGLE aggregation on ordersCustKeyVariable => this is a key + assertTrue(groupProperties.isDistinct(ImmutableSet.of(ordersCustKeyVariable))); + // Since ordersCustKeyVariable == customerCustKeyVariable, customerCustKeyVariable is a key as well + // This is derived through the equivalence classes + assertTrue(groupProperties.isDistinct(ImmutableSet.of(customerCustKeyVariable))); + // Same holds true for lineitemOrderkeyVariable + assertTrue(groupProperties.isDistinct(ImmutableSet.of(lineitemOrderkeyVariable))); + }); + } + @Test public void testTableScanNodeLogicalProperties() { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushSemiJoinThroughUnion.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushSemiJoinThroughUnion.java new file mode 100644 index 0000000000000..00eee9d8f1f9a --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushSemiJoinThroughUnion.java @@ -0,0 +1,261 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.PUSH_SEMI_JOIN_THROUGH_UNION; +import static com.facebook.presto.common.function.OperatorType.MULTIPLY; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.semiJoin; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.union; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; + +public class TestPushSemiJoinThroughUnion + extends BaseRuleTest +{ + @Test + public void testDoesNotFireWhenSourceIsNotUnion() + { + tester().assertThat(new PushSemiJoinThroughUnion()) + .setSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, "true") + .on(p -> { + VariableReferenceExpression sourceJoinVar = p.variable("sourceJoinVar"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + sourceJoinVar, + filterJoinVar, + semiJoinOutput, + Optional.empty(), + Optional.empty(), + p.values(sourceJoinVar), + p.values(filterJoinVar)); + }) + .doesNotFire(); + } + + @Test + public void testPushThroughTwoBranchUnion() + { + tester().assertThat(new PushSemiJoinThroughUnion()) + .setSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, "true") + .on(p -> { + VariableReferenceExpression a = p.variable("a"); + VariableReferenceExpression b = p.variable("b"); + VariableReferenceExpression c = p.variable("c"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + c, + filterJoinVar, + semiJoinOutput, + Optional.empty(), + Optional.empty(), + p.union( + ImmutableListMultimap.builder() + .put(c, a) + .put(c, b) + .build(), + ImmutableList.of( + p.values(a), + p.values(b))), + p.values(filterJoinVar)); + }) + .matches( + union( + semiJoin("a", "filterJoinVar", "semiJoinOutput_0", + values("a"), + values("filterJoinVar")), + semiJoin("b", "filterJoinVar", "semiJoinOutput_1", + values("b"), + values("filterJoinVar")))); + } + + @Test + public void testPushThroughThreeBranchUnion() + { + tester().assertThat(new PushSemiJoinThroughUnion()) + .setSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, "true") + .on(p -> { + VariableReferenceExpression a = p.variable("a"); + VariableReferenceExpression b = p.variable("b"); + VariableReferenceExpression d = p.variable("d"); + VariableReferenceExpression c = p.variable("c"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + c, + filterJoinVar, + semiJoinOutput, + Optional.empty(), + Optional.empty(), + p.union( + ImmutableListMultimap.builder() + .put(c, a) + .put(c, b) + .put(c, d) + .build(), + ImmutableList.of( + p.values(a), + p.values(b), + p.values(d))), + p.values(filterJoinVar)); + }) + .matches( + union( + semiJoin("a", "filterJoinVar", "semiJoinOutput_0", + values("a"), + values("filterJoinVar")), + semiJoin("b", "filterJoinVar", "semiJoinOutput_1", + values("b"), + values("filterJoinVar")), + semiJoin("d", "filterJoinVar", "semiJoinOutput_2", + values("d"), + values("filterJoinVar")))); + } + + @Test + public void testPushThroughProjectOverUnion() + { + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushSemiJoinThroughUnion()) + .setSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, "true") + .on(p -> { + VariableReferenceExpression a = p.variable("a"); + VariableReferenceExpression b = p.variable("b"); + VariableReferenceExpression c = p.variable("c"); + VariableReferenceExpression cTimes3 = p.variable("c_times_3"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + cTimes3, + filterJoinVar, + semiJoinOutput, + Optional.empty(), + Optional.empty(), + p.project( + assignment( + cTimes3, + call("c * 3", functionResolution.arithmeticFunction(MULTIPLY, BIGINT, BIGINT), BIGINT, c, constant(3L, BIGINT))), + p.union( + ImmutableListMultimap.builder() + .put(c, a) + .put(c, b) + .build(), + ImmutableList.of( + p.values(a), + p.values(b)))), + p.values(filterJoinVar)); + }) + .matches( + union( + semiJoin( + project( + ImmutableMap.of("a_times_3", expression("a * 3")), + values("a")), + values("filterJoinVar")), + semiJoin( + project( + ImmutableMap.of("b_times_3", expression("b * 3")), + values("b")), + values("filterJoinVar")))); + } + + @Test + public void testPushThroughUnionWithHashVariables() + { + tester().assertThat(new PushSemiJoinThroughUnion()) + .setSystemProperty(PUSH_SEMI_JOIN_THROUGH_UNION, "true") + .on(p -> { + VariableReferenceExpression a = p.variable("a"); + VariableReferenceExpression b = p.variable("b"); + VariableReferenceExpression c = p.variable("c"); + VariableReferenceExpression aHash = p.variable("aHash"); + VariableReferenceExpression bHash = p.variable("bHash"); + VariableReferenceExpression cHash = p.variable("cHash"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression filterHash = p.variable("filterHash"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + c, + filterJoinVar, + semiJoinOutput, + Optional.of(cHash), + Optional.of(filterHash), + p.union( + ImmutableListMultimap.builder() + .put(c, a) + .put(c, b) + .put(cHash, aHash) + .put(cHash, bHash) + .build(), + ImmutableList.of( + p.values(a, aHash), + p.values(b, bHash))), + p.values(filterJoinVar, filterHash)); + }) + .matches( + union( + semiJoin("a", "filterJoinVar", "semiJoinOutput_0", + values("a", "aHash"), + values("filterJoinVar", "filterHash")), + semiJoin("b", "filterJoinVar", "semiJoinOutput_1", + values("b", "bHash"), + values("filterJoinVar", "filterHash")))); + } + + @Test + public void testDoesNotFireWhenDisabled() + { + tester().assertThat(new PushSemiJoinThroughUnion()) + .on(p -> { + VariableReferenceExpression a = p.variable("a"); + VariableReferenceExpression b = p.variable("b"); + VariableReferenceExpression c = p.variable("c"); + VariableReferenceExpression filterJoinVar = p.variable("filterJoinVar"); + VariableReferenceExpression semiJoinOutput = p.variable("semiJoinOutput", BOOLEAN); + return p.semiJoin( + c, + filterJoinVar, + semiJoinOutput, + Optional.empty(), + Optional.empty(), + p.union( + ImmutableListMultimap.builder() + .put(c, a) + .put(c, b) + .build(), + ImmutableList.of( + p.values(a), + p.values(b))), + p.values(filterJoinVar)); + }) + .doesNotFire(); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushdownThroughUnnest.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushdownThroughUnnest.java new file mode 100644 index 0000000000000..37fb2feda4bf5 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPushdownThroughUnnest.java @@ -0,0 +1,404 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_THROUGH_UNNEST; +import static com.facebook.presto.common.function.OperatorType.ADD; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.unnest; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; +import static com.facebook.presto.sql.relational.Expressions.call; +import static com.facebook.presto.sql.relational.Expressions.constant; + +public class TestPushdownThroughUnnest + extends BaseRuleTest +{ + @Test + public void testDoesNotFireWithNoUnnestChild() + { + // Project over values (not unnest) should not fire + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> + p.project( + assignment(p.variable("x"), constant(3L, BIGINT)), + p.values(p.variable("a")))) + .doesNotFire(); + } + + @Test + public void testDoesNotFireWhenAllAssignmentsDependOnUnnest() + { + // When all projected expressions depend on unnest output, nothing to push + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + // y + 1 depends on unnest output y + assignment(p.variable("y_plus_1", BIGINT), + call("y + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, y, constant(1L, BIGINT))), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty())); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireWhenOnlyIdentityAssignments() + { + // When all non-unnest-dependent assignments are just identity (pass-through), don't fire + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment(x, x, y, y), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty())); + }) + .doesNotFire(); + } + + @Test + public void testPushesNonDependentExpression() + { + // select x+1, y from t cross join unnest(a) t(y) + // x+1 should be pushed below the unnest + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment( + p.variable("x_plus_1", BIGINT), + call("x + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, x, constant(1L, BIGINT)), + y, y), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty())); + }) + .matches( + project( + ImmutableMap.of("x_plus_1", expression("x_plus_1"), "y", expression("y")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + project( + ImmutableMap.of("x", expression("x"), "a", expression("a"), "x_plus_1", expression("x + 1")), + values("x", "a"))))); + } + + @Test + public void testPushesConstantExpression() + { + // Constant expressions don't depend on unnest outputs and should be pushed down + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment( + p.variable("const", BIGINT), constant(42L, BIGINT), + y, y), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty())); + }) + .matches( + project( + ImmutableMap.of("const", expression("const"), "y", expression("y")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + project( + ImmutableMap.of("x", expression("x"), "a", expression("a"), "const", expression("42")), + values("x", "a"))))); + } + + @Test + public void testPushesFilterConjunctBelowUnnest() + { + // Project -> Filter(x > 10) -> Unnest + // x > 10 doesn't depend on unnest output y, so it should be pushed below + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment(x, x, y, y), + p.filter( + p.rowExpression("x > BIGINT '10'"), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty()))); + }) + .matches( + project( + ImmutableMap.of("x", expression("x"), "y", expression("y")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + filter("x > BIGINT '10'", + values("x", "a"))))); + } + + @Test + public void testDoesNotFireWhenFilterDependsOnUnnest() + { + // Project -> Filter(y > 0) -> Unnest + // y > 0 depends on unnest output y, and project has only identity assignments + // Nothing is pushable + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment(x, x, y, y), + p.filter( + p.rowExpression("y > BIGINT '0'"), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty()))); + }) + .doesNotFire(); + } + + @Test + public void testPushesMixedFilterConjuncts() + { + // Project -> Filter(x > 10 AND y > 0) -> Unnest + // x > 10 can be pushed below, y > 0 must remain above + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment(x, x, y, y), + p.filter( + p.rowExpression("x > BIGINT '10' AND y > BIGINT '0'"), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty()))); + }) + .matches( + project( + ImmutableMap.of("x", expression("x"), "y", expression("y")), + filter("y > BIGINT '0'", + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + filter("x > BIGINT '10'", + values("x", "a")))))); + } + + @Test + public void testPushesBothProjectionAndFilter() + { + // Project(x+1, y) -> Filter(x > 10) -> Unnest + // Both x+1 projection and x > 10 filter can be pushed below + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + return p.project( + assignment( + p.variable("x_plus_1", BIGINT), + call("x + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, x, constant(1L, BIGINT)), + y, y), + p.filter( + p.rowExpression("x > BIGINT '10'"), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.empty()))); + }) + .matches( + project( + ImmutableMap.of("x_plus_1", expression("x_plus_1"), "y", expression("y")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + project( + ImmutableMap.of("x", expression("x"), "a", expression("a"), "x_plus_1", expression("x + 1")), + filter("x > BIGINT '10'", + values("x", "a")))))); + } + + @Test + public void testDoesNotPushProjectionReferencingOrdinality() + { + // Ordinality is an unnest-produced variable, so expressions referencing it should not be pushed + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + VariableReferenceExpression ord = p.variable("ord", BIGINT); + return p.project( + assignment( + p.variable("ord_plus_1", BIGINT), + call("ord + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, ord, constant(1L, BIGINT))), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.of(ord))); + }) + .doesNotFire(); + } + + @Test + public void testPushesWithOrdinalityPresent() + { + // x+1 doesn't depend on unnest output y or ordinality ord, so it should be pushed + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y = p.variable("y", BIGINT); + VariableReferenceExpression ord = p.variable("ord", BIGINT); + return p.project( + Assignments.builder() + .put(p.variable("x_plus_1", BIGINT), + call("x + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, x, constant(1L, BIGINT))) + .put(y, y) + .put(ord, ord) + .build(), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y)), + Optional.of(ord))); + }) + .matches( + project( + ImmutableMap.of("x_plus_1", expression("x_plus_1"), "y", expression("y")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y")), + project( + ImmutableMap.of("x", expression("x"), "a", expression("a"), "x_plus_1", expression("x + 1")), + values("x", "a"))))); + } + + @Test + public void testDoesNotPushWithMultipleUnnestOutputs() + { + // y1 + y2 depends on unnest outputs, should not be pushed + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y1 = p.variable("y1", BIGINT); + VariableReferenceExpression y2 = p.variable("y2", BIGINT); + return p.project( + assignment( + p.variable("y1_plus_y2", BIGINT), + call("y1 + y2", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, y1, y2)), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y1, y2)), + Optional.empty())); + }) + .doesNotFire(); + } + + @Test + public void testPushesWithMultipleUnnestOutputs() + { + // x+1 depends only on replicated x, should be pushed even with multiple unnest outputs + FunctionResolution functionResolution = new FunctionResolution(tester().getMetadata().getFunctionAndTypeManager().getFunctionAndTypeResolver()); + tester().assertThat(new PushdownThroughUnnest(tester().getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression a = p.variable("a", BIGINT); + VariableReferenceExpression y1 = p.variable("y1", BIGINT); + VariableReferenceExpression y2 = p.variable("y2", BIGINT); + return p.project( + Assignments.builder() + .put(p.variable("x_plus_1", BIGINT), + call("x + 1", functionResolution.arithmeticFunction(ADD, BIGINT, BIGINT), BIGINT, x, constant(1L, BIGINT))) + .put(y1, y1) + .put(y2, y2) + .build(), + p.unnest( + p.values(x, a), + ImmutableList.of(x), + ImmutableMap.of(a, ImmutableList.of(y1, y2)), + Optional.empty())); + }) + .matches( + project( + ImmutableMap.of("x_plus_1", expression("x_plus_1"), "y1", expression("y1"), "y2", expression("y2")), + unnest( + ImmutableMap.of("a", ImmutableList.of("y1", "y2")), + project( + ImmutableMap.of("x", expression("x"), "a", expression("a"), "x_plus_1", expression("x + 1")), + values("x", "a"))))); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveCrossJoinWithConstantInput.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveCrossJoinWithConstantInput.java index 9514346f1b0db..98bcff8ed68cd 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveCrossJoinWithConstantInput.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveCrossJoinWithConstantInput.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.TestingColumnHandle; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -38,6 +39,7 @@ import static com.facebook.presto.common.block.MethodHandleUtil.nativeValueGetter; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.plan.ProjectNode.Locality.UNKNOWN; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; @@ -45,6 +47,8 @@ import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.testing.TestingEnvironment.getOperatorMethodHandle; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; public class TestRemoveCrossJoinWithConstantInput extends BaseRuleTest @@ -296,10 +300,57 @@ public void testOneColumnValuesNodeExpression() p.values(ImmutableList.of(leftKey), ImmutableList.of(ImmutableList.of(constant(1L, BIGINT)), ImmutableList.of(constant(2L, BIGINT)))), p.project( assignment(rightKey2, p.rowExpression("cast(right_k1 as varchar)")), - p.values(ImmutableList.of(rightKey1), ImmutableList.of(ImmutableList.of(constant(1L, BIGINT)))))); + p.values(ImmutableList.of(rightKey1), ImmutableList.of(ImmutableList.of(constant(1L, BIGINT)))))); }) .matches( project(ImmutableMap.of("left_k1", expression("left_k1"), "right_k2", expression("cast(1 as varchar)")), values("left_k1"))); } + + @Test + public void testProjectNodeLocalityIsUnknown() + { + // Test that the generated ProjectNode has UNKNOWN locality, which allows subsequent optimizers + // to determine the optimal locality based on the context (e.g., if the projection involves remote functions) + PlanNode result = tester().assertThat(new RemoveCrossJoinWithConstantInput(getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(REMOVE_CROSS_JOIN_WITH_CONSTANT_SINGLE_ROW_INPUT, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_k1", BIGINT); + p.variable("right_k1", BIGINT); + return p.join(JoinType.INNER, + p.tableScan(ImmutableList.of(leftKey), ImmutableMap.of(leftKey, new TestingColumnHandle("col"))), + p.values(ImmutableList.of(p.variable("right_k1")), ImmutableList.of(ImmutableList.of(constant(1L, BIGINT))))); + }) + .get(); + + assertTrue(result instanceof ProjectNode, "Expected result to be ProjectNode"); + ProjectNode projectNode = (ProjectNode) result; + assertEquals(projectNode.getLocality(), UNKNOWN, "ProjectNode locality should be UNKNOWN to allow subsequent optimizers to set it"); + } + + @Test + public void testProjectNodeLocalityIsUnknownWithFilter() + { + // Test that when there's a join filter, the ProjectNode underneath the FilterNode has UNKNOWN locality + PlanNode result = tester().assertThat(new RemoveCrossJoinWithConstantInput(getMetadata().getFunctionAndTypeManager())) + .setSystemProperty(REMOVE_CROSS_JOIN_WITH_CONSTANT_SINGLE_ROW_INPUT, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_k1", BIGINT); + p.variable("right_k1", BIGINT); + return p.join(JoinType.INNER, + p.tableScan(ImmutableList.of(leftKey), ImmutableMap.of(leftKey, new TestingColumnHandle("col"))), + p.values(ImmutableList.of(p.variable("right_k1")), ImmutableList.of(ImmutableList.of(constant(1L, BIGINT)))), + p.rowExpression("left_k1 + right_k1 > 2")); + }) + .get(); + + assertTrue(result instanceof FilterNode, "Expected result to be FilterNode"); + FilterNode filterNode = (FilterNode) result; + PlanNode source = filterNode.getSource(); + assertTrue(source instanceof ProjectNode, "Expected FilterNode source to be ProjectNode"); + ProjectNode projectNode = (ProjectNode) source; + assertEquals(projectNode.getLocality(), UNKNOWN, "ProjectNode locality should be UNKNOWN"); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteRowExpressions.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteRowExpressions.java index bb29b7774a4ac..3e5fe694744d9 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteRowExpressions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRewriteRowExpressions.java @@ -211,7 +211,7 @@ public void testProjectRuleRewritesConstantArithmetic() .on(p -> { VariableReferenceExpression a = p.variable("a", BIGINT); return p.project( - assignment(a, p.rowExpression("1 + 2")), + assignment(a, p.rowExpression("bigint '1' + 2")), p.values()); }) .matches( diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyAggregationsOverConstant.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyAggregationsOverConstant.java new file mode 100644 index 0000000000000..6897892287e72 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyAggregationsOverConstant.java @@ -0,0 +1,383 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_AGGREGATIONS_OVER_CONSTANT; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; + +public class TestSimplifyAggregationsOverConstant + extends BaseRuleTest +{ + @Test + public void testFoldsMinOverConstant() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 7L))))); + }) + .matches(values(ImmutableMap.of("min_1", 0))); + } + + @Test + public void testFoldsMaxOverConstant() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("max_1", BIGINT), + p.rowExpression("max(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 3L))))); + }) + .matches(values(ImmutableMap.of("max_1", 0))); + } + + @Test + public void testFoldsArbitraryOverConstant() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("arb_1", BIGINT), + p.rowExpression("arbitrary(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 9L))))); + }) + .matches(values(ImmutableMap.of("arb_1", 0))); + } + + @Test + public void testFoldsApproxDistinctOverConstant() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("approx_1", BIGINT), + p.rowExpression("approx_distinct(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 42L))))); + }) + .matches(values(ImmutableMap.of("approx_1", 0))); + } + + @Test + public void testFoldsMinOverConstantWithNonScalarSource() + { + // Simulates: SELECT min(orderkey) FROM orders WHERE orderkey = 7 + // After constant propagation, plan is: Agg[min(x)] -> Project[x := 7] -> Filter -> TableScan + // MIN of the same constant value over any number of rows is still that value + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.project( + com.facebook.presto.spi.plan.Assignments.builder() + .put(x, p.rowExpression("BIGINT '7'")) + .build(), + p.filter( + p.rowExpression("true"), + p.tableScan(ImmutableList.of(), ImmutableMap.of()))))); + }) + .matches(values(ImmutableMap.of("min_1", 0))); + } + + @Test + public void testFoldsMinOverDerivedConstantExpression() + { + // Tests that RowExpressionOptimizer resolves non-literal constant expressions + // e.g., CAST(7 AS BIGINT) is a CallExpression, not a ConstantExpression, + // but the optimizer can evaluate it to a constant + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.project( + com.facebook.presto.spi.plan.Assignments.builder() + .put(x, p.rowExpression("CAST(7 AS BIGINT)")) + .build(), + p.tableScan(ImmutableList.of(), ImmutableMap.of())))); + }) + .matches(values(ImmutableMap.of("min_1", 0))); + } + + @Test + public void testFoldsWithGroupByOverConstant() + { + // MIN(constant) with GROUP BY should remove the aggregation and project constant + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression key = p.variable("key", BIGINT); + VariableReferenceExpression val = p.variable("val", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(val)")) + .singleGroupingSet(key) + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(key, val), + ImmutableList.of(constantExpressions(BIGINT, 1L, 5L))))); + }) + .matches(project( + aggregation( + ImmutableMap.of(), + values(ImmutableMap.of("key", 0, "val", 1))))); + } + + @Test + public void testDoesNotFoldSumOverConstant() + { + // SUM depends on row count, should NOT be folded + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("sum_1", BIGINT), + p.rowExpression("sum(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 5L))))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFoldCountOverConstant() + { + // COUNT depends on row count, should NOT be folded + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("count_1", BIGINT), + p.rowExpression("count(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 5L))))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFoldCountStar() + { + // COUNT(*) has no arguments, should NOT be folded by this rule + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> p.aggregation(a -> a + .addAggregation( + p.variable("count_1", BIGINT), + p.rowExpression("count()")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(p.variable("x", BIGINT)), + ImmutableList.of(constantExpressions(BIGINT, 1L)))))) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnNonConstantArgument() + { + // MIN over a non-constant variable should not fire + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + p.registerVariable(p.variable("x", BIGINT)); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.tableScan(ImmutableList.of(), ImmutableMap.of()))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnPartialStep() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.PARTIAL) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 1L))))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireWhenDisabled() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "false") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 1L))))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnFilteredAggregation() + { + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + VariableReferenceExpression filterVar = p.variable("f", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)"), + Optional.empty(), + Optional.empty(), + false, + Optional.of(filterVar)) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x, filterVar), + ImmutableList.of(constantExpressions(BIGINT, 5L, 1L))))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnNonConstantSourceWithSum() + { + // SUM over non-constant source should not fire + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + p.registerVariable(p.variable("x", DOUBLE)); + return p.aggregation(a -> a + .addAggregation( + p.variable("sum_1", DOUBLE), + p.rowExpression("sum(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.tableScan(ImmutableList.of(), ImmutableMap.of()))); + }) + .doesNotFire(); + } + + @Test + public void testFoldsMixedAggregationsPartially() + { + // When some aggregations can be folded (MIN) and others cannot (SUM), + // fold the ones that can and keep the rest + tester().assertThat(new SimplifyAggregationsOverConstant(getFunctionManager())) + .setSystemProperty(SIMPLIFY_AGGREGATIONS_OVER_CONSTANT, "true") + .on(p -> { + VariableReferenceExpression x = p.variable("x", BIGINT); + return p.aggregation(a -> a + .addAggregation( + p.variable("min_1", BIGINT), + p.rowExpression("min(x)")) + .addAggregation( + p.variable("sum_1", BIGINT), + p.rowExpression("sum(x)")) + .globalGrouping() + .step(AggregationNode.Step.SINGLE) + .source(p.values( + ImmutableList.of(x), + ImmutableList.of(constantExpressions(BIGINT, 5L))))); + }) + .matches(project( + aggregation( + ImmutableMap.of("sum_1", com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall("sum", ImmutableList.of("x"))), + values(ImmutableMap.of("x", 0))))); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyCoalesceOverJoinKeys.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyCoalesceOverJoinKeys.java new file mode 100644 index 0000000000000..13d1bb3003cfb --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyCoalesceOverJoinKeys.java @@ -0,0 +1,402 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_COALESCE_OVER_JOIN_KEYS; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.COALESCE; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.equiJoinClause; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.project; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; + +public class TestSimplifyCoalesceOverJoinKeys + extends BaseRuleTest +{ + @Test + public void testLeftJoinCoalesceLeftRight() + { + // COALESCE(l.x, r.y) on LEFT JOIN l.x = r.y -> l.x + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("left_key")), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testLeftJoinCoalesceRightLeft() + { + // COALESCE(r.y, l.x) on LEFT JOIN l.x = r.y -> l.x (left key is always non-null) + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, rightKey, leftKey)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("left_key")), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testRightJoinCoalesceLeftRight() + { + // COALESCE(l.x, r.y) on RIGHT JOIN l.x = r.y -> r.y (right key is always non-null) + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.RIGHT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("right_key")), + join(JoinType.RIGHT, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testRightJoinCoalesceRightLeft() + { + // COALESCE(r.y, l.x) on RIGHT JOIN l.x = r.y -> r.y + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, rightKey, leftKey)) + .build(), + p.join(JoinType.RIGHT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("right_key")), + join(JoinType.RIGHT, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testInnerJoinCoalesceLeftRight() + { + // COALESCE(l.x, r.y) on INNER JOIN l.x = r.y -> l.x (first arg, since both non-null) + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.INNER, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("left_key")), + join(JoinType.INNER, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testInnerJoinCoalesceRightLeft() + { + // COALESCE(r.y, l.x) on INNER JOIN l.x = r.y -> r.y (first arg, since both non-null) + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, rightKey, leftKey)) + .build(), + p.join(JoinType.INNER, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("right_key")), + join(JoinType.INNER, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key"), + values("right_key")))); + } + + @Test + public void testDoesNotFireOnFullJoin() + { + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.FULL, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnCrossJoin() + { + // Cross join has no equi-join criteria + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.INNER, + p.values(leftKey), + p.values(rightKey))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnNonJoinKeyCoalesce() + { + // COALESCE(l.val, r.val) where val columns are NOT join keys + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression leftVal = p.variable("left_val", BIGINT); + VariableReferenceExpression rightVal = p.variable("right_val", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftVal, rightVal)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey, leftVal), + p.values(rightKey, rightVal), + new EquiJoinClause(leftKey, rightKey))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireWhenDisabled() + { + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "false") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .doesNotFire(); + } + + @Test + public void testDoesNotFireOnNonCoalesceProject() + { + // Project with identity assignments (no COALESCE) + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + return p.project( + Assignments.builder() + .put(leftKey, leftKey) + .put(rightKey, rightKey) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .doesNotFire(); + } + + @Test + public void testMultipleJoinKeys() + { + // Multiple join keys, COALESCE on both + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey1 = p.variable("left_key1", BIGINT); + VariableReferenceExpression rightKey1 = p.variable("right_key1", BIGINT); + VariableReferenceExpression leftKey2 = p.variable("left_key2", BIGINT); + VariableReferenceExpression rightKey2 = p.variable("right_key2", BIGINT); + VariableReferenceExpression output1 = p.variable("output1", BIGINT); + VariableReferenceExpression output2 = p.variable("output2", BIGINT); + return p.project( + Assignments.builder() + .put(output1, new SpecialFormExpression(COALESCE, BIGINT, leftKey1, rightKey1)) + .put(output2, new SpecialFormExpression(COALESCE, BIGINT, leftKey2, rightKey2)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey1, leftKey2), + p.values(rightKey1, rightKey2), + new EquiJoinClause(leftKey1, rightKey1), + new EquiJoinClause(leftKey2, rightKey2))); + }) + .matches( + project(ImmutableMap.of("output1", expression("left_key1"), "output2", expression("left_key2")), + join(JoinType.LEFT, + ImmutableList.of(equiJoinClause("left_key1", "right_key1"), equiJoinClause("left_key2", "right_key2")), + Optional.empty(), + values("left_key1", "left_key2"), + values("right_key1", "right_key2")))); + } + + @Test + public void testMixedCoalesceAndIdentity() + { + // One assignment is COALESCE over join keys, another is identity + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression leftVal = p.variable("left_val", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey)) + .put(leftVal, leftVal) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey, leftVal), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .matches( + project(ImmutableMap.of("output", expression("left_key"), "left_val", expression("left_val")), + join(JoinType.LEFT, ImmutableList.of(equiJoinClause("left_key", "right_key")), Optional.empty(), + values("left_key", "left_val"), + values("right_key")))); + } + + @Test + public void testDoesNotFireOnThreeArgCoalesce() + { + // COALESCE with 3 arguments -- should not simplify + tester().assertThat(new SimplifyCoalesceOverJoinKeys()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .on(p -> + { + VariableReferenceExpression leftKey = p.variable("left_key", BIGINT); + VariableReferenceExpression rightKey = p.variable("right_key", BIGINT); + VariableReferenceExpression extra = p.variable("extra", BIGINT); + VariableReferenceExpression output = p.variable("output", BIGINT); + return p.project( + Assignments.builder() + .put(output, new SpecialFormExpression(COALESCE, BIGINT, leftKey, rightKey, extra)) + .build(), + p.join(JoinType.LEFT, + p.values(leftKey, extra), + p.values(rightKey), + new EquiJoinClause(leftKey, rightKey))); + }) + .doesNotFire(); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java index 78ff684386afc..2214946b08ffd 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestSimplifyRowExpressions.java @@ -143,6 +143,40 @@ public void testExtractCommonPredicates() " OR (A51 AND A52) OR (A53 AND A54) OR (A55 AND A56) OR (A57 AND A58) OR (A59 AND A60)"); } + @Test + public void testSimplifyNestedIf() + { + // Basic: IF(X, IF(Y, V, null), null) → IF(X AND Y, V, null) + assertSimplifies( + "IF(X, IF(Y, V, CAST(null AS boolean)), CAST(null AS boolean))", + "IF(X AND Y, V)"); + + // Omitted ELSE (defaults to null): IF(X, IF(Y, V)) + assertSimplifies( + "IF(X, IF(Y, V))", + "IF(X AND Y, V)"); + + // Triple nesting flattened in a single pass (bottom-up) + assertSimplifies( + "IF(X, IF(Y, IF(Z, V, CAST(null AS boolean)), CAST(null AS boolean)), CAST(null AS boolean))", + "IF((X AND Y) AND Z, V)"); + + // Matching non-null else branches: IF(X, IF(Y, V, Z), Z) → IF(X AND Y, V, Z) + assertSimplifies( + "IF(X, IF(Y, V, Z), Z)", + "IF(X AND Y, V, Z)"); + + // No simplification: else branches differ + assertSimplifies( + "IF(X, IF(Y, V, Z), A)", + "IF(X, IF(Y, V, Z), A)"); + + // No simplification: true branch is not an IF + assertSimplifies( + "IF(X, V, CAST(null AS boolean))", + "IF(X, V)"); + } + @Test public void testCastBigintToBoundedVarchar() { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestDifferentialPlanRewriter.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestDifferentialPlanRewriter.java new file mode 100644 index 0000000000000..3c32076248319 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestDifferentialPlanRewriter.java @@ -0,0 +1,672 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule.materializedview; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.block.SortOrder; +import com.facebook.presto.common.predicate.Domain; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.EquiJoinClause; +import com.facebook.presto.spi.plan.ExceptNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.JoinNode; +import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.spi.plan.Ordering; +import com.facebook.presto.spi.plan.OrderingScheme; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.SortNode; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.UnionNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.Lookup; +import com.facebook.presto.testing.LocalQueryRunner; +import com.facebook.presto.tpch.TpchConnectorFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.plan.JoinType.INNER; +import static com.facebook.presto.spi.plan.JoinType.LEFT; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static io.airlift.slice.Slices.utf8Slice; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestDifferentialPlanRewriter +{ + private static final String CATALOG = "local"; + private static final SchemaTableName ORDERS_TABLE = new SchemaTableName("tiny", "orders"); + private static final SchemaTableName CUSTOMER_TABLE = new SchemaTableName("tiny", "customer"); + + private LocalQueryRunner queryRunner; + private Metadata metadata; + private Session session; + private PlanNodeIdAllocator idAllocator; + private VariableAllocator variableAllocator; + private Lookup lookup; + + @BeforeClass + public void setUp() + { + Session baseSession = testSessionBuilder() + .setCatalog(CATALOG) + .setSchema("tiny") + .build(); + queryRunner = new LocalQueryRunner(baseSession); + queryRunner.createCatalog(CATALOG, new TpchConnectorFactory(1), ImmutableMap.of()); + metadata = queryRunner.getMetadata(); + // Create a session with a transaction for metadata access + session = baseSession.beginTransactionId( + queryRunner.getTransactionManager().beginTransaction(false), + queryRunner.getTransactionManager(), + queryRunner.getAccessControl()); + idAllocator = new PlanNodeIdAllocator(); + variableAllocator = new VariableAllocator(); + lookup = Lookup.noLookup(); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + closeAllRuntimeException(queryRunner); + queryRunner = null; + } + + @Test + public void testTableScanDeltaStructure() + { + // Given: A TableScan with stale partition predicate on 'orderdate' column + TableScanNode tableScan = createOrdersTableScan(); + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + Map identityMapping = + createIdentityMapping(tableScan.getOutputVariables()); + DifferentialPlanRewriter.NodeWithMapping result = builder.buildDeltaPlan(tableScan, identityMapping); + + // Then: Result should be a FilterNode (stale predicate) over TableScan + assertNotNull(result); + assertNotNull(result.getNode()); + assertTrue(result.getNode() instanceof FilterNode, "Delta should be FilterNode, got: " + result.getNode().getClass().getSimpleName()); + + FilterNode filterNode = (FilterNode) result.getNode(); + assertTrue(filterNode.getSource() instanceof TableScanNode, "Filter source should be TableScan"); + } + + @Test + public void testJoinDeltaProducesUnion() + { + // Given: Join of two TableScans, both with stale partitions + TableScanNode orders = createOrdersTableScan(); + TableScanNode customers = createCustomerTableScan(); + JoinNode join = createInnerJoin(orders, customers); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01"))))), + CUSTOMER_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "mktsegment", Domain.singleValue(VARCHAR, utf8Slice("BUILDING")))))); + + PassthroughColumnEquivalences columnEquivalences = createJoinPassthroughColumnEquivalences(); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + Map identityMapping = + createIdentityMapping(join.getOutputVariables()); + DifferentialPlanRewriter.NodeWithMapping result = builder.buildDeltaPlan(join, identityMapping); + + // Then: Result should be UnionNode (∆R ⋈ S') ∪ (R ⋈ ∆S) + assertNotNull(result); + assertTrue(result.getNode() instanceof UnionNode, "Join delta should be UnionNode, got: " + result.getNode().getClass().getSimpleName()); + + UnionNode unionNode = (UnionNode) result.getNode(); + assertEquals(unionNode.getSources().size(), 2, "Union should have 2 sources"); + assertTrue(unionNode.getSources().get(0) instanceof JoinNode, "First union source should be JoinNode"); + assertTrue(unionNode.getSources().get(1) instanceof JoinNode, "Second union source should be JoinNode"); + } + + @Test(expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = ".*Outer joins not supported.*") + public void testOuterJoinThrowsException() + { + // Given: LEFT JOIN + TableScanNode orders = createOrdersTableScan(); + TableScanNode customers = createCustomerTableScan(); + JoinNode leftJoin = createJoin(orders, customers, LEFT); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of(TupleDomain.all())); + + PassthroughColumnEquivalences columnEquivalences = createJoinPassthroughColumnEquivalences(); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + // Then: UnsupportedOperationException is thrown + Map identityMapping = + createIdentityMapping(leftJoin.getOutputVariables()); + builder.buildDeltaPlan(leftJoin, identityMapping); + } + + @Test + public void testUnionDeltaIsUnionOfDeltas() + { + // Given: Union of two TableScans of same table + TableScanNode orders1 = createOrdersTableScan(); + TableScanNode orders2 = createOrdersTableScan(); + UnionNode unionNode = createUnion(orders1, orders2); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + Map identityMapping = + createIdentityMapping(unionNode.getOutputVariables()); + DifferentialPlanRewriter.NodeWithMapping result = builder.buildDeltaPlan(unionNode, identityMapping); + + // Then: Result should be UnionNode (∆R ∪ ∆S) + assertNotNull(result); + assertTrue(result.getNode() instanceof UnionNode, "Union delta should be UnionNode"); + + UnionNode resultUnion = (UnionNode) result.getNode(); + assertEquals(resultUnion.getSources().size(), 2, "Delta union should have 2 sources"); + } + + @Test + public void testMappingsAreComposedCorrectly() + { + // Given: A simple TableScan with known variables + TableScanNode tableScan = createOrdersTableScan(); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called with a non-identity mapping + VariableReferenceExpression mvOutputVar = variableAllocator.newVariable("mv_orderkey", BIGINT); + VariableReferenceExpression viewQueryVar = tableScan.getOutputVariables().get(0); + Map viewQueryMapping = + ImmutableMap.of(mvOutputVar, viewQueryVar); + + DifferentialPlanRewriter.NodeWithMapping result = builder.buildDeltaPlan(tableScan, viewQueryMapping); + + // Then: The result mapping should map mvOutputVar to the delta variable + assertNotNull(result.getMapping()); + assertTrue(result.getMapping().containsKey(mvOutputVar), "Result mapping should contain MV output variable"); + } + + @Test(expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = ".*Sort cannot be differentially stitched.*") + public void testSortThrowsException() + { + // Given: Sort over TableScan + TableScanNode tableScan = createOrdersTableScan(); + SortNode sortNode = createSort(tableScan); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + // Then: UnsupportedOperationException is thrown + Map identityMapping = + createIdentityMapping(sortNode.getOutputVariables()); + builder.buildDeltaPlan(sortNode, identityMapping); + } + + @Test(expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = ".*Limit cannot be differentially stitched.*") + public void testLimitThrowsException() + { + // Given: Limit over TableScan + TableScanNode tableScan = createOrdersTableScan(); + LimitNode limitNode = createLimit(tableScan, 10); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + // Then: UnsupportedOperationException is thrown + Map identityMapping = + createIdentityMapping(limitNode.getOutputVariables()); + builder.buildDeltaPlan(limitNode, identityMapping); + } + + @Test(expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = ".*TopN cannot be differentially stitched.*") + public void testTopNThrowsException() + { + // Given: TopN over TableScan + TableScanNode tableScan = createOrdersTableScan(); + TopNNode topNNode = createTopN(tableScan, 10); + + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))))); + + PassthroughColumnEquivalences columnEquivalences = createSimplePassthroughColumnEquivalences(ORDERS_TABLE, "orderdate"); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + // Then: UnsupportedOperationException is thrown + Map identityMapping = + createIdentityMapping(topNNode.getOutputVariables()); + builder.buildDeltaPlan(topNNode, identityMapping); + } + + @Test + public void testExceptDeltaRightUsesUnchanged() + { + TableScanNode orders = createOrdersTableScan(); + TableScanNode customers = createCustomerTableScan(); + ExceptNode exceptNode = createExcept(orders, customers); + + // Both tables have stale partitions + Map>> staleConstraints = ImmutableMap.of( + ORDERS_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01"))))), + CUSTOMER_TABLE, ImmutableList.of( + TupleDomain.withColumnDomains(ImmutableMap.of( + "mktsegment", Domain.singleValue(VARCHAR, utf8Slice("BUILDING")))))); + + PassthroughColumnEquivalences columnEquivalences = createJoinPassthroughColumnEquivalences(); + + DifferentialPlanRewriter builder = new DifferentialPlanRewriter( + metadata, + session, + idAllocator, + variableAllocator, + staleConstraints, + columnEquivalences, + lookup, + WarningCollector.NOOP); + + // When: buildDeltaPlan is called + Map identityMapping = + createIdentityMapping(exceptNode.getOutputVariables()); + DifferentialPlanRewriter.NodeWithMapping result = builder.buildDeltaPlan(exceptNode, identityMapping); + + // Then: Result should be UnionNode (deltaLeft ∪ deltaRight) + assertNotNull(result); + assertTrue(result.getNode() instanceof UnionNode, "Except delta should be UnionNode, got: " + result.getNode().getClass().getSimpleName()); + + UnionNode unionNode = (UnionNode) result.getNode(); + assertEquals(unionNode.getSources().size(), 2, "Union should have 2 sources"); + + // Both union sources should be ExceptNodes + assertTrue(unionNode.getSources().get(0) instanceof ExceptNode, "First union source (deltaLeft) should be ExceptNode"); + assertTrue(unionNode.getSources().get(1) instanceof ExceptNode, "Second union source (deltaRight) should be ExceptNode"); + + // deltaRight: The left side of the EXCEPT should use R (unchanged), not R' (current) + // This means: Filter[S's stale predicate] -> Filter[NOT R's stale predicate] -> TableScan + ExceptNode deltaRight = (ExceptNode) unionNode.getSources().get(1); + PlanNode deltaRightLeftSource = deltaRight.getSources().get(0); + + // The left source should be: Filter[S's stale predicate] -> Filter[NOT R's stale predicate] -> TableScan + assertTrue(deltaRightLeftSource instanceof FilterNode, "deltaRight left source should be FilterNode"); + FilterNode outerFilter = (FilterNode) deltaRightLeftSource; + + // The source of the outer filter should be another Filter (the unchanged filter for R) + assertTrue(outerFilter.getSource() instanceof FilterNode, + "deltaRight should use R (unchanged - Filter -> TableScan), not R' (current - TableScan directly). " + + "This prevents double-counting when R and S share stale partitions. " + + "Got: " + outerFilter.getSource().getClass().getSimpleName()); + + // Verify the inner filter wraps a TableScan + FilterNode innerFilter = (FilterNode) outerFilter.getSource(); + assertTrue(innerFilter.getSource() instanceof TableScanNode, + "Inner filter should wrap TableScan. Got: " + innerFilter.getSource().getClass().getSimpleName()); + } + + // Helper methods + + private TableScanNode createOrdersTableScan() + { + QualifiedObjectName tableName = QualifiedObjectName.valueOf(CATALOG + ".tiny.orders"); + TableHandle tableHandle = metadata.getHandleVersion(session, tableName, Optional.empty()) + .orElseThrow(() -> new IllegalStateException("Table not found: " + tableName)); + + Map columnHandles = metadata.getColumnHandles(session, tableHandle); + ColumnHandle orderkeyHandle = columnHandles.get("orderkey"); + ColumnHandle orderdateHandle = columnHandles.get("orderdate"); + + VariableReferenceExpression orderkey = variableAllocator.newVariable("orderkey", BIGINT); + VariableReferenceExpression orderdate = variableAllocator.newVariable("orderdate", VARCHAR); + + return new TableScanNode( + Optional.empty(), + idAllocator.getNextId(), + tableHandle, + ImmutableList.of(orderkey, orderdate), + ImmutableMap.of(orderkey, orderkeyHandle, orderdate, orderdateHandle), + TupleDomain.all(), + TupleDomain.all(), + Optional.empty()); + } + + private TableScanNode createCustomerTableScan() + { + QualifiedObjectName tableName = QualifiedObjectName.valueOf(CATALOG + ".tiny.customer"); + TableHandle tableHandle = metadata.getHandleVersion(session, tableName, Optional.empty()) + .orElseThrow(() -> new IllegalStateException("Table not found: " + tableName)); + + Map columnHandles = metadata.getColumnHandles(session, tableHandle); + ColumnHandle custkeyHandle = columnHandles.get("custkey"); + ColumnHandle mktsegmentHandle = columnHandles.get("mktsegment"); + + VariableReferenceExpression custkey = variableAllocator.newVariable("custkey", BIGINT); + VariableReferenceExpression mktsegment = variableAllocator.newVariable("mktsegment", VARCHAR); + + return new TableScanNode( + Optional.empty(), + idAllocator.getNextId(), + tableHandle, + ImmutableList.of(custkey, mktsegment), + ImmutableMap.of(custkey, custkeyHandle, mktsegment, mktsegmentHandle), + TupleDomain.all(), + TupleDomain.all(), + Optional.empty()); + } + + private JoinNode createInnerJoin(TableScanNode left, TableScanNode right) + { + return createJoin(left, right, INNER); + } + + private JoinNode createJoin(TableScanNode left, TableScanNode right, JoinType joinType) + { + VariableReferenceExpression leftJoinKey = left.getOutputVariables().get(0); + VariableReferenceExpression rightJoinKey = right.getOutputVariables().get(0); + + ImmutableList.Builder outputVariables = ImmutableList.builder(); + outputVariables.addAll(left.getOutputVariables()); + outputVariables.addAll(right.getOutputVariables()); + + return new JoinNode( + Optional.empty(), + idAllocator.getNextId(), + joinType, + left, + right, + ImmutableList.of(new EquiJoinClause(leftJoinKey, rightJoinKey)), + outputVariables.build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableMap.of()); + } + + private ExceptNode createExcept(TableScanNode left, TableScanNode right) + { + ImmutableList.Builder outputVariables = ImmutableList.builder(); + ImmutableMap.Builder> variableMapping = ImmutableMap.builder(); + + // Use left source's output variables as output schema + for (VariableReferenceExpression outputVar : left.getOutputVariables()) { + VariableReferenceExpression exceptOutput = variableAllocator.newVariable(outputVar.getName() + "_except", outputVar.getType()); + outputVariables.add(exceptOutput); + variableMapping.put(exceptOutput, ImmutableList.of( + outputVar, + right.getOutputVariables().get(left.getOutputVariables().indexOf(outputVar)))); + } + + return new ExceptNode( + Optional.empty(), + idAllocator.getNextId(), + ImmutableList.of(left, right), + outputVariables.build(), + variableMapping.build()); + } + + private UnionNode createUnion(TableScanNode... sources) + { + ImmutableList.Builder sourceNodes = ImmutableList.builder(); + ImmutableList.Builder outputVariables = ImmutableList.builder(); + ImmutableMap.Builder> variableMapping = ImmutableMap.builder(); + + // Use first source's output variables as output schema + for (VariableReferenceExpression outputVar : sources[0].getOutputVariables()) { + VariableReferenceExpression unionOutput = variableAllocator.newVariable(outputVar.getName() + "_union", outputVar.getType()); + outputVariables.add(unionOutput); + + ImmutableList.Builder sourceVars = ImmutableList.builder(); + for (int i = 0; i < sources.length; i++) { + sourceVars.add(sources[i].getOutputVariables().get(sources[0].getOutputVariables().indexOf(outputVar))); + } + variableMapping.put(unionOutput, sourceVars.build()); + } + + for (TableScanNode source : sources) { + sourceNodes.add(source); + } + + return new UnionNode( + Optional.empty(), + idAllocator.getNextId(), + sourceNodes.build(), + outputVariables.build(), + variableMapping.build()); + } + + private Map createIdentityMapping( + List variables) + { + ImmutableMap.Builder mapping = ImmutableMap.builder(); + for (VariableReferenceExpression variable : variables) { + mapping.put(variable, variable); + } + return mapping.build(); + } + + private PassthroughColumnEquivalences createSimplePassthroughColumnEquivalences(SchemaTableName table, String partitionColumn) + { + SchemaTableName dataTable = new SchemaTableName("schema", "__mv_storage__test_mv"); + MaterializedViewDefinition mvDefinition = new MaterializedViewDefinition( + "SELECT * FROM " + table.getTableName(), + dataTable.getSchemaName(), + dataTable.getTableName(), + ImmutableList.of(table), + Optional.empty(), + Optional.empty(), + ImmutableList.of( + new MaterializedViewDefinition.ColumnMapping( + new MaterializedViewDefinition.TableColumn(dataTable, partitionColumn), + ImmutableList.of(new MaterializedViewDefinition.TableColumn(table, partitionColumn)))), + ImmutableList.of(), + Optional.empty()); + + return new PassthroughColumnEquivalences(mvDefinition, dataTable); + } + + private PassthroughColumnEquivalences createJoinPassthroughColumnEquivalences() + { + SchemaTableName dataTable = new SchemaTableName("schema", "__mv_storage__test_mv"); + MaterializedViewDefinition mvDefinition = new MaterializedViewDefinition( + "SELECT * FROM orders JOIN customer ON orders.orderdate = customer.mktsegment", + dataTable.getSchemaName(), + dataTable.getTableName(), + ImmutableList.of(ORDERS_TABLE, CUSTOMER_TABLE), + Optional.empty(), + Optional.empty(), + ImmutableList.of( + new MaterializedViewDefinition.ColumnMapping( + new MaterializedViewDefinition.TableColumn(dataTable, "orderdate"), + ImmutableList.of(new MaterializedViewDefinition.TableColumn(ORDERS_TABLE, "orderdate"))), + new MaterializedViewDefinition.ColumnMapping( + new MaterializedViewDefinition.TableColumn(dataTable, "mktsegment"), + ImmutableList.of(new MaterializedViewDefinition.TableColumn(CUSTOMER_TABLE, "mktsegment")))), + ImmutableList.of(), + Optional.empty()); + + return new PassthroughColumnEquivalences(mvDefinition, dataTable); + } + + private SortNode createSort(PlanNode source) + { + VariableReferenceExpression sortKey = source.getOutputVariables().get(0); + OrderingScheme orderingScheme = new OrderingScheme( + ImmutableList.of(new Ordering(sortKey, SortOrder.ASC_NULLS_FIRST))); + + return new SortNode( + Optional.empty(), + idAllocator.getNextId(), + source, + orderingScheme, + false, + ImmutableList.of()); + } + + private LimitNode createLimit(PlanNode source, long count) + { + return new LimitNode( + Optional.empty(), + idAllocator.getNextId(), + source, + count, + LimitNode.Step.FINAL); + } + + private TopNNode createTopN(PlanNode source, long count) + { + VariableReferenceExpression sortKey = source.getOutputVariables().get(0); + OrderingScheme orderingScheme = new OrderingScheme( + ImmutableList.of(new Ordering(sortKey, SortOrder.ASC_NULLS_FIRST))); + + return new TopNNode( + Optional.empty(), + idAllocator.getNextId(), + source, + count, + orderingScheme, + TopNNode.Step.SINGLE); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMaterializedViewRewrite.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestMaterializedViewRewrite.java similarity index 83% rename from presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMaterializedViewRewrite.java rename to presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestMaterializedViewRewrite.java index 6295e4b13de6a..fb3e721060a20 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestMaterializedViewRewrite.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestMaterializedViewRewrite.java @@ -11,13 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.iterative.rule; +package com.facebook.presto.sql.planner.iterative.rule.materializedview; import com.facebook.airlift.units.Duration; import com.facebook.presto.Session; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.common.transaction.TransactionId; import com.facebook.presto.metadata.AbstractMockMetadata; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; @@ -33,7 +35,11 @@ import com.facebook.presto.spi.analyzer.ViewDefinition; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.spi.security.AccessControlContext; import com.facebook.presto.spi.security.AllowAllAccessControl; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.spi.security.ViewExpression; +import com.facebook.presto.spi.security.ViewSecurity; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; @@ -86,7 +92,7 @@ public void testUseFreshDataWhenFullyMaterialized() { QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); - Metadata metadata = new TestingMetadataWithMaterializedViewStatus(true); + Metadata metadata = new TestingMetadataWithMaterializedViewStatus(tester().getMetadata(), true); tester().assertThat(new MaterializedViewRewrite(metadata, new AllowAllAccessControl())) .on(planBuilder -> { @@ -113,7 +119,7 @@ public void testUseViewQueryWhenNotFullyMaterialized() { QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); - Metadata metadata = new TestingMetadataWithMaterializedViewStatus(false); + Metadata metadata = new TestingMetadataWithMaterializedViewStatus(tester().getMetadata(), false); tester().assertThat(new MaterializedViewRewrite(metadata, new AllowAllAccessControl())) .on(planBuilder -> { @@ -140,7 +146,7 @@ public void testMultipleOutputVariables() { QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); - Metadata metadata = new TestingMetadataWithMaterializedViewStatus(true); + Metadata metadata = new TestingMetadataWithMaterializedViewStatus(tester().getMetadata(), true); tester().assertThat(new MaterializedViewRewrite(metadata, new AllowAllAccessControl())) .on(planBuilder -> { @@ -172,7 +178,7 @@ public void testUseViewQueryWhenBaseTableDoesNotExist() { QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); - Metadata metadata = new TestingMetadataWithMissingBaseTable(true); + Metadata metadata = new TestingMetadataWithMissingBaseTable(tester().getMetadata(), true); tester().assertThat(new MaterializedViewRewrite(metadata, new AllowAllAccessControl())) .on(planBuilder -> { @@ -222,7 +228,7 @@ public void testFailWhenStaleAndSessionPropertyIsFail() QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); - Metadata metadata = new TestingMetadataWithMaterializedViewStatus(false); + Metadata metadata = new TestingMetadataWithMaterializedViewStatus(testerWithFail.getMetadata(), false); PrestoException exception = expectThrows(PrestoException.class, () -> testerWithFail.assertThat(new MaterializedViewRewrite(metadata, new AllowAllAccessControl())) @@ -381,21 +387,88 @@ public void testFailWhenNeverRefreshedWithStalenessConfig() assertEquals(exception.getErrorCode(), MATERIALIZED_VIEW_STALE.toErrorCode()); } + @Test + public void testStitchingBlockedByRowFilterUsesViewQuery() + { + QualifiedObjectName materializedViewName = QualifiedObjectName.valueOf("catalog.schema.mv"); + + MaterializedViewStalenessConfig stalenessConfig = new MaterializedViewStalenessConfig( + MaterializedViewStaleReadBehavior.USE_STITCHING, + new Duration(0, TimeUnit.SECONDS)); + + Metadata metadata = new TestingMetadataWithStalenessConfig( + false, + stalenessConfig, + Optional.empty(), + INVOKER, + ImmutableMap.of( + new SchemaTableName("schema", "base_table"), + new MaterializedViewStatus.MaterializedDataPredicates( + ImmutableList.of(TupleDomain.all()), + ImmutableList.of("ds")))); + + tester().assertThat(new MaterializedViewRewrite(metadata, new AccessControlWithRowFilter())) + .on(planBuilder -> { + VariableReferenceExpression outputA = planBuilder.variable("a", BIGINT); + VariableReferenceExpression dataTableA = planBuilder.variable("data_table_a", BIGINT); + VariableReferenceExpression viewQueryA = planBuilder.variable("view_query_a", BIGINT); + + return planBuilder.materializedViewScan( + materializedViewName, + planBuilder.values(dataTableA), + planBuilder.values(viewQueryA), + ImmutableMap.of(outputA, dataTableA), + ImmutableMap.of(outputA, viewQueryA), + outputA); + }) + .matches( + project( + ImmutableMap.of("a", expression("view_query_a")), + values("view_query_a"))); + } + + private static class AccessControlWithRowFilter + extends AllowAllAccessControl + { + @Override + public List getRowFilters(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + if (tableName.getObjectName().equals("base_table")) { + return ImmutableList.of(new ViewExpression("test_user", Optional.of("catalog"), Optional.of("schema"), "true")); + } + return ImmutableList.of(); + } + } + private static class TestingMetadataWithStalenessConfig extends AbstractMockMetadata { private final boolean isFullyMaterialized; private final MaterializedViewStalenessConfig stalenessConfig; private final Optional lastFreshTime; + private final ViewSecurity securityMode; + private final Map partitionsFromBaseTables; public TestingMetadataWithStalenessConfig( boolean isFullyMaterialized, MaterializedViewStalenessConfig stalenessConfig, Optional lastFreshTime) + { + this(isFullyMaterialized, stalenessConfig, lastFreshTime, DEFINER, ImmutableMap.of()); + } + + public TestingMetadataWithStalenessConfig( + boolean isFullyMaterialized, + MaterializedViewStalenessConfig stalenessConfig, + Optional lastFreshTime, + ViewSecurity securityMode, + Map partitionsFromBaseTables) { this.isFullyMaterialized = isFullyMaterialized; this.stalenessConfig = stalenessConfig; this.lastFreshTime = lastFreshTime; + this.securityMode = securityMode; + this.partitionsFromBaseTables = ImmutableMap.copyOf(partitionsFromBaseTables); } @Override @@ -405,7 +478,9 @@ public MetadataResolver getMetadataResolver(Session session) super.getMetadataResolver(session), isFullyMaterialized, stalenessConfig, - lastFreshTime); + lastFreshTime, + securityMode, + partitionsFromBaseTables); } } @@ -416,17 +491,23 @@ private static class MaterializedViewTestingMetadataResolverWithStalenessConfig private final boolean isFullyMaterialized; private final MaterializedViewStalenessConfig stalenessConfig; private final Optional lastFreshTime; + private final ViewSecurity securityMode; + private final Map partitionsFromBaseTables; protected MaterializedViewTestingMetadataResolverWithStalenessConfig( MetadataResolver delegate, boolean isFullyMaterialized, MaterializedViewStalenessConfig stalenessConfig, - Optional lastFreshTime) + Optional lastFreshTime, + ViewSecurity securityMode, + Map partitionsFromBaseTables) { this.delegate = delegate; this.isFullyMaterialized = isFullyMaterialized; this.stalenessConfig = stalenessConfig; this.lastFreshTime = lastFreshTime; + this.securityMode = securityMode; + this.partitionsFromBaseTables = ImmutableMap.copyOf(partitionsFromBaseTables); } @Override @@ -474,7 +555,7 @@ public Optional getMaterializedView(QualifiedObjectN "mv", ImmutableList.of(new SchemaTableName("schema", "base_table")), Optional.of("test_owner"), - Optional.of(DEFINER), + Optional.of(securityMode), ImmutableList.of(), ImmutableList.of(), Optional.empty(), @@ -487,7 +568,7 @@ public MaterializedViewStatus getMaterializedViewStatus(QualifiedObjectName mate { return new MaterializedViewStatus( isFullyMaterialized ? FULLY_MATERIALIZED : PARTIALLY_MATERIALIZED, - ImmutableMap.of(), + partitionsFromBaseTables, lastFreshTime); } } @@ -495,13 +576,21 @@ public MaterializedViewStatus getMaterializedViewStatus(QualifiedObjectName mate private static class TestingMetadataWithMaterializedViewStatus extends AbstractMockMetadata { + private final Metadata delegate; private final boolean isFullyMaterialized; - public TestingMetadataWithMaterializedViewStatus(boolean isFullyMaterialized) + public TestingMetadataWithMaterializedViewStatus(Metadata delegate, boolean isFullyMaterialized) { + this.delegate = delegate; this.isFullyMaterialized = isFullyMaterialized; } + @Override + public FunctionAndTypeManager getFunctionAndTypeManager() + { + return delegate.getFunctionAndTypeManager(); + } + @Override public MetadataResolver getMetadataResolver(Session session) { @@ -512,13 +601,21 @@ public MetadataResolver getMetadataResolver(Session session) private static class TestingMetadataWithMissingBaseTable extends AbstractMockMetadata { + private final Metadata delegate; private final boolean isFullyMaterialized; - public TestingMetadataWithMissingBaseTable(boolean isFullyMaterialized) + public TestingMetadataWithMissingBaseTable(Metadata delegate, boolean isFullyMaterialized) { + this.delegate = delegate; this.isFullyMaterialized = isFullyMaterialized; } + @Override + public FunctionAndTypeManager getFunctionAndTypeManager() + { + return delegate.getFunctionAndTypeManager(); + } + @Override public MetadataResolver getMetadataResolver(Session session) { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestPassthroughColumnEquivalences.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestPassthroughColumnEquivalences.java new file mode 100644 index 0000000000000..e43b761f1395b --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/materializedview/TestPassthroughColumnEquivalences.java @@ -0,0 +1,434 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule.materializedview; + +import com.facebook.presto.common.predicate.Domain; +import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewDefinition.ColumnMapping; +import com.facebook.presto.spi.MaterializedViewDefinition.TableColumn; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static io.airlift.slice.Slices.utf8Slice; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestPassthroughColumnEquivalences +{ + private static final SchemaTableName ORDERS_TABLE = new SchemaTableName("catalog", "orders"); + private static final SchemaTableName CUSTOMER_TABLE = new SchemaTableName("catalog", "customer"); + private static final SchemaTableName MV_DATA_TABLE = new SchemaTableName("catalog", "__mv_storage__test_mv"); + + private Metadata metadata; + private RowExpressionDomainTranslator translator; + + @BeforeClass + public void setUp() + { + metadata = MetadataManager.createTestMetadataManager(); + translator = new RowExpressionDomainTranslator(metadata); + } + + @Test + public void testBasicEquivalence() + { + // Single column mapping: orders.orderdate -> mv.orderdate + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Both columns should have equivalence + assertTrue(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderdate"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderdate"))); + + // Non-mapped column should not have equivalence + assertFalse(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderkey"))); + assertFalse(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderkey"))); + } + + @Test + public void testMultipleColumnMappings() + { + // Multiple column mappings + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true), + createColumnMapping("orderstatus", ORDERS_TABLE, "orderstatus", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Both mappings should create equivalences + assertTrue(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderdate"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderdate"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderstatus"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderstatus"))); + } + + @Test + public void testDirectMappedFilterTrue() + { + // Direct mapped column should be in equivalence class + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderdate"))); + } + + @Test + public void testDirectMappedFilterFalse() + { + // Non-direct mapped column should NOT be in equivalence class + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", false))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // MV column exists but base table column is not direct mapped + // Since there's only one column in the equivalence class, no equivalence is created + assertFalse(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderdate"))); + assertFalse(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderdate"))); + } + + @Test + public void testJoinEquivalences() + { + // Join condition: orders.dt = customer.dt both map to mv.dt + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE, CUSTOMER_TABLE), + ImmutableList.of( + createColumnMappingWithMultipleSources("dt", + ImmutableList.of( + new TableColumn(ORDERS_TABLE, "dt", Optional.of(true)), + new TableColumn(CUSTOMER_TABLE, "dt", Optional.of(true)))))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // All three columns should be equivalent + assertTrue(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "dt"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "dt"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(CUSTOMER_TABLE, "dt"))); + } + + @Test + public void testGetEquivalentPredicatesFromBaseTable() + { + // Predicate on base table should translate to MV data table + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))); + + Map> result = equivalences.getEquivalentPredicates(ORDERS_TABLE, predicate); + + // Should translate to MV data table + assertEquals(result.size(), 1); + assertTrue(result.containsKey(MV_DATA_TABLE)); + assertTrue(result.get(MV_DATA_TABLE).getDomains().isPresent()); + assertTrue(result.get(MV_DATA_TABLE).getDomains().get().containsKey("orderdate")); + } + + @Test + public void testGetEquivalentPredicatesFromMvDataTable() + { + // Predicate on MV data table should translate to base tables + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))); + + Map> result = equivalences.getEquivalentPredicates(MV_DATA_TABLE, predicate); + + // Should translate to base table + assertEquals(result.size(), 1); + assertTrue(result.containsKey(ORDERS_TABLE)); + assertTrue(result.get(ORDERS_TABLE).getDomains().isPresent()); + assertTrue(result.get(ORDERS_TABLE).getDomains().get().containsKey("orderdate")); + } + + @Test + public void testGetEquivalentPredicatesWithJoin() + { + // Join equivalence: predicate on orders.dt should translate to both mv.dt and customer.dt + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE, CUSTOMER_TABLE), + ImmutableList.of( + createColumnMappingWithMultipleSources("dt", + ImmutableList.of( + new TableColumn(ORDERS_TABLE, "dt", Optional.of(true)), + new TableColumn(CUSTOMER_TABLE, "dt", Optional.of(true)))))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of("dt", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))); + + Map> result = equivalences.getEquivalentPredicates(ORDERS_TABLE, predicate); + + // Should translate to both MV data table and customer table + assertEquals(result.size(), 2); + assertTrue(result.containsKey(MV_DATA_TABLE)); + assertTrue(result.containsKey(CUSTOMER_TABLE)); + } + + @Test + public void testGetEquivalentPredicatesColumnWithoutEquivalence() + { + // Column without equivalence should be skipped + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Predicate on non-mapped column + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of("orderkey", Domain.singleValue(BIGINT, 123L))); + + Map> result = equivalences.getEquivalentPredicates(ORDERS_TABLE, predicate); + + // No equivalents found + assertTrue(result.isEmpty()); + } + + @Test + public void testGetEquivalentPredicatesEmptyDomains() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Predicate with no domains + TupleDomain predicate = TupleDomain.none(); + + Map> result = equivalences.getEquivalentPredicates(ORDERS_TABLE, predicate); + + assertTrue(result.isEmpty()); + } + + @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = "Unknown table.*") + public void testGetEquivalentPredicatesUnknownTable() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + SchemaTableName unknownTable = new SchemaTableName("catalog", "unknown_table"); + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of("col", Domain.singleValue(VARCHAR, utf8Slice("value")))); + + equivalences.getEquivalentPredicates(unknownTable, predicate); + } + + @Test + public void testTranslatePredicatesToVariablesEmpty() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + List result = equivalences.translatePredicatesToVariables( + ORDERS_TABLE, + ImmutableList.of(), + ImmutableMap.of(), + translator); + + assertTrue(result.isEmpty()); + } + + @Test + public void testTranslatePredicatesToVariables() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Create variable mapping for MV data table column + VariableReferenceExpression orderdateVar = new VariableReferenceExpression(Optional.empty(), "orderdate", VARCHAR); + Map columnToVariable = ImmutableMap.of( + new TableColumn(MV_DATA_TABLE, "orderdate"), orderdateVar); + + List> stalePredicates = ImmutableList.of( + TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01"))))); + + List result = equivalences.translatePredicatesToVariables( + ORDERS_TABLE, + stalePredicates, + columnToVariable, + translator); + + assertNotNull(result); + assertEquals(result.size(), 1); + } + + @Test(expectedExceptions = UnsupportedOperationException.class, expectedExceptionsMessageRegExp = "Cannot map stale predicates.*") + public void testTranslatePredicatesToVariablesNoMapping() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // No variable mapping provided - should throw + List> stalePredicates = ImmutableList.of( + TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01"))))); + + equivalences.translatePredicatesToVariables( + ORDERS_TABLE, + stalePredicates, + ImmutableMap.of(), // Empty mapping + translator); + } + + @Test + public void testTranslatePredicatesToVariablesMultiplePredicates() + { + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMapping("orderdate", ORDERS_TABLE, "orderdate", true))); + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + VariableReferenceExpression orderdateVar = new VariableReferenceExpression(Optional.empty(), "orderdate", VARCHAR); + Map columnToVariable = ImmutableMap.of( + new TableColumn(MV_DATA_TABLE, "orderdate"), orderdateVar); + + // Multiple stale predicates (disjuncts) + List> stalePredicates = ImmutableList.of( + TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-01")))), + TupleDomain.withColumnDomains( + ImmutableMap.of("orderdate", Domain.singleValue(VARCHAR, utf8Slice("2024-01-02"))))); + + List result = equivalences.translatePredicatesToVariables( + ORDERS_TABLE, + stalePredicates, + columnToVariable, + translator); + + // Should produce one expression per predicate + assertEquals(result.size(), 2); + } + + @Test + public void testDirectMappedDefaultsToTrue() + { + // When isDirectMapped is not specified (empty Optional), it defaults to true + MaterializedViewDefinition mvDefinition = createMvDefinition( + ImmutableList.of(ORDERS_TABLE), + ImmutableList.of( + createColumnMappingWithMultipleSources("orderdate", + ImmutableList.of( + new TableColumn(ORDERS_TABLE, "orderdate", Optional.empty()))))); // No isDirectMapped specified + + PassthroughColumnEquivalences equivalences = new PassthroughColumnEquivalences(mvDefinition, MV_DATA_TABLE); + + // Should default to direct mapped = true + assertTrue(equivalences.hasEquivalence(new TableColumn(ORDERS_TABLE, "orderdate"))); + assertTrue(equivalences.hasEquivalence(new TableColumn(MV_DATA_TABLE, "orderdate"))); + } + + // Helper methods + + private MaterializedViewDefinition createMvDefinition( + List baseTables, + List columnMappings) + { + return new MaterializedViewDefinition( + "SELECT * FROM test", + MV_DATA_TABLE.getSchemaName(), + MV_DATA_TABLE.getTableName(), + baseTables, + Optional.empty(), + Optional.empty(), + columnMappings, + ImmutableList.of(), + Optional.empty()); + } + + private ColumnMapping createColumnMapping( + String viewColumnName, + SchemaTableName baseTable, + String baseColumnName, + boolean isDirectMapped) + { + return new ColumnMapping( + new TableColumn(MV_DATA_TABLE, viewColumnName), + ImmutableList.of(new TableColumn(baseTable, baseColumnName, Optional.of(isDirectMapped)))); + } + + private ColumnMapping createColumnMappingWithMultipleSources( + String viewColumnName, + List baseColumns) + { + return new ColumnMapping( + new TableColumn(MV_DATA_TABLE, viewColumnName), + baseColumns); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java index 8bfca678a940d..34cecae8d8c23 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/RuleAssert.java @@ -50,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; @@ -58,6 +59,7 @@ import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textLogicalPlan; import static com.facebook.presto.transaction.TransactionBuilder.transaction; import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.testng.Assert.fail; @@ -131,7 +133,7 @@ public PlanNode get() TypeProvider types = ruleApplication.types; if (!ruleApplication.wasRuleApplied()) { - fail(String.format( + fail(format( "%s did not fire for:\n%s", rule.getClass().getName(), formatPlan(plan, types))); @@ -145,7 +147,7 @@ public void doesNotFire() RuleApplication ruleApplication = applyRule(); if (ruleApplication.wasRuleApplied()) { - fail(String.format( + fail(format( "Expected %s to not fire for:\n%s", rule.getClass().getName(), inTransaction(session -> textLogicalPlan(plan, ruleApplication.types, StatsAndCosts.empty(), metadata.getFunctionAndTypeManager(), session, 2)))); @@ -158,7 +160,7 @@ public void matches(PlanMatchPattern pattern) TypeProvider types = ruleApplication.types; if (!ruleApplication.wasRuleApplied()) { - fail(String.format( + fail(format( "%s did not fire for:\n%s", rule.getClass().getName(), formatPlan(plan, types))); @@ -167,14 +169,14 @@ public void matches(PlanMatchPattern pattern) PlanNode actual = ruleApplication.getTransformedPlan(); if (actual == plan) { // plans are not comparable, so we can only ensure they are not the same instance - fail(String.format( + fail(format( "%s: rule fired but return the original plan:\n%s", rule.getClass().getName(), formatPlan(plan, types))); } if (!ImmutableSet.copyOf(plan.getOutputVariables()).equals(ImmutableSet.copyOf(actual.getOutputVariables()))) { - fail(String.format( + fail(format( "%s: output schema of transformed and original plans are not equivalent\n" + "\texpected: %s\n" + "\tactual: %s", @@ -189,28 +191,35 @@ public void matches(PlanMatchPattern pattern) }); } - public void matches(LogicalProperties expectedLogicalProperties) + public void assertLogicalProperties(Consumer matcher) { RuleApplication ruleApplication = applyRule(); TypeProvider types = ruleApplication.types; if (!ruleApplication.wasRuleApplied()) { - fail(String.format( + fail(format( "%s did not fire for:\n%s", rule.getClass().getName(), formatPlan(plan, types))); } - // ensure that the logical properties of the root group are equivalent to the expected logical properties LogicalProperties rootNodeLogicalProperties = ruleApplication.getMemo().getLogicalProperties(ruleApplication.getMemo().getRootGroup()).get(); - if (!((LogicalPropertiesImpl) rootNodeLogicalProperties).equals((LogicalPropertiesImpl) expectedLogicalProperties)) { - fail(String.format( - "Logical properties of root node doesn't match expected logical properties\n" + - "\texpected: %s\n" + - "\tactual: %s", - expectedLogicalProperties, - rootNodeLogicalProperties)); - } + matcher.accept(rootNodeLogicalProperties); + } + + public void matches(LogicalProperties expectedLogicalProperties) + { + // Ensure that the logical properties of the root group are equivalent to the expected logical properties + assertLogicalProperties(rootNodeLogicalProperties -> { + if (!((LogicalPropertiesImpl) rootNodeLogicalProperties).equals((LogicalPropertiesImpl) expectedLogicalProperties)) { + fail(format( + "Logical properties of root node doesn't match expected logical properties\n" + + "\texpected: %s\n" + + "\tactual: %s", + expectedLogicalProperties, + rootNodeLogicalProperties)); + } + }); } private RuleApplication applyRule() diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchanges.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchanges.java index 12200a14fa73a..cfd56907e5880 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchanges.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchanges.java @@ -242,19 +242,11 @@ public void testPickLayoutPartitionedOnSingle() .add(builder() .global(streamPartitionedOn("a")) .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) .add(builder() .global(hashDistributedOn("a")) .build()) - .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) - .build()) .add(builder() .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) .build()) .add(builder() .global(arbitraryPartition()) @@ -263,6 +255,14 @@ public void testPickLayoutPartitionedOnSingle() .add(builder() .global(arbitraryPartition()) .build()) + .add(builder() + .global(singleStream()) + .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) + .build()) + .add(builder() + .global(singleStreamPartition()) + .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) + .build()) .build(); assertEquals(stableSort(input, preference), expected); } @@ -307,22 +307,14 @@ public void testPickLayoutPartitionedOnMultiple() .add(builder() .global(streamPartitionedOn("a")) .build()) - .add(builder() - .global(singleStreamPartition()) - .build()) .add(builder() .global(hashDistributedOn("a")) .build()) .add(builder() - .global(singleStream()) - .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) + .global(hashDistributedOn("a")) .build()) .add(builder() .global(singleStreamPartition()) - .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) - .build()) - .add(builder() - .global(hashDistributedOn("a")) .build()) .add(builder() .global(arbitraryPartition()) @@ -331,6 +323,14 @@ public void testPickLayoutPartitionedOnMultiple() .add(builder() .global(arbitraryPartition()) .build()) + .add(builder() + .global(singleStream()) + .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) + .build()) + .add(builder() + .global(singleStreamPartition()) + .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) + .build()) .build(); assertEquals(stableSort(input, preference), expected); } @@ -726,13 +726,13 @@ public void testPickLayoutPartitionedWithGroup() .global(streamPartitionedOn("a")) .build()) .add(builder() - .global(singleStreamPartition()) + .global(hashDistributedOn("a")) .build()) .add(builder() .global(hashDistributedOn("a")) .build()) .add(builder() - .global(hashDistributedOn("a")) + .global(singleStreamPartition()) .build()) .add(builder() .global(arbitraryPartition()) diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlansWithFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlansWithFunctions.java index ee2bcf0316a61..d00291a229e98 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlansWithFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestAddExchangesPlansWithFunctions.java @@ -36,6 +36,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.testing.LocalQueryRunner; import com.facebook.presto.tpch.TpchConnectorFactory; import com.facebook.presto.type.BigintOperators; @@ -47,12 +48,16 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static com.facebook.presto.SystemSessionProperties.REMOTE_FUNCTIONS_ENABLED; +import static com.facebook.presto.SystemSessionProperties.REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM; +import static com.facebook.presto.SystemSessionProperties.SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.operator.scalar.annotations.ScalarFromAnnotationsParser.parseFunctionDefinitions; import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; import static com.facebook.presto.spi.function.RoutineCharacteristics.Determinism.DETERMINISTIC; import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.JAVA; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.PYTHON; import static com.facebook.presto.spi.function.RoutineCharacteristics.NullCallClause.RETURNS_NULL_ON_NULL_INPUT; import static com.facebook.presto.spi.plan.JoinType.INNER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -66,6 +71,7 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Scope.REMOTE_STREAMING; import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.GATHER; +import static com.facebook.presto.sql.planner.plan.ExchangeNode.Type.REPARTITION; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; /** @@ -143,6 +149,34 @@ public TestAddExchangesPlansWithFunctions() "", notVersioned()); + // External/Remote functions using PYTHON language mapped to THRIFT implementation type for testing REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM + private static final SqlInvokedFunction REMOTE_FOO = new SqlInvokedFunction( + new QualifiedObjectName("dummy", "unittest", "remote_foo"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.BIGINT), + "remote_foo(x)", + RoutineCharacteristics.builder().setLanguage(PYTHON).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), + "", + notVersioned()); + + private static final SqlInvokedFunction REMOTE_BAR = new SqlInvokedFunction( + new QualifiedObjectName("dummy", "unittest", "remote_bar"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.BIGINT), + "remote_bar(x)", + RoutineCharacteristics.builder().setLanguage(PYTHON).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), + "", + notVersioned()); + + private static final SqlInvokedFunction REMOTE_BAZ = new SqlInvokedFunction( + new QualifiedObjectName("dummy", "unittest", "remote_baz"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.BIGINT), + "remote_baz(x)", + RoutineCharacteristics.builder().setLanguage(PYTHON).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), + "", + notVersioned()); + private static LocalQueryRunner createTestQueryRunner() { LocalQueryRunner queryRunner = new LocalQueryRunner(testSessionBuilder() @@ -160,15 +194,19 @@ private static LocalQueryRunner createTestQueryRunner() new SqlFunctionExecutors( ImmutableMap.of( CPP, FunctionImplementationType.CPP, - JAVA, FunctionImplementationType.JAVA), + JAVA, FunctionImplementationType.JAVA, + PYTHON, FunctionImplementationType.THRIFT), new NoopSqlFunctionExecutor()), - new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("cpp"))); + new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("cpp,python"))); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(CPP_FOO, true); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(CPP_BAZ, true); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(JAVA_BAR, true); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(JAVA_FEE, true); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(NOT, true); queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(CPP_ARRAY_CONSTRUCTOR, true); + queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(REMOTE_FOO, true); + queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(REMOTE_BAR, true); + queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(REMOTE_BAZ, true); parseFunctionDefinitions(BigintOperators.class).stream() .map(TestAddExchangesPlansWithFunctions::convertToSqlInvokedFunction) .forEach(function -> queryRunner.getMetadata().getFunctionAndTypeManager().createFunction(function, true)); @@ -669,7 +707,7 @@ public void testSystemTableFilterWithMultipleColumnsAndPartialSelection() "WHERE cpp_foo(ordinal_position) > 0 AND cpp_baz(ordinal_position) < 100", output( project(ImmutableMap.of("table_schema", expression("table_schema"), - "table_name", expression("table_name")), + "table_name", expression("table_name")), filter("cpp_foo(ordinal_position) > BIGINT'0' AND cpp_baz(ordinal_position) < BIGINT'100'", exchange(REMOTE_STREAMING, GATHER, tableScan("columns", ImmutableMap.of( @@ -678,6 +716,191 @@ public void testSystemTableFilterWithMultipleColumnsAndPartialSelection() "table_name", "table_name"))))))); } + @Test + public void testRemoteFunctionNamesForFixedParallelismWithExactMatch() + { + // Test that REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM with exact function name + // causes round-robin exchanges to be added before and after the remote project + // Note: The function must be an external function (isExternalExecution() = true) for this feature to work + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_foo") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithRegexWildcard() + { + // Test that regex pattern with wildcard matches function names + // remote_foo matches the pattern "remote_.*" + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_.*") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithNonMatchingRegex() + { + // Test that when the regex doesn't match the function name, + // no extra round-robin exchanges are added + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "nonmatching_.*") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + tableScan("nation", ImmutableMap.of("nationkey", "nationkey")))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithEmptyString() + { + // Test that empty string means the feature is disabled (no extra exchanges) + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + tableScan("nation", ImmutableMap.of("nationkey", "nationkey")))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithMultipleFunctions() + { + // Test regex that matches multiple function names using OR pattern + // Both remote_foo and remote_baz should trigger the exchange insertion + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey), remote_baz(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_(foo|baz)") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of( + "remote_foo", expression("remote_foo(nationkey)"), + "remote_baz", expression("remote_baz(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithPartialMatch() + { + // Test that regex requires full match (not partial) by using anchored pattern + // "remote_f" should NOT match "remote_foo" because matches() requires full string match + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_f") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + tableScan("nation", ImmutableMap.of("nationkey", "nationkey")))))); + } + + @Test + public void testRemoteFunctionFixedParallelismSkipsTrailingExchangeWhenUndistributed() + { + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation LIMIT 1", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_foo") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + output( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + anyTree( + tableScan("nation", ImmutableMap.of("nationkey", "nationkey")))))))); + + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_foo") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + } + + @Test + public void testRemoteFunctionNamesForFixedParallelismWithComplexRegex() + { + // Test complex regex pattern with character classes + assertNativeDistributedPlanWithSession( + "SELECT remote_foo(nationkey) FROM nation", + testSessionBuilder() + .setCatalog("tpch") + .setSchema("tiny") + .setSystemProperty(REMOTE_FUNCTION_NAMES_FOR_FIXED_PARALLELISM, "dummy.unittest.remote_[a-z]+") + .setSystemProperty(REMOTE_FUNCTIONS_ENABLED, "true") + .setSystemProperty(SKIP_PUSHDOWN_THROUGH_EXCHANGE_FOR_REMOTE_PROJECTION, "true") + .build(), + anyTree( + exchange(REMOTE_STREAMING, GATHER, + project(ImmutableMap.of("remote_foo", expression("remote_foo(nationkey)")), + exchange(REMOTE_STREAMING, REPARTITION, + tableScan("nation", ImmutableMap.of("nationkey", "nationkey"))))))); + } + + private void assertNativeDistributedPlanWithSession(String sql, com.facebook.presto.Session session, PlanMatchPattern pattern) + { + assertDistributedPlan(sql, session, pattern); + } + private static class NoOpExpressionOptimizerFactory implements ExpressionOptimizerFactory { diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java index 57550cc78755b..41d00d17568f9 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestEliminateSorts.java @@ -141,7 +141,7 @@ public void assertUnitPlan(@Language("SQL") String sql, PlanMatchPattern pattern getQueryRunner().getCostCalculator(), ImmutableSet.of(new RemoveRedundantIdentityProjections())), new AddExchanges(getQueryRunner().getMetadata(), new PartitioningProviderManager(), false), - new AddLocalExchanges(getMetadata(), false), + new AddLocalExchanges(getMetadata(), getQueryRunner().getStatsCalculator(), false), new UnaliasSymbolReferences(getMetadata().getFunctionAndTypeManager()), new PruneUnreferencedOutputs(), new IterativeOptimizer( diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestPartitioningUtils.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestPartitioningUtils.java new file mode 100644 index 0000000000000..2946ce5f11dad --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestPartitioningUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.spi.plan.Partitioning; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.COORDINATOR_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; +import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION; +import static com.facebook.presto.sql.planner.optimizations.PartitioningUtils.isPartitionedOn; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestPartitioningUtils +{ + @Test + public void testIsPartitionedOnEmptyArgumentsSingleDistribution() + { + Partitioning partitioning = Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsCoordinatorDistribution() + { + Partitioning partitioning = Partitioning.create(COORDINATOR_DISTRIBUTION, ImmutableList.of()); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsHashDistribution() + { + Partitioning partitioning = Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.of()); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsBroadcastDistribution() + { + Partitioning partitioning = Partitioning.create(FIXED_BROADCAST_DISTRIBUTION, ImmutableList.of()); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsArbitraryDistribution() + { + Partitioning partitioning = Partitioning.create(FIXED_ARBITRARY_DISTRIBUTION, ImmutableList.of()); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsSourceDistribution() + { + Partitioning partitioning = Partitioning.create(SOURCE_DISTRIBUTION, ImmutableList.of()); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsScaledWriterDistribution() + { + Partitioning partitioning = Partitioning.create(SCALED_WRITER_DISTRIBUTION, ImmutableList.of()); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnWithMatchingColumns() + { + VariableReferenceExpression column = new VariableReferenceExpression(Optional.empty(), "col", BIGINT); + Partitioning partitioning = Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.of(column)); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(column), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnWithNonMatchingColumns() + { + VariableReferenceExpression column1 = new VariableReferenceExpression(Optional.empty(), "col1", BIGINT); + VariableReferenceExpression column2 = new VariableReferenceExpression(Optional.empty(), "col2", BIGINT); + Partitioning partitioning = Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.of(column1)); + assertFalse(isPartitionedOn(partitioning, ImmutableList.of(column2), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnWithKnownConstants() + { + VariableReferenceExpression column = new VariableReferenceExpression(Optional.empty(), "col", BIGINT); + Partitioning partitioning = Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.of(column)); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(), ImmutableSet.of(column))); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsSingleDistributionWithColumns() + { + VariableReferenceExpression column = new VariableReferenceExpression(Optional.empty(), "col", BIGINT); + Partitioning partitioning = Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(column), ImmutableSet.of())); + } + + @Test + public void testIsPartitionedOnEmptyArgumentsCoordinatorDistributionWithColumns() + { + VariableReferenceExpression column = new VariableReferenceExpression(Optional.empty(), "col", BIGINT); + Partitioning partitioning = Partitioning.create(COORDINATOR_DISTRIBUTION, ImmutableList.of()); + assertTrue(isPartitionedOn(partitioning, ImmutableList.of(column), ImmutableSet.of())); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java index 6d0a3c59959c7..275d32ee0f89e 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestReplaceConstantVariableReferencesWithConstants.java @@ -14,16 +14,28 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.cost.VariableStatsEstimate; +import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.JoinType; +import com.facebook.presto.spi.plan.OutputNode; +import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.facebook.presto.sql.planner.assertions.MatchResult; +import com.facebook.presto.sql.planner.assertions.Matcher; import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; +import com.facebook.presto.sql.planner.assertions.SymbolAliases; import com.facebook.presto.sql.planner.iterative.rule.test.RuleTester; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.util.Map; + +import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_THROUGH_UNNEST; import static com.facebook.presto.SystemSessionProperties.REWRITE_EXPRESSION_WITH_CONSTANT_EXPRESSION; import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; @@ -48,10 +60,21 @@ import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.sql.tree.SortItem.NullOrdering.LAST; import static com.facebook.presto.sql.tree.SortItem.Ordering.ASCENDING; +import static org.testng.Assert.assertEquals; public class TestReplaceConstantVariableReferencesWithConstants extends BasePlanTest { + @DataProvider + public static Object[][] filterProviders() + { + return new Object[][] { + {"3-MEDIUM"}, + {"2-LOOOOOOONG"}, + {"123456789012345"} // orderpriority is a VARCHAR(15), this is the max length after which a Constant replacement is not applied + }; + } + private Session enableOptimization() { return Session.builder(this.getQueryRunner().getDefaultSession()) @@ -94,6 +117,23 @@ public void testUnnest() tableScan("orders", ImmutableMap.of("orderkey", "orderkey", "orderpriority", "orderpriority")))))))); } + @Test + public void testUnnestWithPushdownThroughUnnest() + { + Session session = Session.builder(this.getQueryRunner().getDefaultSession()) + .setSystemProperty(REWRITE_EXPRESSION_WITH_CONSTANT_EXPRESSION, "true") + .setSystemProperty(PUSHDOWN_THROUGH_UNNEST, "true") + .build(); + assertPlan("select orderkey, orderpriority, idx from orders cross join unnest(array[1, 2]) t(idx) where orderpriority='3-MEDIUM'", + session, + output( + unnest( + anyTree( + filter( + "orderpriority = '3-MEDIUM'", + tableScan("orders", ImmutableMap.of("orderkey", "orderkey", "orderpriority", "orderpriority"))))))); + } + @Test public void testInnerJoin() { @@ -179,18 +219,46 @@ public void testSemiJoin() tableScan("lineitem", ImmutableMap.of("orderkey_1", "orderkey")))))))))); } - @Test - public void testSimpleFilter() + @Test(dataProvider = "filterProviders") + public void testSimpleFilter(String filter) { - assertPlan("select orderkey, orderpriority from orders where orderpriority='3-MEDIUM'", + assertPlan("select orderkey, orderpriority from orders where orderpriority='" + filter + "'", enableOptimization(), output( ImmutableList.of("orderkey", "expr_6"), project( - ImmutableMap.of("expr_6", expression("'3-MEDIUM'")), + ImmutableMap.of("expr_6", expression("'" + filter + "'")), filter( - "orderpriority = '3-MEDIUM'", - tableScan("orders", ImmutableMap.of("orderkey", "orderkey", "orderpriority", "orderpriority")))))); + "orderpriority = '" + filter + "'", + tableScan("orders", ImmutableMap.of("orderkey", "orderkey", "orderpriority", "orderpriority"))))) + .with(new Matcher() + { + @Override + public boolean shapeMatches(PlanNode node) + { + return node instanceof OutputNode; + } + + @Override + public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session session, Metadata metadata, SymbolAliases symbolAliases) + { + // Assert additionally ont size estimate of the replaced VARCHAR ConstantExpression + VariableStatsEstimate expectedStats = VariableStatsEstimate.builder() + .setAverageRowSize(filter.length()) + .setDistinctValuesCount(1) + .setNullsFraction(0.0) + .build(); + + VariableStatsEstimate actualStats = stats.getStats(node).getVariableStatistics().entrySet().stream() + .filter(es -> es.getKey().getName().equals("orderpriority")) + .map(Map.Entry::getValue) + .findFirst() + .orElseThrow(() -> new AssertionError("Variable 'orderpriority' not found in statistics")); + + assertEquals(actualStats, expectedStats); + return MatchResult.match(); + } + })); } @Test diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSymbolMapper.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSymbolMapper.java new file mode 100644 index 0000000000000..6dbe919638901 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestSymbolMapper.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.optimizations; + +import com.facebook.presto.common.type.BigintType; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertSame; + +public class TestSymbolMapper +{ + @Test + public void testBasicVariableMapping() + { + VariableReferenceExpression varA = new VariableReferenceExpression(Optional.empty(), "a", BigintType.BIGINT); + VariableReferenceExpression varB = new VariableReferenceExpression(Optional.empty(), "b", BigintType.BIGINT); + + SymbolMapper mapper = new SymbolMapper(ImmutableMap.of(varA, varB), WarningCollector.NOOP); + + VariableReferenceExpression mapped = mapper.map(varA); + assertEquals(mapped.getName(), "b"); + assertEquals(mapped.getType(), BigintType.BIGINT); + } + + @Test + public void testUnmappedVariableReturnsSameInstance() + { + VariableReferenceExpression varA = new VariableReferenceExpression(Optional.empty(), "a", BigintType.BIGINT); + VariableReferenceExpression varB = new VariableReferenceExpression(Optional.empty(), "b", BigintType.BIGINT); + + SymbolMapper mapper = new SymbolMapper(ImmutableMap.of(varA, varB), WarningCollector.NOOP); + + VariableReferenceExpression varC = new VariableReferenceExpression(Optional.empty(), "c", BigintType.BIGINT); + VariableReferenceExpression mapped = mapper.map(varC); + assertSame(mapped, varC); + } + + @Test + public void testTransitiveMapping() + { + VariableReferenceExpression varA = new VariableReferenceExpression(Optional.empty(), "a", BigintType.BIGINT); + VariableReferenceExpression varB = new VariableReferenceExpression(Optional.empty(), "b", BigintType.BIGINT); + VariableReferenceExpression varC = new VariableReferenceExpression(Optional.empty(), "c", BigintType.BIGINT); + + // A -> B -> C should resolve A to C + Map mapping = new HashMap<>(); + mapping.put(varA, varB); + mapping.put(varB, varC); + + SymbolMapper mapper = new SymbolMapper(mapping, WarningCollector.NOOP); + + VariableReferenceExpression mapped = mapper.map(varA); + assertEquals(mapped.getName(), "c"); + } + + @Test + public void testMapList() + { + VariableReferenceExpression varA = new VariableReferenceExpression(Optional.empty(), "a", BigintType.BIGINT); + VariableReferenceExpression varB = new VariableReferenceExpression(Optional.empty(), "b", BigintType.BIGINT); + VariableReferenceExpression varX = new VariableReferenceExpression(Optional.empty(), "x", VarcharType.VARCHAR); + VariableReferenceExpression varY = new VariableReferenceExpression(Optional.empty(), "y", VarcharType.VARCHAR); + + Map mapping = ImmutableMap.of(varA, varB, varX, varY); + SymbolMapper mapper = new SymbolMapper(mapping, WarningCollector.NOOP); + + List result = mapper.map(List.of(varA, varX)); + assertEquals(result.size(), 2); + assertEquals(result.get(0).getName(), "b"); + assertEquals(result.get(1).getName(), "y"); + } + + @Test + public void testBuilderPattern() + { + VariableReferenceExpression varA = new VariableReferenceExpression(Optional.empty(), "a", BigintType.BIGINT); + VariableReferenceExpression varB = new VariableReferenceExpression(Optional.empty(), "b", BigintType.BIGINT); + + SymbolMapper.Builder builder = SymbolMapper.builder(); + builder.put(varA, varB); + SymbolMapper mapper = builder.build(); + + VariableReferenceExpression mapped = mapper.map(varA); + assertEquals(mapped.getName(), "b"); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestWindowFilterPushDown.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestWindowFilterPushDown.java index 3d0b32fceeb34..3719322bb8922 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestWindowFilterPushDown.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/optimizations/TestWindowFilterPushDown.java @@ -15,12 +15,14 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.sql.planner.assertions.BasePlanTest; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import org.intellij.lang.annotations.Language; import org.testng.annotations.Test; +import static com.facebook.presto.SystemSessionProperties.NATIVE_EXECUTION_ENABLED; +import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_TOP_N_RANK; import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_TOP_N_ROW_NUMBER; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; @@ -31,15 +33,11 @@ public class TestWindowFilterPushDown extends BasePlanTest { - @Test - public void testLimitAboveWindow() + private void testLimitSql(String sql, boolean rowNumber) { - @Language("SQL") String sql = "SELECT " + - "row_number() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem LIMIT 10"; - assertPlanWithSession( sql, - optimizeTopNRowNumber(true), + rowNumber ? optimizeTopNRowNumber(true) : optimizeTopNRank(true), true, anyTree( limit(10, anyTree( @@ -49,25 +47,47 @@ public void testLimitAboveWindow() assertPlanWithSession( sql, - optimizeTopNRowNumber(false), + rowNumber ? optimizeTopNRowNumber(false) : optimizeTopNRank(false), true, anyTree( limit(10, anyTree( node(WindowNode.class, anyTree( tableScan("lineitem"))))))); - } + if (!rowNumber) { + assertPlanWithSession( + sql, + optimizeTopNRankWithoutNative(true), + true, + anyTree( + limit(10, anyTree( + node(WindowNode.class, + anyTree( + tableScan("lineitem"))))))); + } + } @Test - public void testFilterAboveWindow() + public void testLimitAboveWindow() { - @Language("SQL") String sql = "SELECT * FROM " + - "(SELECT row_number() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem) " + - "WHERE partition_row_number < 10"; + @Language("SQL") String sql = "SELECT " + + "row_number() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem LIMIT 10"; + testLimitSql(sql, true); + + sql = "SELECT " + + "rank() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem LIMIT 10"; + testLimitSql(sql, false); + sql = "SELECT " + + "dense_rank() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem LIMIT 10"; + testLimitSql(sql, false); + } + + private void testFilterSql(String sql, boolean rowNumber) + { assertPlanWithSession( sql, - optimizeTopNRowNumber(true), + rowNumber ? optimizeTopNRowNumber(true) : optimizeTopNRank(true), true, anyTree( anyNot(FilterNode.class, @@ -77,7 +97,7 @@ public void testFilterAboveWindow() assertPlanWithSession( sql, - optimizeTopNRowNumber(false), + rowNumber ? optimizeTopNRowNumber(false) : optimizeTopNRank(false), true, anyTree( node(FilterNode.class, @@ -85,6 +105,38 @@ public void testFilterAboveWindow() node(WindowNode.class, anyTree( tableScan("lineitem"))))))); + + if (!rowNumber) { + assertPlanWithSession( + sql, + optimizeTopNRankWithoutNative(true), + true, + anyTree( + node(FilterNode.class, + anyTree( + node(WindowNode.class, + anyTree( + tableScan("lineitem"))))))); + } + } + @Test + public void testFilterAboveWindow() + { + @Language("SQL") String sql = "SELECT * FROM " + + "(SELECT row_number() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_row_number FROM lineitem) " + + "WHERE partition_row_number < 10"; + + testFilterSql(sql, true); + + sql = "SELECT * FROM " + + "(SELECT rank() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_rank FROM lineitem) " + + "WHERE partition_rank < 10"; + testFilterSql(sql, false); + + sql = "SELECT * FROM " + + "(SELECT dense_rank() OVER (PARTITION BY suppkey ORDER BY orderkey) partition_dense_rank FROM lineitem) " + + "WHERE partition_dense_rank < 10"; + testFilterSql(sql, false); } private Session optimizeTopNRowNumber(boolean enabled) @@ -93,4 +145,20 @@ private Session optimizeTopNRowNumber(boolean enabled) .setSystemProperty(OPTIMIZE_TOP_N_ROW_NUMBER, Boolean.toString(enabled)) .build(); } + + private Session optimizeTopNRank(boolean enabled) + { + return Session.builder(this.getQueryRunner().getDefaultSession()) + .setSystemProperty(NATIVE_EXECUTION_ENABLED, Boolean.toString(enabled)) + .setSystemProperty(OPTIMIZE_TOP_N_RANK, Boolean.toString(enabled)) + .build(); + } + + private Session optimizeTopNRankWithoutNative(boolean enabled) + { + return Session.builder(this.getQueryRunner().getDefaultSession()) + .setSystemProperty(NATIVE_EXECUTION_ENABLED, Boolean.toString(false)) + .setSystemProperty(OPTIMIZE_TOP_N_RANK, Boolean.toString(enabled)) + .build(); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java index d19e911fc2f60..8ba19fb23f05f 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestCheckUnsupportedPrestissimoTypes.java @@ -31,11 +31,11 @@ import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.assignment; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; public class TestCheckUnsupportedPrestissimoTypes extends BasePlanTest diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestPlanPrinterSmoke.java b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestPlanPrinterSmoke.java new file mode 100644 index 0000000000000..05b19c949ea41 --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/planner/sanity/TestPlanPrinterSmoke.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.sanity; + +import com.facebook.presto.cost.CachingCostProvider; +import com.facebook.presto.cost.CachingStatsProvider; +import com.facebook.presto.cost.CostProvider; +import com.facebook.presto.cost.StatsAndCosts; +import com.facebook.presto.cost.StatsProvider; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.planner.Plan; +import com.facebook.presto.sql.planner.assertions.BasePlanTest; +import com.facebook.presto.sql.planner.planPrinter.PlanPrinter; +import com.facebook.presto.testing.LocalQueryRunner; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestPlanPrinterSmoke + extends BasePlanTest +{ + @Test + public void testLogicalPlanTextSizeEstimates() + { + String testSql = "SELECT \n" + + " *\n" + + "FROM \n" + + " supplier s,\n" + + " lineitem l1,\n" + + " orders o,\n" + + " nation n\n" + + "WHERE \n" + + " s.suppkey = l1.suppkey \n" + + " AND o.orderkey = l1.orderkey\n" + + " AND s.nationkey = n.nationkey \n" + + "\n"; + try (LocalQueryRunner localQueryRunner = createQueryRunner(ImmutableMap.of())) { + localQueryRunner.inTransaction(localQueryRunner.getDefaultSession(), transactionSession -> { + Plan actualPlan = localQueryRunner.createPlan( + transactionSession, + testSql, + WarningCollector.NOOP); + + StatsProvider statsProvider = new CachingStatsProvider(localQueryRunner.getStatsCalculator(), transactionSession, actualPlan.getTypes()); + CostProvider costProvider = new CachingCostProvider(localQueryRunner.getEstimatedExchangesCostCalculator(), statsProvider, transactionSession); + + String textLogicalPlan = PlanPrinter.textLogicalPlan(actualPlan.getRoot(), + actualPlan.getTypes(), + StatsAndCosts.create(actualPlan.getRoot(), statsProvider, costProvider, transactionSession), + localQueryRunner.getFunctionAndTypeManager(), + transactionSession, + 1); + + // `nation` scan + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 25 (2.89kB), cpu: 2,734.00, memory: 0.00, network: 0.00}"); + // `supplier` scan + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 100 (17.14kB), cpu: 16,652.00, memory: 0.00, network: 0.00}"); + // `orders` scan + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 15,000 (1.99MB), cpu: 1,948,552.00, memory: 0.00, network: 0.00}"); + // `lineitem` scan + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 60,175 (9.29MB), cpu: 9,197,910.00, memory: 0.00, network: 0.00}"); + + // JOINs + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 60,175 (15.71MB), cpu: 53,349,364.11, memory: 2,083,552.00, network: 2,083,552.00}"); + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 60,175 (30.51MB), cpu: 119,543,090.43, memory: 2,114,099.00, network: 2,114,099.00}"); + assertThat(textLogicalPlan).contains("Estimates: {source: CostBasedSourceInfo, rows: 100 (26.06kB), cpu: 90,055.00, memory: 2,959.00, network: 2,959.00}"); + + return null; + }); + } + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java index 7c1d54dbfad49..b83540d2c05c4 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressOperators.java @@ -25,9 +25,9 @@ import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.io.BaseEncoding.base16; public class TestIpAddressOperators diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java index cacd4b3ad252c..39a4ec375d253 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpAddressType.java @@ -20,7 +20,7 @@ import io.airlift.slice.Slices; import org.testng.annotations.Test; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; import static org.testng.Assert.assertEquals; public class TestIpAddressType diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java index cbd9dff770e28..77f3280787539 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixOperators.java @@ -24,9 +24,9 @@ import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; import static java.lang.System.arraycopy; public class TestIpPrefixOperators diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java index 50b5eb672708a..7a3a6d94337d7 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/TestIpPrefixType.java @@ -20,7 +20,7 @@ import io.airlift.slice.Slices; import org.testng.annotations.Test; -import static com.facebook.presto.type.IpPrefixType.IPPREFIX; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.google.common.base.Preconditions.checkState; import static java.lang.System.arraycopy; import static org.testng.Assert.assertEquals; diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java new file mode 100644 index 0000000000000..b3ade52608e85 --- /dev/null +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestIpPrefix.java @@ -0,0 +1,324 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.operator.scalar; + +import com.facebook.presto.common.type.ArrayType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; + +public interface AbstractTestIpPrefix + extends TestFunctions +{ + @DataProvider(name = "public-ip-provider") + default Object[] publicIpProvider() + { + return new Object[] { + "6.7.8.9", + "157.240.200.99", + "8.8.8.8", + "128.1.2.8", + "2a03:2880:f031:12:face:b00c:0:2", + "2600:1406:6c00::173c:ad43", + "2607:f8b0:4007:818::2004" + }; + } + + @DataProvider(name = "private-ip-provider") + default Object[][] privateIpProvider() + { + return new Object[][] { + // The first and last IP address in each private range + {"0.0.0.0"}, {"0.255.255.255"}, // 0.0.0.0/8 RFC1122: "This host on this network" + {"10.0.0.0"}, {"10.255.255.255"}, // 10.0.0.0/8 RFC1918: Private-Use + {"100.64.0.0"}, {"100.127.255.255"}, // 100.64.0.0/10 RFC6598: Shared Address Space + {"127.0.0.0"}, {"127.255.255.255"}, // 127.0.0.0/8 RFC1122: Loopback + {"169.254.0.0"}, {"169.254.255.255"}, // 169.254.0.0/16 RFC3927: Link Local + {"172.16.0.0"}, {"172.31.255.255"}, // 172.16.0.0/12 RFC1918: Private-Use + {"192.0.0.0"}, {"192.0.0.255"}, // 192.0.0.0/24 RFC6890: IETF Protocol Assignments + {"192.0.2.0"}, {"192.0.2.255"}, // 192.0.2.0/24 RFC5737: Documentation (TEST-NET-1) + {"192.88.99.0"}, {"192.88.99.255"}, // 192.88.99.0/24 RFC3068: 6to4 Relay anycast + {"192.168.0.0"}, {"192.168.255.255"}, // 192.168.0.0/16 RFC1918: Private-Use + {"198.18.0.0"}, {"198.19.255.255"}, // 198.18.0.0/15 RFC2544: Benchmarking + {"198.51.100.0"}, {"198.51.100.255"}, // 198.51.100.0/24 RFC5737: Documentation (TEST-NET-2) + {"203.0.113.0"}, {"203.0.113.255"}, // 203.0.113.0/24 RFC5737: Documentation (TEST-NET-3) + {"240.0.0.0"}, {"255.255.255.255"}, // 240.0.0.0/4 RFC1112: Reserved + {"::"}, {"::"}, // ::/128 RFC4291: Unspecified address + {"::1"}, {"::1"}, // ::1/128 RFC4291: Loopback address + {"100::"}, {"100::ffff:ffff:ffff:ffff"}, // 100::/64 RFC6666: Discard-Only Address Block + {"64:ff9b:1::"}, {"64:ff9b:1:ffff:ffff:ffff:ffff:ffff"}, // 64:ff9b:1::/48 RFC8215: IPv4-IPv6 Translation + {"2001:2::"}, {"2001:2:0:ffff:ffff:ffff:ffff:ffff"}, // 2001:2::/48 RFC5180,RFC Errata 1752: Benchmarking + {"2001:db8::"}, {"2001:db8:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001:db8::/32 RFC3849: Documentation + {"2001::"}, {"2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 2001::/23 RFC2928: IETF Protocol Assignments + {"5f00::"}, {"5f00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // 5f00::/16 RFC-ietf-6man-sids-06: Segment Routing (SRv6) + {"fe80::"}, {"febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fe80::/10 RFC4291: Link-Local Unicast + {"fc00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, // fc00::/7 RFC4193, RFC8190: Unique Local + // some IPs in the middle of ranges + {"10.1.2.3"}, + {"100.64.3.2"}, + {"192.168.55.99"}, + {"2001:0DB8:0000:0000:face:b00c:0000:0000"}, + {"0100:0000:0000:0000:ffff:ffff:0000:0000"} + }; + } + + @Test + default void testIpAddressIpPrefix() + { + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX(IPADDRESS '1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX(IPADDRESS '64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX(IPADDRESS '64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + } + + @Test + default void testStringIpPrefix() + { + assertFunction("IP_PREFIX('1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('1.2.3.4', 32)", IPPREFIX, "1.2.3.4/32"); + assertFunction("IP_PREFIX('1.2.3.4', 0)", IPPREFIX, "0.0.0.0/0"); + assertFunction("IP_PREFIX('::ffff:1.2.3.4', 24)", IPPREFIX, "1.2.3.0/24"); + assertFunction("IP_PREFIX('64:ff9b::17', 64)", IPPREFIX, "64:ff9b::/64"); + assertFunction("IP_PREFIX('64:ff9b::17', 127)", IPPREFIX, "64:ff9b::16/127"); + assertFunction("IP_PREFIX('64:ff9b::17', 128)", IPPREFIX, "64:ff9b::17/128"); + assertFunction("IP_PREFIX('64:ff9b::17', 0)", IPPREFIX, "::/0"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', -1)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('::ffff:1.2.3.4', 33)", "IPv4 subnet size must be in range [0, 32]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', -1)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidFunction("IP_PREFIX('64:ff9b::10', 129)", "IPv6 subnet size must be in range [0, 128]"); + assertInvalidCast("IP_PREFIX('localhost', 24)", "Cannot cast value to IPADDRESS: localhost"); + assertInvalidCast("IP_PREFIX('64::ff9b::10', 24)", "Cannot cast value to IPADDRESS: 64::ff9b::10"); + assertInvalidCast("IP_PREFIX('64:face:book::10', 24)", "Cannot cast value to IPADDRESS: 64:face:book::10"); + assertInvalidCast("IP_PREFIX('123.456.789.012', 24)", "Cannot cast value to IPADDRESS: 123.456.789.012"); + } + + @Test + default void testIpSubnetMin() + { + assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/24')", IPADDRESS, "1.2.3.0"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '1.2.3.4/32')", IPADDRESS, "1.2.3.4"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/127')", IPADDRESS, "64:ff9b::16"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/128')", IPADDRESS, "64:ff9b::17"); + assertFunction("IP_SUBNET_MIN(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "::"); + } + + @Test + default void testIpSubnetMax() + { + assertFunction("IP_SUBNET_MAX(IPPREFIX '1.2.3.128/26')", IPADDRESS, "1.2.3.191"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '192.168.128.4/32')", IPADDRESS, "192.168.128.4"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '10.1.16.3/9')", IPADDRESS, "10.127.255.255"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/127')", IPADDRESS, "2001:db8::17"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '2001:db8::16/128')", IPADDRESS, "2001:db8::16"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/64')", IPADDRESS, "64:ff9b::ffff:ffff:ffff:ffff"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/72')", IPADDRESS, "64:ff9b::ff:ffff:ffff:ffff"); + assertFunction("IP_SUBNET_MAX(IPPREFIX '64:ff9b::17/0')", IPADDRESS, "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + } + + @Test + default void testIpSubnetRange() + { + assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.160/24')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.0", "1.2.3.255")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '1.2.3.128/31')", new ArrayType(IPADDRESS), ImmutableList.of("1.2.3.128", "1.2.3.129")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/32')", new ArrayType(IPADDRESS), ImmutableList.of("10.1.6.46", "10.1.6.46")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '10.1.6.46/0')", new ArrayType(IPADDRESS), ImmutableList.of("0.0.0.0", "255.255.255.255")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/64')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::", "64:ff9b::ffff:ffff:ffff:ffff")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::52f4/120')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::5200", "64:ff9b::52ff")); + assertFunction("IP_SUBNET_RANGE(IPPREFIX '64:ff9b::17/128')", new ArrayType(IPADDRESS), ImmutableList.of("64:ff9b::17", "64:ff9b::17")); + } + + @Test + default void testIsSubnetOf() + { + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.3.129')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPADDRESS '1.2.5.1')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/32', IPADDRESS '1.2.3.128')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/0', IPADDRESS '192.168.5.1')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ff9b::ffff:ff')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPADDRESS '64:ffff::17')", BOOLEAN, false); + + assertFunction("IS_SUBNET_OF(IPPREFIX '192.168.3.131/26', IPPREFIX '192.168.3.144/30')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.5.1/30')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '1.2.3.128/26', IPPREFIX '1.2.3.128/26')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ff9b::ff:25/80')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '64:ff9b::17/64', IPPREFIX '64:ffff::17/64')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/37', IPPREFIX '2804:431:b000::/38')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '2804:431:b000::/38', IPPREFIX '2804:431:b000::/37')", BOOLEAN, false); + assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/22', IPPREFIX '170.0.52.0/24')", BOOLEAN, true); + assertFunction("IS_SUBNET_OF(IPPREFIX '170.0.52.0/24', IPPREFIX '170.0.52.0/22')", BOOLEAN, false); + } + + @Test + default void testIpv4PrefixCollapse() + { + // simple + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.1.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/23")); + + // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.1.0/24', IPPREFIX '192.168.0.0/24', IPPREFIX '192.168.2.0/24', IPPREFIX '192.168.9.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/23", "192.168.2.0/24", "192.168.9.0/24")); + } + + @Test + default void testIpv6PrefixCollapse() + { + // simple + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090::/48', IPPREFIX '2620:10d:c091::/48'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2620:10d:c090::/47")); + + // unsorted input, 1 adjacent prefix that cannot be aggregated, and one disjoint. + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2804:13c:4d6:e200::/56', IPPREFIX '2804:13c:4d6:dd00::/56', IPPREFIX '2804:13c:4d6:dc00::/56', IPPREFIX '2804:13c:4d6:de00::/56'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2804:13c:4d6:dc00::/55", "2804:13c:4d6:de00::/56", "2804:13c:4d6:e200::/56")); + } + + @Test + default void testIpPrefixCollapseIpv4SingleIPs() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.1/32', IPPREFIX '192.168.33.1/32'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.1/32", "192.168.33.1/32")); + } + + @Test + default void testIpPrefixCollapseIpv6SingleIPs() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '2620:10d:c090:400::5:a869/128', IPPREFIX '2620:10d:c091:400::5:a869/128'])", + new ArrayType(IPPREFIX), + ImmutableList.of("2620:10d:c090:400::5:a869/128", "2620:10d:c091:400::5:a869/128")); + } + + @Test + default void testIpPrefixCollapseSinglePrefixReturnsSamePrefix() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + } + + @Test + default void testIpPrefixCollapseOverlappingPrefixes() + { + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.0.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.2.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '192.168.3.0/24'])", + new ArrayType(IPPREFIX), + ImmutableList.of("192.168.0.0/22")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.64.0/18', IPPREFIX '10.2.0.0/15', IPPREFIX '10.0.0.0/8', IPPREFIX '11.0.0.0/8', IPPREFIX '172.168.32.0/20', IPPREFIX '172.168.0.0/18'])", + new ArrayType(IPPREFIX), + ImmutableList.of("10.0.0.0/7", "172.168.0.0/18")); + assertFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '10.0.0.0/8', IPPREFIX '10.0.0.0/7'])", + new ArrayType(IPPREFIX), + ImmutableList.of("10.0.0.0/7")); + } + + @Test + default void testIpPrefixCollapseEmptyArrayInput() + { + assertFunction("IP_PREFIX_COLLAPSE(CAST(ARRAY[] AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), ImmutableList.of()); + } + + @Test + default void testIpPrefixCollapseNullInput() + { + assertFunction("IP_PREFIX_COLLAPSE(CAST(NULL AS ARRAY(IPPREFIX)))", new ArrayType(IPPREFIX), null); + } + + @Test + default void testIpPrefixCollapseNoNullPrefixesError() + { + assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', CAST(NULL AS IPPREFIX)])", + "ip_prefix_collapse does not support null elements"); + } + + @Test + default void testIpPrefixCollapseMixedIpVersionError() + { + assertInvalidFunction("IP_PREFIX_COLLAPSE(ARRAY[IPPREFIX '192.168.0.0/22', IPPREFIX '2409:4043:251a:d200::/56'])", + "All IPPREFIX elements must be the same IP version."); + } + + @Test (dataProvider = "private-ip-provider") + default void testIsPrivateTrue(String ipAddress) + { + assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, true); + } + + @Test (dataProvider = "public-ip-provider") + default void testIsPrivateIpFalse(String ipAddress) + { + assertFunction("IS_PRIVATE_IP(IPADDRESS '" + ipAddress + "')", BOOLEAN, false); + } + + @Test + default void testIsPrivateIpNull() + { + assertFunction("IS_PRIVATE_IP(NULL)", BOOLEAN, null); + } + + @Test + default void testIpPrefixSubnets() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 25)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/25", "192.168.1.128/25")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 26)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.0.0/26", "192.168.0.64/26", "192.168.0.128/26", "192.168.0.192/26")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2A03:2880:C000::/34', 37)", + new ArrayType(IPPREFIX), + ImmutableList.of("2a03:2880:c000::/37", "2a03:2880:c800::/37", "2a03:2880:d000::/37", "2a03:2880:d800::/37", "2a03:2880:e000::/37", "2a03:2880:e800::/37", "2a03:2880:f000::/37", "2a03:2880:f800::/37")); + } + + @Test + default void testIpPrefixSubnetsReturnSelf() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.1.0/24', 24)", new ArrayType(IPPREFIX), ImmutableList.of("192.168.1.0/24")); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '2804:431:b000::/38', 38)", new ArrayType(IPPREFIX), ImmutableList.of("2804:431:b000::/38")); + } + + @Test + default void testIpPrefixSubnetsNewPrefixLengthLongerReturnsEmpty() + { + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 23)", new ArrayType(IPPREFIX), ImmutableList.of()); + assertFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 48)", new ArrayType(IPPREFIX), ImmutableList.of()); + } + + @Test + default void testIpPrefixSubnetsInvalidPrefixLengths() + { + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', -1)", "Invalid prefix length for IPv4: -1"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '192.168.0.0/24', 33)", "Invalid prefix length for IPv4: 33"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', -1)", "Invalid prefix length for IPv6: -1"); + assertInvalidFunction("IP_PREFIX_SUBNETS(IPPREFIX '64:ff9b::17/64', 129)", "Invalid prefix length for IPv6: 129"); + } +} diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java index e72644201ef3b..01a008b62296b 100644 --- a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java @@ -27,4 +27,16 @@ public interface TestFunctions * Asserts that the projection is not supported and that it fails with the expected error message. */ void assertNotSupported(String projection, String message); + + /** + * Asserts that the projection contains an invalid function call and fails + * with the specified functional error message. + */ + void assertInvalidFunction(String projection, String message); + + /** + * Asserts that the projection contains an invalid type conversion (cast) + * and fails with the expected casting error message. + */ + void assertInvalidCast(String projection, String message); } diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 7c65fc1e81e85..f4d43c061dc65 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -311,7 +311,6 @@ io.projectreactor reactor-core - 3.8.0-M2 @@ -397,6 +396,12 @@ json-smart + + com.facebook.presto + presto-internal-communication + ${project.version} + + com.facebook.presto @@ -472,10 +477,10 @@ org.testcontainers - postgresql + testcontainers-postgresql test - + com.github.luben zstd-jni diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/ForMetadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/ForMetadata.java new file mode 100644 index 0000000000000..a67563749bd25 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/ForMetadata.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@BindingAnnotation +@Retention(RUNTIME) +@Target({ FIELD, PARAMETER, METHOD }) +public @interface ForMetadata +{ +} diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataStatsModule.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataStatsModule.java new file mode 100644 index 0000000000000..3de9c8414a6f0 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataStatsModule.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Singleton; + +import static com.google.inject.Scopes.SINGLETON; +import static org.weakref.jmx.guice.ExportBinder.newExporter; + +public class MetadataStatsModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(MetadataManagerStats.class).in(SINGLETON); + newExporter(binder) + .export(MetadataManagerStats.class) + .withGeneratedName(); + } + + @Provides + @Singleton + public Metadata provideStatsRecordingMetadata( + @ForMetadata Metadata delegate, + MetadataManagerStats stats) + { + return new StatsRecordingMetadataManager(delegate, stats); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index c201aa36d7771..8109e34287b36 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -148,6 +148,13 @@ public class CoordinatorModule "default-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + "font-src 'self' https://fonts.gstatic.com; frame-ancestors 'self'; img-src 'self' data:; form-action 'self'"; + private final boolean isWebUIEnabled; + + public CoordinatorModule(boolean webUIEnabled) + { + this.isWebUIEnabled = webUIEnabled; + } + public static HttpResourceBinding webUIBinder(Binder binder, String path, String classPathResourceBase) { return httpServerBinder(binder).bindResource(path, classPathResourceBase) @@ -158,9 +165,14 @@ public static HttpResourceBinding webUIBinder(Binder binder, String path, String @Override protected void setup(Binder binder) { - webUIBinder(binder, "/ui/dev", "webapp/dev").withWelcomeFile("index.html"); - webUIBinder(binder, "/ui", "webapp").withWelcomeFile("index.html"); - webUIBinder(binder, "/tableau", "webapp/tableau"); + if (isWebUIEnabled) { + webUIBinder(binder, "/ui/dev", "webapp/dev").withWelcomeFile("index.html"); + webUIBinder(binder, "/ui", "webapp").withWelcomeFile("index.html"); + webUIBinder(binder, "/tableau", "webapp/tableau"); + } + else { + webUIBinder(binder, "/ui", "nowebapp").withWelcomeFile("index.html"); + } // discovery server install(installModuleIf(EmbeddedDiscoveryConfig.class, EmbeddedDiscoveryConfig::isEnabled, new EmbeddedDiscoveryModule())); diff --git a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java index fc0ef9ea8abe3..ebfdbda4bc423 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/InternalCommunicationModule.java @@ -14,25 +14,10 @@ package com.facebook.presto.server; import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; -import com.facebook.airlift.http.client.HttpClientConfig; -import com.facebook.airlift.http.client.spnego.KerberosConfig; import com.facebook.presto.server.remotetask.ReactorNettyHttpClientConfig; -import com.facebook.presto.server.security.InternalAuthenticationFilter; import com.google.inject.Binder; -import com.google.inject.Module; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Locale; - -import static com.facebook.airlift.configuration.ConditionalModule.installModuleIf; import static com.facebook.airlift.configuration.ConfigBinder.configBinder; -import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; -import static com.facebook.airlift.http.server.KerberosConfig.HTTP_SERVER_AUTHENTICATION_KRB5_KEYTAB; -import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static com.facebook.presto.server.InternalCommunicationConfig.INTERNAL_COMMUNICATION_KERBEROS_ENABLED; -import static com.google.common.base.Verify.verify; public class InternalCommunicationModule extends AbstractConfigurationAwareModule @@ -41,18 +26,7 @@ public class InternalCommunicationModule protected void setup(Binder binder) { InternalCommunicationConfig internalCommunicationConfig = buildConfigObject(InternalCommunicationConfig.class); - configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, config -> { - config.setKeyStorePath(internalCommunicationConfig.getKeyStorePath()); - config.setKeyStorePassword(internalCommunicationConfig.getKeyStorePassword()); - config.setTrustStorePath(internalCommunicationConfig.getTrustStorePath()); - config.setTrustStorePassword(internalCommunicationConfig.getTrustStorePassword()); - if (internalCommunicationConfig.getIncludedCipherSuites().isPresent()) { - config.setHttpsIncludedCipherSuites(internalCommunicationConfig.getIncludedCipherSuites().get()); - } - if (internalCommunicationConfig.getExcludeCipherSuites().isPresent()) { - config.setHttpsExcludedCipherSuites(internalCommunicationConfig.getExcludeCipherSuites().get()); - } - }); + install(new CommonInternalCommunicationModule()); configBinder(binder).bindConfigGlobalDefaults(ReactorNettyHttpClientConfig.class, config -> { config.setHttpsEnabled(internalCommunicationConfig.isHttpsRequired()); @@ -63,42 +37,5 @@ protected void setup(Binder binder) config.setCipherSuites(internalCommunicationConfig.getIncludedCipherSuites().get()); } }); - - install(installModuleIf(InternalCommunicationConfig.class, InternalCommunicationConfig::isKerberosEnabled, kerberosInternalCommunicationModule())); - binder.bind(InternalAuthenticationManager.class); - httpClientBinder(binder).bindGlobalFilter(InternalAuthenticationManager.class); - jaxrsBinder(binder).bind(InternalAuthenticationFilter.class); - } - - private Module kerberosInternalCommunicationModule() - { - return binder -> { - InternalCommunicationConfig clientKerberosConfig = buildConfigObject(InternalCommunicationConfig.class); - com.facebook.airlift.http.server.KerberosConfig serverKerberosConfig = buildConfigObject(com.facebook.airlift.http.server.KerberosConfig.class); - verify(serverKerberosConfig.getKeytab() != null, "%s must be set when %s is true", HTTP_SERVER_AUTHENTICATION_KRB5_KEYTAB, INTERNAL_COMMUNICATION_KERBEROS_ENABLED); - - configBinder(binder).bindConfigGlobalDefaults(KerberosConfig.class, kerberosConfig -> { - kerberosConfig.setConfig(serverKerberosConfig.getKerberosConfig()); - kerberosConfig.setKeytab(serverKerberosConfig.getKeytab()); - kerberosConfig.setUseCanonicalHostname(clientKerberosConfig.isKerberosUseCanonicalHostname()); - }); - - String kerberosPrincipal = serverKerberosConfig.getServiceName() + "/" + getLocalCanonicalHostName(); - configBinder(binder).bindConfigGlobalDefaults(HttpClientConfig.class, httpClientConfig -> { - httpClientConfig.setAuthenticationEnabled(true); - httpClientConfig.setKerberosPrincipal(kerberosPrincipal); - httpClientConfig.setKerberosRemoteServiceName(serverKerberosConfig.getServiceName()); - }); - }; - } - - private static String getLocalCanonicalHostName() - { - try { - return InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(Locale.US); - } - catch (UnknownHostException e) { - throw new UncheckedIOException(e); - } } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 14c66a6a95483..bf64294640107 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -120,6 +120,13 @@ public void run() verifyJvmRequirements(); verifySystemTimeIsReasonable(); + // Netty 4.2 enables SSL endpoint verification by default. The Drift Netty transport + // does not pass hostnames to the SSL engine, causing SSLHandshakeException. Disable + // the default endpoint verification until Drift is updated to support it. + if (System.getProperty("io.netty.handler.ssl.defaultEndpointVerificationAlgorithm") == null) { + System.setProperty("io.netty.handler.ssl.defaultEndpointVerificationAlgorithm", "NONE"); + } + Logger log = Logger.get(PrestoServer.class); ImmutableList.Builder modules = ImmutableList.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 9a5654c408e3e..67f98fb3f57bc 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -103,6 +103,7 @@ import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.DiscoveryNodeManager; +import com.facebook.presto.metadata.ForMetadata; import com.facebook.presto.metadata.ForNodeManager; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.HandleJsonModule; @@ -110,6 +111,7 @@ import com.facebook.presto.metadata.MaterializedViewPropertyManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.metadata.MetadataStatsModule; import com.facebook.presto.metadata.SchemaPropertyManager; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.metadata.SessionPropertyProviderConfig; @@ -263,6 +265,7 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import io.airlift.slice.Slice; +import io.netty.buffer.PooledByteBufAllocator; import jakarta.annotation.PreDestroy; import jakarta.inject.Singleton; import jakarta.servlet.Filter; @@ -334,7 +337,7 @@ else if (serverConfig.isCatalogServer()) { install(new CatalogServerModule()); } else if (serverConfig.isCoordinator()) { - install(new CoordinatorModule()); + install(new CoordinatorModule(serverConfig.isWebUIEnabled())); } else { install(new WorkerModule()); @@ -626,7 +629,7 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon config.setMaxContentLength(new DataSize(32, MEGABYTE)); }); - binder.install(new DriftNettyClientModule()); + binder.install(new DriftNettyClientModule(PooledByteBufAllocator.DEFAULT)); driftClientBinder(binder).bindDriftClient(ThriftTaskClient.class, ForExchange.class) .withAddressSelector(((addressSelectorBinder, annotation, prefix) -> addressSelectorBinder.bind(AddressSelector.class).annotatedWith(annotation).to(FixedAddressSelector.class))); @@ -673,10 +676,16 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon if (serverConfig.isCatalogServerEnabled() && serverConfig.isCoordinator()) { binder.bind(RemoteMetadataManager.class).in(Scopes.SINGLETON); - binder.bind(Metadata.class).to(RemoteMetadataManager.class).in(Scopes.SINGLETON); + binder.bind(Metadata.class) + .annotatedWith(ForMetadata.class) + .to(RemoteMetadataManager.class) + .in(Scopes.SINGLETON); } else { - binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON); + binder.bind(Metadata.class) + .annotatedWith(ForMetadata.class) + .to(MetadataManager.class) + .in(Scopes.SINGLETON); } // row expression utils @@ -718,6 +727,8 @@ public ListeningExecutorService createResourceManagerExecutor(ResourceManagerCon binder.install(new HandleJsonModule()); binder.bind(ObjectMapper.class).toProvider(JsonObjectMapperProvider.class); + binder.install(new MetadataStatsModule()); + // connector binder.bind(ScalarStatsCalculator.class).in(Scopes.SINGLETON); binder.bind(StatsNormalizer.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java index 294f8d27cf068..b3facd515a963 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskFactory.java @@ -50,7 +50,6 @@ import com.facebook.presto.sql.planner.PlanFragment; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.netty.channel.EventLoop; import io.netty.util.concurrent.AbstractEventExecutorGroup; import jakarta.annotation.PreDestroy; import jakarta.inject.Inject; @@ -205,14 +204,7 @@ else if (binaryTransportEnabled) { this.taskUpdateSizeTrackingEnabled = taskConfig.isTaskUpdateSizeTrackingEnabled(); this.eventLoopGroup = Optional.of(new SafeEventLoopGroup(config.getRemoteTaskMaxCallbackThreads(), - new ThreadFactoryBuilder().setNameFormat("task-event-loop-%s").setDaemon(true).build(), taskConfig.getSlowMethodThresholdOnEventLoop()) - { - @Override - protected EventLoop newChild(Executor executor, Object... args) - { - return new SafeEventLoop(this, executor); - } - }); + new ThreadFactoryBuilder().setNameFormat("task-event-loop-%s").setDaemon(true).build(), taskConfig.getSlowMethodThresholdOnEventLoop())); } @Managed diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index debae20a1aa3c..21a8f4a6d5c2b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -830,7 +830,7 @@ public void startResponding() requestBlocker.unblock(); } - private static void updateConnectorIdAnnouncement(Announcer announcer, ConnectorId connectorId, InternalNodeManager nodeManager) + public static void updateConnectorIdAnnouncement(Announcer announcer, ConnectorId connectorId, InternalNodeManager nodeManager) { // // This code was copied from PrestoServer, and is a hack that should be removed when the connectorId property is removed diff --git a/presto-main/src/main/resources/nowebapp/index.html b/presto-main/src/main/resources/nowebapp/index.html new file mode 100644 index 0000000000000..9a7920b784325 --- /dev/null +++ b/presto-main/src/main/resources/nowebapp/index.html @@ -0,0 +1,35 @@ + + + + + + Presto UI Disabled + + + +
+

The Presto UI has been disabled for this environment

+
+ + diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestMetadataStatsModule.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestMetadataStatsModule.java new file mode 100644 index 0000000000000..2e5d40171d77f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestMetadataStatsModule.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class TestMetadataStatsModule +{ + @Test + public void testModuleCreation() + { + MetadataStatsModule module = new MetadataStatsModule(); + assertNotNull(module, "Module should be created"); + } + + @Test + public void testModuleBindsMetadata() + { + MetadataManager testMetadataManager = MetadataManager.createTestMetadataManager(); + Injector injector = Guice.createInjector(binder -> { + binder.bind(Metadata.class).annotatedWith(ForMetadata.class).toInstance(testMetadataManager); + }, new MetadataStatsModule()); + + MetadataManagerStats stats = injector.getInstance(MetadataManagerStats.class); + assertNotNull(stats, "MetadataManagerStats should be bound"); + + Metadata metadata = injector.getInstance(Metadata.class); + assertNotNull(metadata, "Metadata should be bound"); + + assertTrue(metadata instanceof StatsRecordingMetadataManager, + "Metadata should be wrapped with StatsRecordingMetadataManager"); + } + + @Test + public void testStatsAreSingleton() + { + MetadataManager testMetadataManager = MetadataManager.createTestMetadataManager(); + Injector injector = Guice.createInjector(binder -> { + binder.bind(Metadata.class).annotatedWith(ForMetadata.class).toInstance(testMetadataManager); + }, new MetadataStatsModule()); + + MetadataManagerStats stats1 = injector.getInstance(MetadataManagerStats.class); + MetadataManagerStats stats2 = injector.getInstance(MetadataManagerStats.class); + + assertTrue(stats1 == stats2, "MetadataManagerStats should be singleton"); + + Metadata metadata1 = injector.getInstance(Metadata.class); + Metadata metadata2 = injector.getInstance(Metadata.class); + + assertTrue(metadata1 == metadata2, "Metadata should be singleton"); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/server/security/oauth2/TestingHydraIdentityProvider.java b/presto-main/src/test/java/com/facebook/presto/server/security/oauth2/TestingHydraIdentityProvider.java index 0daa570a29842..9c7c39cfeb149 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/security/oauth2/TestingHydraIdentityProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/server/security/oauth2/TestingHydraIdentityProvider.java @@ -44,10 +44,10 @@ import org.testcontainers.containers.FixedHostPortGenericContainer; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.WaitAllStrategy; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; import java.io.Closeable; @@ -75,7 +75,7 @@ public class TestingHydraIdentityProvider private final Network network = Network.newNetwork(); - private final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>() + private final PostgreSQLContainer databaseContainer = new PostgreSQLContainer("postgres:14") .withNetwork(network) .withNetworkAliases("database") .withUsername("hydra") diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 7de5616087742..364df14a756e9 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -166,7 +166,7 @@ org.testcontainers - mysql + testcontainers-mysql test diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestCredentialPassthrough.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestCredentialPassthrough.java index 1c42cb88d3913..364d21e588ea3 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestCredentialPassthrough.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestCredentialPassthrough.java @@ -18,7 +18,7 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableMap; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -40,13 +40,13 @@ public class TestCredentialPassthrough private static final String TEST_USER = "testuser"; private static final String TEST_PASSWORD = "testpass"; - private final MySQLContainer mysqlContainer; + private final MySQLContainer mysqlContainer; private final QueryRunner mySqlQueryRunner; public TestCredentialPassthrough() throws Exception { - mysqlContainer = new MySQLContainer<>("mysql:8.0") + mysqlContainer = new MySQLContainer("mysql:8.0") .withDatabaseName(TEST_SCHEMA) .withUsername(TEST_USER) .withPassword(TEST_PASSWORD); @@ -77,7 +77,7 @@ public void testCredentialPassthrough() mySqlQueryRunner.execute(getSession(mysqlContainer), "CREATE TABLE test_create (a bigint, b double, c varchar)"); } - public static QueryRunner createQueryRunner(MySQLContainer mysqlContainer) + public static QueryRunner createQueryRunner(MySQLContainer mysqlContainer) throws Exception { DistributedQueryRunner queryRunner = null; @@ -99,7 +99,7 @@ public static QueryRunner createQueryRunner(MySQLContainer mysqlContainer) } } - private static Session getSession(MySQLContainer mysqlContainer) + private static Session getSession(MySQLContainer mysqlContainer) { Map extraCredentials = ImmutableMap.of("mysql.user", mysqlContainer.getUsername(), "mysql.password", mysqlContainer.getPassword()); return testSessionBuilder() @@ -116,7 +116,7 @@ private static Session getSession(MySQLContainer mysqlContainer) .build(); } - private static String getConnectionUrl(MySQLContainer mysqlContainer) + private static String getConnectionUrl(MySQLContainer mysqlContainer) { String jdbcUrlWithoutDatabase = removeDatabaseFromJdbcUrl(mysqlContainer.getJdbcUrl()); return format("%s?useSSL=false&allowPublicKeyRetrieval=true", jdbcUrlWithoutDatabase.split("\\?")[0]); diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java index 9cec5920c9be1..1d2470e1b2135 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlDistributedQueries.java @@ -18,7 +18,7 @@ import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Optional; import org.testng.annotations.Test; @@ -33,11 +33,11 @@ public class TestMySqlDistributedQueries extends AbstractTestDistributedQueries { - private final MySQLContainer mysqlContainer; + private final MySQLContainer mysqlContainer; public TestMySqlDistributedQueries() { - this.mysqlContainer = new MySQLContainer<>("mysql:8.0") + this.mysqlContainer = new MySQLContainer("mysql:8.0") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationMixedCaseTest.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationMixedCaseTest.java index 26f8d6f277e99..0a56bf282ddba 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationMixedCaseTest.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationMixedCaseTest.java @@ -19,7 +19,7 @@ import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -40,12 +40,12 @@ public class TestMySqlIntegrationMixedCaseTest extends AbstractTestQueryFramework { - private final MySQLContainer mysqlContainer; + private final MySQLContainer mysqlContainer; public TestMySqlIntegrationMixedCaseTest() throws Exception { - this.mysqlContainer = new MySQLContainer<>("mysql:8.0") + this.mysqlContainer = new MySQLContainer("mysql:8.0") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java index ec32ea5594b30..4a3de5895fdf0 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlIntegrationSmokeTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -48,12 +48,12 @@ public class TestMySqlIntegrationSmokeTest extends AbstractTestIntegrationSmokeTest { - private final MySQLContainer mysqlContainer; + private final MySQLContainer mysqlContainer; public TestMySqlIntegrationSmokeTest() throws Exception { - this.mysqlContainer = new MySQLContainer<>("mysql:8.0") + this.mysqlContainer = new MySQLContainer("mysql:8.0") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java index 9adc4bfe1169f..5541156aa874a 100644 --- a/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java +++ b/presto-mysql/src/test/java/com/facebook/presto/plugin/mysql/TestMySqlTypeMapping.java @@ -26,7 +26,7 @@ import com.facebook.presto.tests.sql.PrestoSqlExecutor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -66,11 +66,11 @@ public class TestMySqlTypeMapping { private static final String CHARACTER_SET_UTF8 = "CHARACTER SET utf8"; - private final MySQLContainer mysqlContainer; + private final MySQLContainer mysqlContainer; public TestMySqlTypeMapping() { - this.mysqlContainer = new MySQLContainer<>("mysql:8.0") + this.mysqlContainer = new MySQLContainer("mysql:8.0") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-native-execution/CMake/arrow/arrow-flight.patch b/presto-native-execution/CMake/arrow/arrow-flight.patch new file mode 100644 index 0000000000000..385e08f5fcd8d --- /dev/null +++ b/presto-native-execution/CMake/arrow/arrow-flight.patch @@ -0,0 +1,319 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- a/cpp/src/arrow/flight/types.h ++++ b/cpp/src/arrow/flight/types.h +@@ -465,8 +465,150 @@ + static FlightDescriptor Path(std::vector path) { + return FlightDescriptor{PATH, "", std::move(path)}; + } ++}; ++ ++struct ARROW_FLIGHT_EXPORT Ticket : public internal::BaseType { ++ std::string ticket; ++ ++ Ticket() = default; ++ Ticket(std::string ticket) // NOLINT runtime/explicit ++ : ticket(std::move(ticket)) {} ++ ++ std::string ToString() const; ++ bool Equals(const Ticket& other) const; ++ ++ using SuperT::Deserialize; ++ using SuperT::SerializeToString; ++ ++ /// \brief Get the wire-format representation of this type. ++ /// ++ /// Useful when interoperating with non-Flight systems (e.g. REST ++ /// services) that may want to return Flight types. ++ /// ++ /// Use `SerializeToString()` if you want a Result-returning version. ++ arrow::Status SerializeToString(std::string* out) const; ++ ++ /// \brief Parse the wire-format representation of this type. ++ /// ++ /// Useful when interoperating with non-Flight systems (e.g. REST ++ /// services) that may want to return Flight types. ++ /// ++ /// Use `Deserialize(serialized)` if you want a Result-returning version. ++ static arrow::Status Deserialize(std::string_view serialized, Ticket* out); ++}; ++ ++struct ARROW_FLIGHT_EXPORT Location : public internal::BaseType { ++ public: ++ /// \brief Initialize a blank location. ++ Location(); ++ ++ ~Location(); ++ ++ /// \brief Initialize a location by parsing a URI string ++ static arrow::Result Parse(const std::string& uri_string); ++ ++ /// \brief Get the fallback URI. ++ /// ++ /// arrow-flight-reuse-connection://? means that a client may attempt to ++ /// reuse an existing connection to a Flight service to fetch data instead ++ /// of creating a new connection to one of the other locations listed in a ++ /// FlightEndpoint response. ++ static const Location& ReuseConnection(); ++ ++ /// \brief Initialize a location for a non-TLS, gRPC-based Flight ++ /// service from a host and port ++ /// \param[in] host The hostname to connect to ++ /// \param[in] port The port ++ /// \return Arrow result with the resulting location ++ static arrow::Result ForGrpcTcp(const std::string& host, const int port); ++ ++ /// \brief Initialize a location for a TLS-enabled, gRPC-based Flight ++ /// service from a host and port ++ /// \param[in] host The hostname to connect to ++ /// \param[in] port The port ++ /// \return Arrow result with the resulting location ++ static arrow::Result ForGrpcTls(const std::string& host, const int port); ++ ++ /// \brief Initialize a location for a domain socket-based Flight ++ /// service ++ /// \param[in] path The path to the domain socket ++ /// \return Arrow result with the resulting location ++ static arrow::Result ForGrpcUnix(const std::string& path); ++ ++ /// \brief Initialize a location based on a URI scheme ++ static arrow::Result ForScheme(const std::string& scheme, ++ const std::string& host, const int port); ++ ++ /// \brief Get the scheme of this URI. ++ std::string scheme() const; ++ ++ /// \brief Get a representation of this URI as a string. ++ std::string ToString() const; ++ bool Equals(const Location& other) const; ++ ++ using SuperT::Deserialize; ++ using SuperT::SerializeToString; ++ ++ /// \brief Serialize this message to its wire-format representation. ++ /// ++ /// Use `SerializeToString()` if you want a Result-returning version. ++ arrow::Status SerializeToString(std::string* out) const; ++ ++ /// \brief Deserialize this message from its wire-format representation. ++ /// ++ /// Use `Deserialize(serialized)` if you want a Result-returning version. ++ static arrow::Status Deserialize(std::string_view serialized, Location* out); ++ ++ private: ++ friend class FlightClient; ++ friend class FlightServerBase; ++ std::shared_ptr uri_; ++}; ++ ++/// \brief A flight ticket and list of locations where the ticket can be ++struct ARROW_FLIGHT_EXPORT FlightEndpoint : public internal::BaseType { ++ /// Opaque ticket identify; use with DoGet RPC ++ Ticket ticket; ++ ++ /// List of locations where ticket can be redeemed. If the list is empty, the ++ /// ticket can only be redeemed on the current service where the ticket was ++ /// generated ++ std::vector locations; ++ ++ /// Expiration time of this stream. If present, clients may assume ++ /// they can retry DoGet requests. Otherwise, clients should avoid ++ /// retrying DoGet requests. ++ std::optional expiration_time; ++ ++ /// Opaque Application-defined metadata ++ std::string app_metadata; ++ ++ FlightEndpoint() = default; ++ FlightEndpoint(Ticket ticket, std::vector locations, ++ std::optional expiration_time, std::string app_metadata) ++ : ticket(std::move(ticket)), ++ locations(std::move(locations)), ++ expiration_time(expiration_time), ++ app_metadata(std::move(app_metadata)) {} ++ ++ std::string ToString() const; ++ bool Equals(const FlightEndpoint& other) const; ++ ++ using SuperT::Deserialize; ++ using SuperT::SerializeToString; ++ ++ /// \brief Serialize this message to its wire-format representation. ++ /// ++ /// Use `SerializeToString()` if you want a Result-returning version. ++ arrow::Status SerializeToString(std::string* out) const; ++ ++ /// \brief Deserialize this message from its wire-format representation. ++ /// ++ /// Use `Deserialize(serialized)` if you want a Result-returning version. ++ static arrow::Status Deserialize(std::string_view serialized, FlightEndpoint* out); + }; + ++/// \brief The request of the RenewFlightEndpoint action. + /// \brief The access coordinates for retrieval of a dataset, returned by + /// GetFlightInfo + class ARROW_FLIGHT_EXPORT FlightInfo +@@ -704,150 +846,7 @@ + + /// \brief Data structure providing an opaque identifier or credential to use + /// when requesting a data stream with the DoGet RPC +-struct ARROW_FLIGHT_EXPORT Ticket : public internal::BaseType { +- std::string ticket; +- +- Ticket() = default; +- Ticket(std::string ticket) // NOLINT runtime/explicit +- : ticket(std::move(ticket)) {} +- +- std::string ToString() const; +- bool Equals(const Ticket& other) const; +- +- using SuperT::Deserialize; +- using SuperT::SerializeToString; +- +- /// \brief Get the wire-format representation of this type. +- /// +- /// Useful when interoperating with non-Flight systems (e.g. REST +- /// services) that may want to return Flight types. +- /// +- /// Use `SerializeToString()` if you want a Result-returning version. +- arrow::Status SerializeToString(std::string* out) const; +- +- /// \brief Parse the wire-format representation of this type. +- /// +- /// Useful when interoperating with non-Flight systems (e.g. REST +- /// services) that may want to return Flight types. +- /// +- /// Use `Deserialize(serialized)` if you want a Result-returning version. +- static arrow::Status Deserialize(std::string_view serialized, Ticket* out); +-}; +- +-/// \brief A host location (a URI) +-struct ARROW_FLIGHT_EXPORT Location : public internal::BaseType { +- public: +- /// \brief Initialize a blank location. +- Location(); +- +- ~Location(); +- +- /// \brief Initialize a location by parsing a URI string +- static arrow::Result Parse(const std::string& uri_string); +- +- /// \brief Get the fallback URI. +- /// +- /// arrow-flight-reuse-connection://? means that a client may attempt to +- /// reuse an existing connection to a Flight service to fetch data instead +- /// of creating a new connection to one of the other locations listed in a +- /// FlightEndpoint response. +- static const Location& ReuseConnection(); +- +- /// \brief Initialize a location for a non-TLS, gRPC-based Flight +- /// service from a host and port +- /// \param[in] host The hostname to connect to +- /// \param[in] port The port +- /// \return Arrow result with the resulting location +- static arrow::Result ForGrpcTcp(const std::string& host, const int port); +- +- /// \brief Initialize a location for a TLS-enabled, gRPC-based Flight +- /// service from a host and port +- /// \param[in] host The hostname to connect to +- /// \param[in] port The port +- /// \return Arrow result with the resulting location +- static arrow::Result ForGrpcTls(const std::string& host, const int port); +- +- /// \brief Initialize a location for a domain socket-based Flight +- /// service +- /// \param[in] path The path to the domain socket +- /// \return Arrow result with the resulting location +- static arrow::Result ForGrpcUnix(const std::string& path); +- +- /// \brief Initialize a location based on a URI scheme +- static arrow::Result ForScheme(const std::string& scheme, +- const std::string& host, const int port); +- +- /// \brief Get the scheme of this URI. +- std::string scheme() const; +- +- /// \brief Get a representation of this URI as a string. +- std::string ToString() const; +- bool Equals(const Location& other) const; +- +- using SuperT::Deserialize; +- using SuperT::SerializeToString; +- +- /// \brief Serialize this message to its wire-format representation. +- /// +- /// Use `SerializeToString()` if you want a Result-returning version. +- arrow::Status SerializeToString(std::string* out) const; +- +- /// \brief Deserialize this message from its wire-format representation. +- /// +- /// Use `Deserialize(serialized)` if you want a Result-returning version. +- static arrow::Status Deserialize(std::string_view serialized, Location* out); +- +- private: +- friend class FlightClient; +- friend class FlightServerBase; +- std::shared_ptr uri_; +-}; +- +-/// \brief A flight ticket and list of locations where the ticket can be + /// redeemed +-struct ARROW_FLIGHT_EXPORT FlightEndpoint : public internal::BaseType { +- /// Opaque ticket identify; use with DoGet RPC +- Ticket ticket; +- +- /// List of locations where ticket can be redeemed. If the list is empty, the +- /// ticket can only be redeemed on the current service where the ticket was +- /// generated +- std::vector locations; +- +- /// Expiration time of this stream. If present, clients may assume +- /// they can retry DoGet requests. Otherwise, clients should avoid +- /// retrying DoGet requests. +- std::optional expiration_time; +- +- /// Opaque Application-defined metadata +- std::string app_metadata; +- +- FlightEndpoint() = default; +- FlightEndpoint(Ticket ticket, std::vector locations, +- std::optional expiration_time, std::string app_metadata) +- : ticket(std::move(ticket)), +- locations(std::move(locations)), +- expiration_time(expiration_time), +- app_metadata(std::move(app_metadata)) {} +- +- std::string ToString() const; +- bool Equals(const FlightEndpoint& other) const; +- +- using SuperT::Deserialize; +- using SuperT::SerializeToString; +- +- /// \brief Serialize this message to its wire-format representation. +- /// +- /// Use `SerializeToString()` if you want a Result-returning version. +- arrow::Status SerializeToString(std::string* out) const; +- +- /// \brief Deserialize this message from its wire-format representation. +- /// +- /// Use `Deserialize(serialized)` if you want a Result-returning version. +- static arrow::Status Deserialize(std::string_view serialized, FlightEndpoint* out); +-}; +- +-/// \brief The request of the RenewFlightEndpoint action. + struct ARROW_FLIGHT_EXPORT RenewFlightEndpointRequest + : public internal::BaseType { + FlightEndpoint endpoint; + diff --git a/presto-native-execution/CMakeLists.txt b/presto-native-execution/CMakeLists.txt index fc5d1ddf62696..77eb57bea59fd 100644 --- a/presto-native-execution/CMakeLists.txt +++ b/presto-native-execution/CMakeLists.txt @@ -77,10 +77,8 @@ option(PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR "Enable Arrow Flight connector" OFF) option(PRESTO_ENABLE_SPATIAL "Enable spatial support" ON) -# Set all Velox options below and make sure that if we include folly headers or -# other dependency headers that include folly headers we turn off the coroutines -# and turn on int128. -add_compile_definitions(FOLLY_HAVE_INT128_T=1 FOLLY_CFG_NO_COROUTINES) +# Turn on folly int128 support. +add_compile_definitions(FOLLY_HAVE_INT128_T=1) set(VELOX_ENABLE_S3 ${PRESTO_ENABLE_S3} CACHE BOOL "Build S3 support") @@ -184,6 +182,9 @@ find_library(PROXYGEN_HTTP_SERVER proxygenhttpserver) find_library(FIZZ fizz) find_library(WANGLE wangle) find_library(MVFST_EXCEPTION mvfst_exception) +find_library(MVFST_FOLLY_UTILS mvfst_folly_utils) +find_library(MVFST_CODEC_TYPES mvfst_codec_types) +find_library(MVFST_CONTIGUOUS_CURSOR mvfst_contiguous_cursor) find_library(RE2 re2) @@ -199,6 +200,9 @@ set( ${WANGLE} ${FIZZ} ${MVFST_EXCEPTION} + ${MVFST_FOLLY_UTILS} + ${MVFST_CODEC_TYPES} + ${MVFST_CONTIGUOUS_CURSOR} ) find_path(PROXYGEN_DIR NAMES include/proxygen) set(PROXYGEN_INCLUDE_DIR "${PROXYGEN_DIR}/include/") @@ -231,6 +235,9 @@ if(PRESTO_ENABLE_JWT) add_compile_definitions(PRESTO_ENABLE_JWT) endif() +find_package(DataSketches) +add_compile_definitions(PRESTO_ENABLE_THETA_SKETCH) + if("${MAX_LINK_JOBS}") set_property(GLOBAL APPEND PROPERTY JOB_POOLS "presto_link_job_pool=${MAX_LINK_JOBS}") else() diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile index 8b82be02f75ed..9397a4c632daa 100644 --- a/presto-native-execution/Makefile +++ b/presto-native-execution/Makefile @@ -95,7 +95,7 @@ clean: #: Delete all build artifacts rm -rf $(BUILD_BASE_DIR) velox-submodule: #: Check out code for velox submodule - git submodule sync --recursive + git submodule sync --recursive && \ git submodule update --init --recursive submodules: velox-submodule @@ -107,7 +107,7 @@ build: #: Build the software based in BUILD_DIR and BUILD_TYPE variables cmake --build $(BUILD_BASE_DIR)/$(BUILD_DIR) -j $(NUM_THREADS) debug: #: Build with debugging symbols - $(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=Debug + $(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=Debug && \ $(MAKE) build BUILD_DIR=debug release: #: Build the release version @@ -115,7 +115,7 @@ release: #: Build the release version $(MAKE) build BUILD_DIR=release cmake-and-build: #: cmake and build without updating submodules which requires git - cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) $(EXTRA_CMAKE_FLAGS) + cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) $(EXTRA_CMAKE_FLAGS) && \ cmake --build $(BUILD_BASE_DIR)/$(BUILD_DIR) -j $(NUM_THREADS) unittest: debug #: Build with debugging and run unit tests diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index 38211e1442ccf..821f2cda0f931 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -158,6 +158,24 @@ test + + com.facebook.presto + presto-jmx + test + + + + com.facebook.airlift + stats + test + + + + com.facebook.airlift + discovery + test + + com.facebook.presto @@ -194,18 +212,6 @@ org.apache.hudi hudi-presto-bundle - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-common - - - org.apache.parquet - parquet-format-structures - org.apache.commons commons-lang3 @@ -223,21 +229,29 @@ com.esotericsoftware kryo-shaded + + + + + org.apache.iceberg + iceberg-core + ${dep.iceberg.version} + tests + test + - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-common - - - org.apache.parquet - parquet-format-structures + org.slf4j + slf4j-api + + com.facebook.airlift + http-server + test + + com.facebook.presto presto-delta @@ -510,7 +524,7 @@ org.apache.maven.plugins maven-surefire-plugin - writer,parquet,remote-function,textfile,async_data_cache + writer,parquet,remote-function,textfile diff --git a/presto-native-execution/presto_cpp/main/Announcer.cpp b/presto-native-execution/presto_cpp/main/Announcer.cpp index 0c9433ac335e0..6051fc290b356 100644 --- a/presto-native-execution/presto_cpp/main/Announcer.cpp +++ b/presto-native-execution/presto_cpp/main/Announcer.cpp @@ -16,8 +16,6 @@ #include #include #include -#include -#include #include "presto_cpp/external/json/nlohmann/json.hpp" namespace facebook::presto { diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index 9138bb9be597b..7de56fb0379c7 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -69,6 +69,7 @@ target_link_libraries( presto_session_properties presto_velox_plan_conversion presto_hive_functions + presto_theta_sketch_functions velox_abfs velox_aggregates velox_caching @@ -93,6 +94,7 @@ target_link_libraries( velox_hive_connector velox_hive_iceberg_splitreader velox_hive_partition_function + velox_key_encoder velox_presto_serializer velox_presto_type_parser velox_s3fs diff --git a/presto-native-execution/presto_cpp/main/PeriodicHeartbeatManager.cpp b/presto-native-execution/presto_cpp/main/PeriodicHeartbeatManager.cpp index efad3c23b51c2..b6406222c9407 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicHeartbeatManager.cpp +++ b/presto-native-execution/presto_cpp/main/PeriodicHeartbeatManager.cpp @@ -12,7 +12,6 @@ * limitations under the License. */ #include "presto_cpp/main/PeriodicHeartbeatManager.h" -#include namespace facebook::presto { PeriodicHeartbeatManager::PeriodicHeartbeatManager( diff --git a/presto-native-execution/presto_cpp/main/PeriodicServiceInventoryManager.cpp b/presto-native-execution/presto_cpp/main/PeriodicServiceInventoryManager.cpp index a1f745ccaad07..ebb5519ecec28 100644 --- a/presto-native-execution/presto_cpp/main/PeriodicServiceInventoryManager.cpp +++ b/presto-native-execution/presto_cpp/main/PeriodicServiceInventoryManager.cpp @@ -14,6 +14,7 @@ #include "presto_cpp/main/PeriodicServiceInventoryManager.h" #include #include +#include "presto_cpp/main/common/Configs.h" namespace facebook::presto { PeriodicServiceInventoryManager::PeriodicServiceInventoryManager( @@ -80,6 +81,8 @@ void PeriodicServiceInventoryManager::sendRequest() { LOG(INFO) << "Service Inventory changed to " << newAddress.getAddressStr() << ":" << newAddress.getPort(); std::swap(serviceAddress_, newAddress); + auto systemConfig = SystemConfig::instance(); + auto httpClientOptions = systemConfig->httpClientOptions(); client_ = std::make_shared( eventBaseThread_.getEventBase(), nullptr, @@ -91,7 +94,8 @@ void PeriodicServiceInventoryManager::sendRequest() { std::chrono::milliseconds(10'000), std::chrono::milliseconds(0), pool_, - sslContext_); + sslContext_, + std::move(httpClientOptions)); } } catch (const std::exception& ex) { LOG(WARNING) << "Error occurred during updating service address: " diff --git a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp index 0ae186f8bde39..df2e28dd98dba 100644 --- a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.cpp @@ -14,12 +14,13 @@ #include "presto_cpp/main/PrestoExchangeSource.h" #include +#include #include #include #include -#include "presto_cpp/main/QueryContextManager.h" #include "presto_cpp/main/common/Counters.h" +#include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/common/base/Exceptions.h" #include "velox/common/testutil/TestValue.h" @@ -37,8 +38,7 @@ std::string extractTaskId(const std::string& path) { VLOG(1) << "Failed to extract task ID from remote split: " << path; - throw std::invalid_argument( - fmt::format("Cannot extract task ID from remote split URL: {}", path)); + VELOX_FAIL("Cannot extract task ID from remote split URL: {}", path); } void onFinalFailure( @@ -103,6 +103,8 @@ PrestoExchangeSource::PrestoExchangeSource( VELOX_CHECK_NOT_NULL(driverExecutor_); VELOX_CHECK_NOT_NULL(ioEventBase); VELOX_CHECK_NOT_NULL(pool_); + auto systemConfig = SystemConfig::instance(); + auto httpClientOptions = systemConfig->httpClientOptions(); httpClient_ = std::make_shared( ioEventBase, connPool, @@ -111,7 +113,9 @@ PrestoExchangeSource::PrestoExchangeSource( requestTimeoutMs, connectTimeoutMs, immediateBufferTransfer_ ? pool_ : nullptr, - sslContext_); + sslContext_, + std::move(httpClientOptions)); + jwtOptions_ = systemConfig->jwtOptions(); } void PrestoExchangeSource::close() { @@ -185,7 +189,8 @@ void PrestoExchangeSource::doRequest( } else { method = proxygen::HTTPMethod::GET; } - auto requestBuilder = http::RequestBuilder().method(method).url(path); + auto requestBuilder = + http::RequestBuilder().jwtOptions(jwtOptions_).method(method).url(path); velox::common::testutil::TestValue::adjust( "facebook::presto::PrestoExchangeSource::doRequest", this); @@ -292,10 +297,10 @@ void PrestoExchangeSource::processDataResponse( !headers->getIsChunked(), "Chunked http transferring encoding is not supported."); } + const auto contentLengthStr = headers->getHeaders().getSingleOrEmpty( + proxygen::HTTP_HEADER_CONTENT_LENGTH); const uint64_t contentLength = - atol(headers->getHeaders() - .getSingleOrEmpty(proxygen::HTTP_HEADER_CONTENT_LENGTH) - .c_str()); + contentLengthStr.empty() ? 0 : folly::to(contentLengthStr); VLOG(1) << "Fetched data for " << basePath_ << "/" << sequence_ << ": " << contentLength << " bytes"; @@ -326,7 +331,7 @@ void PrestoExchangeSource::processDataResponse( // token so we shouldn't update 'sequence_' if it is empty. Otherwise, // 'sequence_' gets reset and we can't fetch any data from the source with // the rolled back 'sequence_'. - ackSequenceOpt = atol(nextTokenStr.c_str()); + ackSequenceOpt = folly::to(nextTokenStr); } else { VELOX_CHECK_EQ( contentLength, 0, "next token is not set in non-empty data response"); @@ -468,6 +473,7 @@ void PrestoExchangeSource::acknowledgeResults(int64_t ackSequence) { auto ackPath = fmt::format("{}/{}/acknowledge", basePath_, ackSequence); VLOG(1) << "Sending ack " << ackPath; http::RequestBuilder() + .jwtOptions(jwtOptions_) .method(proxygen::HTTPMethod::GET) .url(ackPath) .send(httpClient_.get()) @@ -512,6 +518,7 @@ void PrestoExchangeSource::abortResults() { void PrestoExchangeSource::doAbortResults(int64_t delayMs) { http::RequestBuilder() + .jwtOptions(jwtOptions_) .method(proxygen::HTTPMethod::DELETE) .url(basePath_) .send(httpClient_.get(), "", delayMs) diff --git a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h index 394627776780e..529f1793022e7 100644 --- a/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h +++ b/presto-native-execution/presto_cpp/main/PrestoExchangeSource.h @@ -287,6 +287,7 @@ class PrestoExchangeSource : public velox::exec::ExchangeSource { folly::CPUThreadPoolExecutor* const driverExecutor_; std::shared_ptr httpClient_; + http::JwtOptions jwtOptions_; RetryState dataRequestRetryState_; RetryState abortRetryState_; int failedAttempts_; diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 91e70c4f2c0bd..358baf53cde32 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -85,6 +85,10 @@ #include "presto_cpp/main/RemoteFunctionRegisterer.h" #endif +#ifdef PRESTO_ENABLE_THETA_SKETCH +#include "presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h" +#endif + #ifdef __linux__ // Required by BatchThreadFactory #include @@ -264,15 +268,92 @@ PrestoServer::PrestoServer(const std::string& configDirectoryPath) PrestoServer::~PrestoServer() {} void PrestoServer::run() { + initializeConfigs(); + + registerFileSystems(); + registerFileSinks(); + registerFileReadersAndWriters(); + registerMemoryArbitrators(); + registerShuffleInterfaceFactories(); + registerCustomOperators(); + + // We need to register cuDF before the connectors so that the cuDF connector + // factories can be used. + registerVeloxCudf(); + + // Register Presto connector factories and connectors + registerConnectors(); + + initializeVeloxMemory(); + initializeThreadPools(); + + auto catalogNames = registerVeloxConnectors(fs::path(configDirectoryPath_)); + + initializeHttpServer(); + initializeCoordinatorDiscoverer(); + registerHttpEndpoints(); + + registerFunctions(); + registerRemoteFunctions(); + registerVectorSerdes(); + registerPrestoPlanNodeSerDe(); + registerTraceNodeFactories(); + registerDynamicFunctions(); + registerExchangeSources(); + + initializeTaskResources(); + registerListeners(); + + prestoServerOperations_ = + std::make_unique(taskManager_.get(), this); + registerSystemConnector(); + + // The endpoint used by operation in production. + httpServer_->registerGet( + "/v1/operation/.*", + [this]( + proxygen::HTTPMessage* message, + const std::vector>& /*body*/, + proxygen::ResponseHandler* downstream) { + prestoServerOperations_->runOperation(message, downstream); + }); + + logExecutorInfo(); + + PRESTO_STARTUP_LOG(INFO) << "Starting all periodic tasks"; + + auto* memoryAllocator = velox::memory::memoryManager()->allocator(); + auto* asyncDataCache = velox::cache::AsyncDataCache::getInstance(); + periodicTaskManager_ = std::make_unique( + driverCpuExecutor_, + spillerCpuExecutor_, + httpSrvIoExecutor_.get(), + httpSrvCpuExecutor_.get(), + exchangeHttpIoExecutor_.get(), + exchangeHttpCpuExecutor_.get(), + taskManager_.get(), + memoryAllocator, + asyncDataCache, + velox::connector::getAllConnectors(), + this); + addServerPeriodicTasks(); + addAdditionalPeriodicTasks(); + periodicTaskManager_->start(); + createPeriodicMemoryChecker(); + if (memoryChecker_ != nullptr) { + memoryChecker_->start(); + } + + // Start everything. After the return from the following call we are shutting + // down. + startServer(catalogNames); + + shutdownServer(); +} + +void PrestoServer::initializeConfigs() { auto systemConfig = SystemConfig::instance(); auto nodeConfig = NodeConfig::instance(); - int httpPort{0}; - - std::string certPath; - std::string keyPath; - std::string ciphers; - std::string clientCertAndKeyPath; - std::optional httpsPort; try { // Allow registering extra config properties before we load them from files. @@ -282,12 +363,12 @@ void PrestoServer::run() { nodeConfig->initialize( fmt::format("{}/node.properties", configDirectoryPath_)); - httpPort = systemConfig->httpServerHttpPort(); + httpPort_ = systemConfig->httpServerHttpPort(); if (systemConfig->httpServerHttpsEnabled()) { - httpsPort = systemConfig->httpServerHttpsPort(); + httpsPort_ = systemConfig->httpServerHttpsPort(); - ciphers = systemConfig->httpsSupportedCiphers(); - if (ciphers.empty()) { + ciphers_ = systemConfig->httpsSupportedCiphers(); + if (ciphers_.empty()) { VELOX_USER_FAIL("Https is enabled without ciphers"); } @@ -295,13 +376,13 @@ void PrestoServer::run() { if (!optionalCertPath.has_value()) { VELOX_USER_FAIL("Https is enabled without certificate path"); } - certPath = optionalCertPath.value(); + certPath_ = optionalCertPath.value(); auto optionalKeyPath = systemConfig->httpsKeyPath(); if (!optionalKeyPath.has_value()) { VELOX_USER_FAIL("Https is enabled without key path"); } - keyPath = optionalKeyPath.value(); + keyPath_ = optionalKeyPath.value(); auto optionalClientCertPath = systemConfig->httpsClientCertAndKeyPath(); if (!optionalClientCertPath.has_value()) { @@ -313,7 +394,7 @@ void PrestoServer::run() { sslContext_ = util::createSSLContext( optionalClientCertPath.value(), - ciphers, + ciphers_, systemConfig->httpClientHttp2Enabled()); } @@ -342,64 +423,45 @@ void PrestoServer::run() { PRESTO_STARTUP_LOG(ERROR) << "Failed to start server due to " << e.what(); exit(EXIT_FAILURE); } +} - registerFileSystems(); - registerFileSinks(); - registerFileReadersAndWriters(); - registerMemoryArbitrators(); - registerShuffleInterfaceFactories(); - registerCustomOperators(); - - // We need to register cuDF before the connectors so that the cuDF connector - // factories can be used. - registerVeloxCudf(); - - // Register Presto connector factories and connectors - registerConnectors(); - - initializeVeloxMemory(); - initializeThreadPools(); - - auto catalogNames = registerVeloxConnectors(fs::path(configDirectoryPath_)); - +void PrestoServer::initializeHttpServer() { + auto systemConfig = SystemConfig::instance(); const bool bindToNodeInternalAddressOnly = systemConfig->httpServerBindToNodeInternalAddressOnlyEnabled(); folly::SocketAddress httpSocketAddress; if (bindToNodeInternalAddressOnly) { - httpSocketAddress.setFromHostPort(address_, httpPort); + httpSocketAddress.setFromHostPort(address_, httpPort_); } else { - httpSocketAddress.setFromLocalPort(httpPort); + httpSocketAddress.setFromLocalPort(httpPort_); } PRESTO_STARTUP_LOG(INFO) << fmt::format( "Starting server at {}:{} ({})", httpSocketAddress.getIPAddress().str(), - httpPort, + httpPort_, address_); - initializeCoordinatorDiscoverer(); - - const bool reusePort = SystemConfig::instance()->httpServerReusePort(); + const bool reusePort = systemConfig->httpServerReusePort(); auto httpConfig = std::make_unique(httpSocketAddress, reusePort); std::unique_ptr httpsConfig; - if (httpsPort.has_value()) { + if (httpsPort_.has_value()) { folly::SocketAddress httpsSocketAddress; if (bindToNodeInternalAddressOnly) { - httpsSocketAddress.setFromHostPort(address_, httpsPort.value()); + httpsSocketAddress.setFromHostPort(address_, httpsPort_.value()); } else { - httpsSocketAddress.setFromLocalPort(httpsPort.value()); + httpsSocketAddress.setFromLocalPort(httpsPort_.value()); } - const bool http2Enabled = - SystemConfig::instance()->httpServerHttp2Enabled(); + const bool http2Enabled = systemConfig->httpServerHttp2Enabled(); const std::string clientCaFile = - SystemConfig::instance()->httpsClientCaFile().value_or(""); + systemConfig->httpsClientCaFile().value_or(""); httpsConfig = std::make_unique( httpsSocketAddress, - certPath, - keyPath, - ciphers, + certPath_, + keyPath_, + ciphers_, reusePort, http2Enabled, clientCaFile); @@ -407,6 +469,10 @@ void PrestoServer::run() { httpServer_ = std::make_unique( httpSrvIoExecutor_, std::move(httpConfig), std::move(httpsConfig)); +} + +void PrestoServer::registerHttpEndpoints() { + auto systemConfig = SystemConfig::instance(); httpServer_->registerPost( "/v1/memory", @@ -495,13 +561,9 @@ void PrestoServer::run() { }); } } - registerFunctions(); - registerRemoteFunctions(); - registerVectorSerdes(); - registerPrestoPlanNodeSerDe(); - registerTraceNodeFactories(); - registerDynamicFunctions(); +} +void PrestoServer::registerExchangeSources() { facebook::velox::exec::ExchangeSource::registerFactory( [this]( const std::string& taskId, @@ -525,6 +587,10 @@ void PrestoServer::run() { // Batch broadcast exchange source. velox::exec::ExchangeSource::registerFactory( operators::BroadcastExchangeSource::createExchangeSource); +} + +void PrestoServer::initializeTaskResources() { + auto systemConfig = SystemConfig::instance(); pool_ = velox::memory::MemoryManager::getInstance()->addLeafPool("PrestoServer"); @@ -554,6 +620,11 @@ void PrestoServer::run() { getVeloxPlanValidator(), *taskManager_); taskResource_->registerUris(*httpServer_); +} + +void PrestoServer::registerListeners() { + auto systemConfig = SystemConfig::instance(); + if (systemConfig->enableSerializedPageChecksum()) { enableChecksum(); } @@ -573,19 +644,10 @@ void PrestoServer::run() { velox::exec::registerExprSetListener(listener); } } - prestoServerOperations_ = - std::make_unique(taskManager_.get(), this); - registerSystemConnector(); +} - // The endpoint used by operation in production. - httpServer_->registerGet( - "/v1/operation/.*", - [this]( - proxygen::HTTPMessage* message, - const std::vector>& /*body*/, - proxygen::ResponseHandler* downstream) { - prestoServerOperations_->runOperation(message, downstream); - }); +void PrestoServer::logExecutorInfo() { + auto systemConfig = SystemConfig::instance(); PRESTO_STARTUP_LOG(INFO) << "Driver CPU executor '" << driverCpuExecutor_->getName() << "' has " @@ -614,108 +676,89 @@ void PrestoServer::run() { } else { PRESTO_STARTUP_LOG(INFO) << "Spill executor was not configured."; } +} - PRESTO_STARTUP_LOG(INFO) << "Starting all periodic tasks"; +void PrestoServer::startServer(const std::vector& catalogNames) { + auto systemConfig = SystemConfig::instance(); - auto* memoryAllocator = velox::memory::memoryManager()->allocator(); - auto* asyncDataCache = velox::cache::AsyncDataCache::getInstance(); - periodicTaskManager_ = std::make_unique( - driverCpuExecutor_, - spillerCpuExecutor_, - httpSrvIoExecutor_.get(), - httpSrvCpuExecutor_.get(), - exchangeHttpIoExecutor_.get(), - exchangeHttpCpuExecutor_.get(), - taskManager_.get(), - memoryAllocator, - asyncDataCache, - velox::connector::getAllConnectors(), - this); - addServerPeriodicTasks(); - addAdditionalPeriodicTasks(); - periodicTaskManager_->start(); - createPeriodicMemoryChecker(); - if (memoryChecker_ != nullptr) { - memoryChecker_->start(); - } + auto startupOptions = systemConfig->httpServerStartupOptions(); + httpServer_->start( + std::move(startupOptions), + getHttpServerFilters(), + [&](proxygen::HTTPServer* server) { + const auto addresses = server->addresses(); + for (auto address : addresses) { + PRESTO_STARTUP_LOG(INFO) << fmt::format( + "Server listening at {}:{} - https {}", + address.address.getIPAddress().str(), + address.address.getPort(), + address.sslConfigs.size() != 0); + // We could be bound to both http and https ports. + // If set, we must use the https port and skip http. + if (httpsPort_.has_value() && address.sslConfigs.size() == 0) { + continue; + } - auto setTaskUriCb = [&](bool useHttps, int port) { - std::string taskUri; - if (useHttps) { - taskUri = fmt::format(kTaskUriFormat, kHttps, address_, port); - } else { - taskUri = fmt::format(kTaskUriFormat, kHttp, address_, port); - } - taskManager_->setBaseUri(taskUri); - }; - - auto startAnnouncerAndHeartbeatManagerCb = [&](bool useHttps, int port) { - if (coordinatorDiscoverer_ != nullptr) { - announcer_ = std::make_unique( - address_, - useHttps, - port, - coordinatorDiscoverer_, - nodeVersion_, - environment_, - nodeId_, - nodeLocation_, - nodePoolType_, - systemConfig->prestoNativeSidecar(), - catalogNames, - systemConfig->announcementMaxFrequencyMs(), - sslContext_); - updateAnnouncerDetails(); - announcer_->start(); - - uint64_t heartbeatFrequencyMs = systemConfig->heartbeatFrequencyMs(); - if (heartbeatFrequencyMs > 0) { - heartbeatManager_ = std::make_unique( - address_, - port, - coordinatorDiscoverer_, - sslContext_, - [server = this]() { return server->fetchNodeStatus(); }, - heartbeatFrequencyMs); - heartbeatManager_->start(); - } - } - }; + if (coordinatorDiscoverer_ != nullptr) { + announcer_ = std::make_unique( + address_, + httpsPort_.has_value(), + address.address.getPort(), + coordinatorDiscoverer_, + nodeVersion_, + environment_, + nodeId_, + nodeLocation_, + nodePoolType_, + systemConfig->prestoNativeSidecar(), + catalogNames, + systemConfig->announcementMaxFrequencyMs(), + sslContext_); + updateAnnouncerDetails(); + announcer_->start(); + + uint64_t heartbeatFrequencyMs = + systemConfig->heartbeatFrequencyMs(); + if (heartbeatFrequencyMs > 0) { + heartbeatManager_ = std::make_unique( + address_, + address.address.getPort(), + coordinatorDiscoverer_, + sslContext_, + [server = this]() { return server->fetchNodeStatus(); }, + heartbeatFrequencyMs); + heartbeatManager_->start(); + } + } - // Start everything. After the return from the following call we are shutting - // down. - httpServer_->start(getHttpServerFilters(), [&](proxygen::HTTPServer* server) { - const auto addresses = server->addresses(); - for (auto address : addresses) { - PRESTO_STARTUP_LOG(INFO) << fmt::format( - "Server listening at {}:{} - https {}", - address.address.getIPAddress().str(), - address.address.getPort(), - address.sslConfigs.size() != 0); - // We could be bound to both http and https ports. - // If set, we must use the https port and skip http. - if (httpsPort.has_value() && address.sslConfigs.size() == 0) { - continue; - } - startAnnouncerAndHeartbeatManagerCb( - httpsPort.has_value(), address.address.getPort()); - setTaskUriCb(httpsPort.has_value(), address.address.getPort()); - break; - } + std::string taskUri; + if (httpsPort_.has_value()) { + taskUri = fmt::format( + kTaskUriFormat, kHttps, address_, address.address.getPort()); + } else { + taskUri = fmt::format( + kTaskUriFormat, kHttp, address_, address.address.getPort()); + } + taskManager_->setBaseUri(taskUri); + break; + } - if (coordinatorDiscoverer_ != nullptr) { - VELOX_CHECK_NOT_NULL( - announcer_, - "The announcer is expected to have been created but wasn't."); - const auto heartbeatFrequencyMs = systemConfig->heartbeatFrequencyMs(); - if (heartbeatFrequencyMs > 0) { - VELOX_CHECK_NOT_NULL( - heartbeatManager_, - "The heartbeat manager is expected to have been created but wasn't."); - } - } - }); + if (coordinatorDiscoverer_ != nullptr) { + VELOX_CHECK_NOT_NULL( + announcer_, + "The announcer is expected to have been created but wasn't."); + const auto heartbeatFrequencyMs = + systemConfig->heartbeatFrequencyMs(); + if (heartbeatFrequencyMs > 0) { + VELOX_CHECK_NOT_NULL( + heartbeatManager_, + "The heartbeat manager is expected to have been created but wasn't."); + } + } + }); +} +void PrestoServer::stopAnnouncer() { if (announcer_ != nullptr) { PRESTO_SHUTDOWN_LOG(INFO) << "Stopping announcer"; announcer_->stop(); @@ -725,29 +768,9 @@ void PrestoServer::run() { PRESTO_SHUTDOWN_LOG(INFO) << "Stopping Heartbeat manager"; heartbeatManager_->stop(); } +} - PRESTO_SHUTDOWN_LOG(INFO) << "Stopping all periodic tasks"; - - if (memoryChecker_ != nullptr) { - memoryChecker_->stop(); - } - periodicTaskManager_->stop(); - stopAdditionalPeriodicTasks(); - - // Destroy entities here to ensure we won't get any messages after Server - // object is gone and to have nice log in case shutdown gets stuck. - PRESTO_SHUTDOWN_LOG(INFO) << "Destroying Task Resource"; - taskResource_.reset(); - PRESTO_SHUTDOWN_LOG(INFO) << "Destroying Task Manager"; - taskManager_.reset(); - PRESTO_SHUTDOWN_LOG(INFO) << "Destroying HTTP Server"; - httpServer_.reset(); - - unregisterFileReadersAndWriters(); - unregisterFileSystems(); - unregisterConnectors(); - unregisterVeloxCudf(); - +void PrestoServer::joinExecutors() { PRESTO_SHUTDOWN_LOG(INFO) << "Joining Driver CPU Executor '" << driverCpuExecutor_->getName() << "': threads: " << driverCpuExecutor_->numActiveThreads() << "/" @@ -835,6 +858,34 @@ void PrestoServer::run() { << "': threads: " << pGlobalIOExecutor->numActiveThreads() << "/" << pGlobalIOExecutor->numThreads(); } +} + +void PrestoServer::shutdownServer() { + stopAnnouncer(); + + PRESTO_SHUTDOWN_LOG(INFO) << "Stopping all periodic tasks"; + + if (memoryChecker_ != nullptr) { + memoryChecker_->stop(); + } + periodicTaskManager_->stop(); + stopAdditionalPeriodicTasks(); + + // Destroy entities here to ensure we won't get any messages after Server + // object is gone and to have nice log in case shutdown gets stuck. + PRESTO_SHUTDOWN_LOG(INFO) << "Destroying Task Resource"; + taskResource_.reset(); + PRESTO_SHUTDOWN_LOG(INFO) << "Destroying Task Manager"; + taskManager_.reset(); + PRESTO_SHUTDOWN_LOG(INFO) << "Destroying HTTP Server"; + httpServer_.reset(); + + unregisterFileReadersAndWriters(); + unregisterFileSystems(); + unregisterConnectors(); + unregisterVeloxCudf(); + + joinExecutors(); if (cache_ != nullptr) { PRESTO_SHUTDOWN_LOG(INFO) << "Shutdown AsyncDataCache"; @@ -1073,7 +1124,9 @@ void PrestoServer::initializeVeloxMemory() { velox::cache::AsyncDataCache::Options cacheOptions{ systemConfig->asyncCacheMaxSsdWriteRatio(), systemConfig->asyncCacheSsdSavableRatio(), - systemConfig->asyncCacheMinSsdSavableBytes()}; + systemConfig->asyncCacheMinSsdSavableBytes(), + systemConfig->asyncCacheNumShards(), + systemConfig->asyncCacheSsdFlushThresholdBytes()}; cache_ = velox::cache::AsyncDataCache::create( velox::memory::memoryManager()->allocator(), std::move(ssd), @@ -1194,6 +1247,16 @@ void PrestoServer::addServerPeriodicTasks() { 1'000'000, // 1 second "populate_mem_cpu_info"); + periodicTaskManager_->addTask( + [start = start_]() { + const auto seconds = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + RECORD_METRIC_VALUE(kCounterWorkerRuntimeUptimeSecs, seconds); + }, + 2'000'000, // 2 seconds + "worker_runtime_uptime_secs"); + const auto timeslice = SystemConfig::instance()->taskRunTimeSliceMicros(); if (timeslice > 0) { periodicTaskManager_->addTask( @@ -1435,6 +1498,11 @@ void PrestoServer::registerFunctions() { velox::connector::hasConnector("hive-hadoop2")) { hive::functions::registerHiveNativeFunctions(); } + +#ifdef PRESTO_ENABLE_THETA_SKETCH + functions::aggregate::theta_sketch::registerAllThetaSketchFunctions( + prestoBuiltinFunctionPrefix_); +#endif } void PrestoServer::registerRemoteFunctions() { @@ -1470,15 +1538,13 @@ void PrestoServer::registerVectorSerdes() { if (!velox::isRegisteredVectorSerde()) { velox::serializer::presto::PrestoVectorSerde::registerVectorSerde(); } - if (!velox::isRegisteredNamedVectorSerde(velox::VectorSerde::Kind::kPresto)) { + if (!velox::isRegisteredNamedVectorSerde("Presto")) { velox::serializer::presto::PrestoVectorSerde::registerNamedVectorSerde(); } - if (!velox::isRegisteredNamedVectorSerde( - velox::VectorSerde::Kind::kCompactRow)) { + if (!velox::isRegisteredNamedVectorSerde("CompactRow")) { velox::serializer::CompactRowVectorSerde::registerNamedVectorSerde(); } - if (!velox::isRegisteredNamedVectorSerde( - velox::VectorSerde::Kind::kUnsafeRow)) { + if (!velox::isRegisteredNamedVectorSerde("UnsafeRow")) { velox::serializer::spark::UnsafeRowVectorSerde::registerNamedVectorSerde(); } } diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 5a240a13c44c9..085c76ec7947a 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -204,6 +204,45 @@ class PrestoServer { void addServerPeriodicTasks(); + /// Loads config files, validates HTTPS/JWT settings, and populates member + /// variables from config. Called at the start of run(). + void initializeConfigs(); + + /// Creates the HTTP server with appropriate socket bindings and + /// optional HTTPS configuration. + void initializeHttpServer(); + + /// Registers all HTTP API endpoints on the HTTP server. + void registerHttpEndpoints(); + + /// Registers exchange source factories for Presto exchange, shuffle, and + /// broadcast exchange. + void registerExchangeSources(); + + /// Creates memory pools, task manager, task resource, and related + /// components. + void initializeTaskResources(); + + /// Registers task, split, and expression set listeners based on config. + void registerListeners(); + + /// Starts the HTTP server (blocking), announcer, and heartbeat manager. + /// Returns when the server is stopped. + void startServer(const std::vector& catalogNames); + + /// Shuts down all server components in the correct order after the HTTP + /// server stops. + void shutdownServer(); + + /// Logs thread pool sizes for all executors. + void logExecutorInfo(); + + /// Stops announcer and heartbeat manager. + void stopAnnouncer(); + + /// Joins all thread pool executors in the correct shutdown order. + void joinExecutors(); + void reportMemoryInfo(proxygen::ResponseHandler* downstream); void reportServerInfo(proxygen::ResponseHandler* downstream); @@ -322,6 +361,14 @@ class PrestoServer { std::string nodePoolType_; folly::SSLContextPtr sslContext_; std::string prestoBuiltinFunctionPrefix_; + + // HTTP/HTTPS configuration populated by initializeConfigs() and consumed + // by initializeHttpServer() and startServer(). + int httpPort_{0}; + std::optional httpsPort_; + std::string certPath_; + std::string keyPath_; + std::string ciphers_; }; } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/PrestoServerOperations.cpp b/presto-native-execution/presto_cpp/main/PrestoServerOperations.cpp index 1f85c15fa8e74..acd577ef385d1 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServerOperations.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServerOperations.cpp @@ -174,7 +174,7 @@ std::string PrestoServerOperations::veloxQueryConfigOperation( ServerOperation::targetString(op.target), ServerOperation::actionString(op.action)); return fmt::format( - "Have set system property value '{}' to '{}'. Old value was '{}'.\n", + "Have set velox query config property '{}' to '{}'. Old value was '{}'.\n", name, value, SystemConfig::instance() diff --git a/presto-native-execution/presto_cpp/main/PrestoTask.cpp b/presto-native-execution/presto_cpp/main/PrestoTask.cpp index afdf9f4f82dc5..7e616f5266938 100644 --- a/presto-native-execution/presto_cpp/main/PrestoTask.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoTask.cpp @@ -45,6 +45,29 @@ std::string prestoTaskStateString(PrestoTaskState state) { namespace { +// Splits operator stats for operators that represent multiple plan nodes in +// the Presto plan. Currently only IndexLookupJoin needs expansion because it +// embeds IndexSource as a separate logical plan node. FilterProject is +// intentionally not expanded here because Presto's PlanPrinter displays it as +// a single combined node. +std::vector splitOperatorStats( + const std::vector& operatorStats) { + std::vector expanded; + expanded.reserve(operatorStats.size()); + for (const auto& opStats : operatorStats) { + if (opStats.operatorType == "IndexLookupJoin" && + opStats.statsSplitter.has_value()) { + auto splitStats = opStats.statsSplitter.value()(opStats); + for (auto& s : splitStats) { + expanded.push_back(std::move(s)); + } + } else { + expanded.push_back(opStats); + } + } + return expanded; +} + #define TASK_STATS_SUM(taskStats, statsName, taskStatusSum) \ do { \ for (int i = 0; i < taskStats.pipelineStats.size(); ++i) { \ @@ -356,8 +379,13 @@ void updatePipelineStats( prestoPipelineStats.lastStartTimeInMillis = prestoTaskStats.endTimeInMillis; prestoPipelineStats.lastEndTimeInMillis = prestoTaskStats.endTimeInMillis; - prestoPipelineStats.operatorSummaries.resize( - veloxPipelineStats.operatorStats.size()); + // Split operator stats for operators that represent multiple plan nodes + // in the Presto plan (e.g., IndexLookupJoin -> IndexLookupJoin + + // IndexSource). + const auto expandedOperatorStats = + splitOperatorStats(veloxPipelineStats.operatorStats); + + prestoPipelineStats.operatorSummaries.resize(expandedOperatorStats.size()); prestoPipelineStats.totalScheduledTimeInNanos = {}; prestoPipelineStats.totalCpuTimeInNanos = {}; prestoPipelineStats.totalBlockedTimeInNanos = {}; @@ -367,9 +395,9 @@ void updatePipelineStats( // tasks may fail before any operators are created; // collect stats only when we have operators - if (!veloxPipelineStats.operatorStats.empty()) { - const auto& firstVeloxOpStats = veloxPipelineStats.operatorStats[0]; - const auto& lastVeloxOpStats = veloxPipelineStats.operatorStats.back(); + if (!expandedOperatorStats.empty()) { + const auto& firstVeloxOpStats = expandedOperatorStats[0]; + const auto& lastVeloxOpStats = expandedOperatorStats.back(); prestoPipelineStats.pipelineId = firstVeloxOpStats.pipelineId; prestoPipelineStats.totalDrivers = firstVeloxOpStats.numDrivers; @@ -384,9 +412,9 @@ void updatePipelineStats( prestoPipelineStats.outputDataSizeInBytes = lastVeloxOpStats.outputBytes; } - for (auto j = 0; j < veloxPipelineStats.operatorStats.size(); ++j) { + for (auto j = 0; j < expandedOperatorStats.size(); ++j) { auto& prestoOp = prestoPipelineStats.operatorSummaries[j]; - auto& veloxOp = veloxPipelineStats.operatorStats[j]; + const auto& veloxOp = expandedOperatorStats[j]; prestoOp.stageId = taskId.stageId(); prestoOp.stageExecutionId = taskId.stageExecutionId(); diff --git a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp index 2929ccfcd7c0b..0c3859b6c93e2 100644 --- a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp @@ -124,6 +124,20 @@ void updateFromSystemConfigs( .veloxConfig = velox::core::QueryConfig::kSpillFileCreateConfig, }, + { + .prestoSystemConfig = + std::string(SystemConfig::kSpillerAggregationFileCreateConfig), + .veloxConfig = + velox::core::QueryConfig::kAggregationSpillFileCreateConfig, + }, + + { + .prestoSystemConfig = + std::string(SystemConfig::kSpillerHashJoinFileCreateConfig), + .veloxConfig = + velox::core::QueryConfig::kHashJoinSpillFileCreateConfig, + }, + {.prestoSystemConfig = std::string(SystemConfig::kSpillEnabled), .veloxConfig = velox::core::QueryConfig::kSpillEnabled}, @@ -137,6 +151,9 @@ void updateFromSystemConfigs( std::string(SystemConfig::kAggregationSpillEnabled), .veloxConfig = velox::core::QueryConfig::kAggregationSpillEnabled}, + {.prestoSystemConfig = std::string(SystemConfig::kMaxSpillBytes), + .veloxConfig = velox::core::QueryConfig::kMaxSpillBytes}, + {.prestoSystemConfig = std::string(SystemConfig::kRequestDataSizesMaxWaitSec), .veloxConfig = velox::core::QueryConfig::kRequestDataSizesMaxWaitSec}, diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index 39781c0e778c3..b9a45fd40226f 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -143,6 +143,26 @@ SessionProperties::SessionProperties() { QueryConfig::kSpillFileCreateConfig, c.spillFileCreateConfig()); + addSessionProperty( + kAggregationSpillFileCreateConfig, + "Native Execution only. Config used to create aggregation spill files. " + "This config is provided to underlying file system and the config is " + "free form. The form should be defined by the underlying file system.", + VARCHAR(), + false, + QueryConfig::kAggregationSpillFileCreateConfig, + c.aggregationSpillFileCreateConfig()); + + addSessionProperty( + kHashJoinSpillFileCreateConfig, + "Native Execution only. Config used to create hash join spill files. " + "This config is provided to underlying file system and the config is " + "free form. The form should be defined by the underlying file system.", + VARCHAR(), + false, + QueryConfig::kHashJoinSpillFileCreateConfig, + c.hashJoinSpillFileCreateConfig()); + addSessionProperty( kJoinSpillEnabled, "Native Execution only. Enable join spilling on native engine", @@ -351,6 +371,15 @@ SessionProperties::SessionProperties() { QueryConfig::kMaxPartitionedOutputBufferSize, std::to_string(c.maxPartitionedOutputBufferSize())); + addSessionProperty( + kPartitionedOutputEagerFlush, + "If true, the PartitionedOutput operator will flush rows eagerly, without" + " waiting until buffers reach a certain size. Default is false.", + BOOLEAN(), + false, + QueryConfig::kPartitionedOutputEagerFlush, + "false"); + // If `legacy_timestamp` is true, the coordinator expects timestamp // conversions without a timezone to be converted to the user's // session_timezone. @@ -569,6 +598,18 @@ SessionProperties::SessionProperties() { QueryConfig::kMaxOutputBatchRows, std::to_string(c.maxOutputBatchRows())); + addSessionProperty( + kMergeJoinOutputBatchStartSize, + "Initial output batch size in rows for MergeJoin operator. When non-zero, " + "the batch size starts at this value and is dynamically adjusted based on " + "the average row size of previous output batches. When zero (default), " + "dynamic adjustment is disabled and the batch size is fixed at " + "preferredOutputBatchRows.", + INTEGER(), + false, + QueryConfig::kMergeJoinOutputBatchStartSize, + std::to_string(c.mergeJoinOutputBatchStartSize())); + addSessionProperty( kRowSizeTrackingMode, "Enable (reader) row size tracker as a fallback to file level row size estimates.", @@ -609,6 +650,17 @@ SessionProperties::SessionProperties() { false, QueryConfig::kAggregationCompactionUnusedMemoryRatio, std::to_string(c.aggregationCompactionUnusedMemoryRatio())); + + addSessionProperty( + kAggregationMemoryCompactionReclaimEnabled, + "If true, enables lightweight memory compaction before spilling during " + "memory reclaim in aggregation. When enabled, the aggregation operator " + "will try to compact aggregate function state (for example, free dead strings) " + "before resorting to spilling. Disabled by default.", + BOOLEAN(), + false, + QueryConfig::kAggregationMemoryCompactionReclaimEnabled, + boolToString(c.aggregationMemoryCompactionReclaimEnabled())); } const std::string SessionProperties::toVeloxConfig( diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index ebe416e1e595c..aa8a5a93b3b14 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -128,6 +128,16 @@ class SessionProperties { static constexpr const char* kSpillFileCreateConfig = "native_spill_file_create_config"; + /// Config used to create aggregation spill files. This config is provided to + /// underlying file system and the config is free form. + static constexpr const char* kAggregationSpillFileCreateConfig = + "native_aggregation_spill_file_create_config"; + + /// Config used to create hash join spill files. This config is provided to + /// underlying file system and the config is free form. + static constexpr const char* kHashJoinSpillFileCreateConfig = + "native_hash_join_spill_file_create_config"; + /// Enable window spilling on native engine. static constexpr const char* kWindowSpillEnabled = "native_window_spill_enabled"; @@ -272,6 +282,11 @@ class SessionProperties { static constexpr const char* kMaxPartitionedOutputBufferSize = "native_max_page_partitioning_buffer_size"; + /// If true, the PartitionedOutput operator will flush rows eagerly, without + /// waiting until buffers reach certain size. Default is false. + static constexpr const char* kPartitionedOutputEagerFlush = + "native_partitioned_output_eager_flush"; + /// Maximum number of partitions created by a local exchange. /// Affects concurrency for pipelines containing LocalPartitionNode. static constexpr const char* kMaxLocalExchangePartitionCount = @@ -368,6 +383,14 @@ class SessionProperties { /// output rows. static constexpr const char* kMaxOutputBatchRows = "max_output_batch_rows"; + /// Initial output batch size in rows for MergeJoin operator. When non-zero, + /// the batch size starts at this value and is dynamically adjusted based on + /// the average row size of previous output batches. When zero (default), + /// dynamic adjustment is disabled and the batch size is fixed at + /// preferredOutputBatchRows. + static constexpr const char* kMergeJoinOutputBatchStartSize = + "native_merge_join_output_batch_start_size"; + /// Enable (reader) row size tracker as a fallback to file level row size /// estimates. static constexpr const char* kRowSizeTrackingMode = "row_size_tracking_mode"; @@ -396,6 +419,13 @@ class SessionProperties { static constexpr const char* kAggregationCompactionUnusedMemoryRatio = "native_aggregation_compaction_unused_memory_ratio"; + /// If true, enables lightweight memory compaction before spilling during + /// memory reclaim in aggregation. When enabled, the aggregation operator + /// will try to compact aggregate function state (e.g., free dead strings) + /// before resorting to spilling. Disabled by default. + static constexpr const char* kAggregationMemoryCompactionReclaimEnabled = + "native_aggregation_memory_compaction_reclaim_enabled"; + inline bool hasVeloxConfig(const std::string& key) { auto sessionProperty = sessionProperties_.find(key); if (sessionProperty == sessionProperties_.end()) { diff --git a/presto-native-execution/presto_cpp/main/TaskResource.cpp b/presto-native-execution/presto_cpp/main/TaskResource.cpp index 736fc489ea062..b3c41bd0d3463 100644 --- a/presto-native-execution/presto_cpp/main/TaskResource.cpp +++ b/presto-native-execution/presto_cpp/main/TaskResource.cpp @@ -55,6 +55,59 @@ std::optional getMaxWait(proxygen::HTTPMessage* message) { return protocol::Duration( headers.getSingleOrEmpty(protocol::PRESTO_MAX_WAIT_HTTP_HEADER)); } + +bool shouldUseThrift(const proxygen::HTTPMessage& message) { + const auto& acceptHeader = + message.getHeaders().getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); + return acceptHeader.find(http::kMimeTypeApplicationThrift) != + std::string::npos; +} + +template +void sendPrestoResponse( + proxygen::ResponseHandler* downstream, + const T& data, + bool sendThrift) { + if (sendThrift) { + ThriftT thriftData; + toThrift(data, thriftData); + http::sendOkThriftResponse(downstream, thriftWrite(thriftData)); + } else { + http::sendOkResponse(downstream, json(data)); + } +} + +/// Creates a CallbackRequestHandler that executes a void work function on the +/// given executor, then sends an empty OK response. On exception, sends an +/// error response. Used for simple fire-and-forget handlers. +template +proxygen::RequestHandler* executeAndRespond( + folly::Executor* executor, + WorkFn&& workFn) { + return new http::CallbackRequestHandler( + [executor, work = std::forward(workFn)]( + proxygen::HTTPMessage* /*message*/, + const std::vector>& /*body*/, + proxygen::ResponseHandler* downstream, + std::shared_ptr handlerState) { + folly::via(executor, std::move(work)) + .via( + folly::getKeepAliveToken( + folly::EventBaseManager::get()->getEventBase())) + .thenValue([downstream, handlerState](auto&& /* unused */) { + if (!handlerState->requestExpired()) { + http::sendOkResponse(downstream); + } + }) + .thenError( + folly::tag_t{}, + [downstream, handlerState](auto&& e) { + if (!handlerState->requestExpired()) { + http::sendErrorResponse(downstream, e.what()); + } + }); + }); +} } // namespace void TaskResource::registerUris(http::HttpServer& server) { @@ -136,34 +189,9 @@ proxygen::RequestHandler* TaskResource::abortResults( const std::vector& pathMatch) { protocol::TaskId taskId = pathMatch[1]; long destination = folly::to(pathMatch[2]); - return new http::CallbackRequestHandler( - [this, taskId, destination]( - proxygen::HTTPMessage* /*message*/, - const std::vector>& /*body*/, - proxygen::ResponseHandler* downstream, - std::shared_ptr handlerState) { - folly::via( - httpSrvCpuExecutor_, - [this, taskId, destination, handlerState]() { - taskManager_.abortResults(taskId, destination); - return true; - }) - .via( - folly::getKeepAliveToken( - folly::EventBaseManager::get()->getEventBase())) - .thenValue([downstream, handlerState](auto&& /* unused */) { - if (!handlerState->requestExpired()) { - http::sendOkResponse(downstream); - } - }) - .thenError( - folly::tag_t{}, - [downstream, handlerState](auto&& e) { - if (!handlerState->requestExpired()) { - http::sendErrorResponse(downstream, e.what()); - } - }); - }); + return executeAndRespond(httpSrvCpuExecutor_, [this, taskId, destination]() { + taskManager_.abortResults(taskId, destination); + }); } proxygen::RequestHandler* TaskResource::acknowledgeResults( @@ -172,34 +200,9 @@ proxygen::RequestHandler* TaskResource::acknowledgeResults( protocol::TaskId taskId = pathMatch[1]; long bufferId = folly::to(pathMatch[2]); long token = folly::to(pathMatch[3]); - - return new http::CallbackRequestHandler( - [this, taskId, bufferId, token]( - proxygen::HTTPMessage* /*message*/, - const std::vector>& /*body*/, - proxygen::ResponseHandler* downstream, - std::shared_ptr handlerState) { - folly::via( - httpSrvCpuExecutor_, - [this, taskId, bufferId, token]() { - taskManager_.acknowledgeResults(taskId, bufferId, token); - return true; - }) - .via( - folly::getKeepAliveToken( - folly::EventBaseManager::get()->getEventBase())) - .thenValue([downstream, handlerState](auto&& /* unused */) { - if (!handlerState->requestExpired()) { - http::sendOkResponse(downstream); - } - }) - .thenError( - folly::tag_t{}, - [downstream, handlerState](auto&& e) { - if (!handlerState->requestExpired()) { - http::sendErrorResponse(downstream, e.what()); - } - }); + return executeAndRespond( + httpSrvCpuExecutor_, [this, taskId, bufferId, token]() { + taskManager_.acknowledgeResults(taskId, bufferId, token); }); } @@ -216,10 +219,7 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTaskImpl( bool summarize = message->hasQueryParam("summarize"); const auto& headers = message->getHeaders(); - const auto& acceptHeader = - headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); - const auto sendThrift = - acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; + const auto sendThrift = shouldUseThrift(*message); const auto& contentHeader = headers.getSingleOrEmpty(proxygen::HTTP_HEADER_CONTENT_TYPE); const auto receiveThrift = @@ -282,14 +282,8 @@ proxygen::RequestHandler* TaskResource::createOrUpdateTaskImpl( folly::EventBaseManager::get()->getEventBase())) .thenValue([downstream, handlerState, sendThrift](auto taskInfo) { if (!handlerState->requestExpired()) { - if (sendThrift) { - thrift::TaskInfo thriftTaskInfo; - toThrift(*taskInfo, thriftTaskInfo); - http::sendOkThriftResponse( - downstream, thriftWrite(thriftTaskInfo)); - } else { - http::sendOkResponse(downstream, json(*taskInfo)); - } + sendPrestoResponse( + downstream, *taskInfo, sendThrift); } }) .thenError( @@ -419,11 +413,7 @@ proxygen::RequestHandler* TaskResource::deleteTask( message->getQueryParam(protocol::PRESTO_ABORT_TASK_URL_PARAM) == "true"; } bool summarize = message->hasQueryParam("summarize"); - const auto& headers = message->getHeaders(); - const auto& acceptHeader = - headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); - const auto sendThrift = - acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; + const auto sendThrift = shouldUseThrift(*message); return new http::CallbackRequestHandler( [this, taskId, abort, summarize, sendThrift]( @@ -448,14 +438,8 @@ proxygen::RequestHandler* TaskResource::deleteTask( sendTaskNotFound(downstream, taskId); return; } - if (sendThrift) { - thrift::TaskInfo thriftTaskInfo; - toThrift(*taskInfo, thriftTaskInfo); - http::sendOkThriftResponse( - downstream, thriftWrite(thriftTaskInfo)); - } else { - http::sendOkResponse(downstream, json(*taskInfo)); - } + sendPrestoResponse( + downstream, *taskInfo, sendThrift); } }) .thenError( @@ -565,12 +549,7 @@ proxygen::RequestHandler* TaskResource::getTaskStatus( protocol::TaskId taskId = pathMatch[1]; auto currentState = getCurrentState(message); auto maxWait = getMaxWait(message); - - const auto& headers = message->getHeaders(); - const auto& acceptHeader = - headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); - const auto sendThrift = - acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; + const auto sendThrift = shouldUseThrift(*message); return new http::CallbackRequestHandler( [this, sendThrift, taskId, currentState, maxWait]( @@ -596,15 +575,10 @@ proxygen::RequestHandler* TaskResource::getTaskStatus( [sendThrift, downstream, taskId, handlerState]( std::unique_ptr taskStatus) { if (!handlerState->requestExpired()) { - if (sendThrift) { - thrift::TaskStatus thriftTaskStatus; - toThrift(*taskStatus, thriftTaskStatus); - http::sendOkThriftResponse( - downstream, thriftWrite(thriftTaskStatus)); - } else { - json taskStatusJson = *taskStatus; - http::sendOkResponse(downstream, taskStatusJson); - } + sendPrestoResponse< + protocol::TaskStatus, + thrift::TaskStatus>( + downstream, *taskStatus, sendThrift); } }) .thenError( @@ -629,12 +603,7 @@ proxygen::RequestHandler* TaskResource::getTaskInfo( auto currentState = getCurrentState(message); auto maxWait = getMaxWait(message); bool summarize = message->hasQueryParam("summarize"); - - const auto& headers = message->getHeaders(); - const auto& acceptHeader = - headers.getSingleOrEmpty(proxygen::HTTP_HEADER_ACCEPT); - const auto sendThrift = - acceptHeader.find(http::kMimeTypeApplicationThrift) != std::string::npos; + const auto sendThrift = shouldUseThrift(*message); return new http::CallbackRequestHandler( [this, taskId, currentState, maxWait, summarize, sendThrift]( @@ -661,14 +630,8 @@ proxygen::RequestHandler* TaskResource::getTaskInfo( .thenValue([downstream, taskId, handlerState, sendThrift]( std::unique_ptr taskInfo) { if (!handlerState->requestExpired()) { - if (sendThrift) { - thrift::TaskInfo thriftTaskInfo; - toThrift(*taskInfo, thriftTaskInfo); - http::sendOkThriftResponse( - downstream, thriftWrite(thriftTaskInfo)); - } else { - http::sendOkResponse(downstream, json(*taskInfo)); - } + sendPrestoResponse( + downstream, *taskInfo, sendThrift); } }) .thenError( @@ -690,33 +653,8 @@ proxygen::RequestHandler* TaskResource::removeRemoteSource( const std::vector& pathMatch) { protocol::TaskId taskId = pathMatch[1]; auto remoteId = pathMatch[2]; - - return new http::CallbackRequestHandler( - [this, taskId, remoteId]( - proxygen::HTTPMessage* /*message*/, - const std::vector>& /*body*/, - proxygen::ResponseHandler* downstream, - std::shared_ptr handlerState) { - folly::via( - httpSrvCpuExecutor_, - [this, taskId, remoteId, downstream]() { - taskManager_.removeRemoteSource(taskId, remoteId); - }) - .via( - folly::getKeepAliveToken( - folly::EventBaseManager::get()->getEventBase())) - .thenValue([downstream, handlerState](auto&& /* unused */) { - if (!handlerState->requestExpired()) { - http::sendOkResponse(downstream); - } - }) - .thenError( - folly::tag_t{}, - [downstream, handlerState](const std::exception& e) { - if (!handlerState->requestExpired()) { - http::sendErrorResponse(downstream, e.what()); - } - }); - }); + return executeAndRespond(httpSrvCpuExecutor_, [this, taskId, remoteId]() { + taskManager_.removeRemoteSource(taskId, remoteId); + }); } } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index 89f14085da3e2..1a6af3918b1a8 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -183,6 +183,8 @@ SystemConfig::SystemConfig() { std::round(0.5 * hardwareConcurrency())), NUM_PROP(kSpillerNumCpuThreadsHwMultiplier, 1.0), STR_PROP(kSpillerFileCreateConfig, ""), + STR_PROP(kSpillerAggregationFileCreateConfig, ""), + STR_PROP(kSpillerHashJoinFileCreateConfig, ""), STR_PROP(kSpillerDirectoryCreateConfig, ""), NONE_PROP(kSpillerSpillPath), NUM_PROP(kShutdownOnsetSec, 10), @@ -209,6 +211,8 @@ SystemConfig::SystemConfig() { NUM_PROP(kAsyncCacheMaxSsdWriteRatio, 0.7), NUM_PROP(kAsyncCacheSsdSavableRatio, 0.125), NUM_PROP(kAsyncCacheMinSsdSavableBytes, 1 << 24 /*16MB*/), + NUM_PROP(kAsyncCacheNumShards, 4), + NUM_PROP(kAsyncCacheSsdFlushThresholdBytes, 0), STR_PROP(kAsyncCachePersistenceInterval, "0s"), BOOL_PROP(kAsyncCacheSsdDisableFileCow, false), BOOL_PROP(kSsdCacheChecksumEnabled, false), @@ -377,6 +381,25 @@ bool SystemConfig::httpServerEnableGzipCompression() const { return optionalProperty(kHttpServerEnableGzipCompression).value(); } +http::HttpServerStartupOptions SystemConfig::httpServerStartupOptions() const { + http::HttpServerStartupOptions options; + options.idleTimeoutMs = httpServerIdleTimeoutMs(); + options.http2InitialReceiveWindow = httpServerHttp2InitialReceiveWindow(); + options.http2ReceiveStreamWindowSize = + httpServerHttp2ReceiveStreamWindowSize(); + options.http2ReceiveSessionWindowSize = + httpServerHttp2ReceiveSessionWindowSize(); + options.http2MaxConcurrentStreams = httpServerHttp2MaxConcurrentStreams(); + options.enableContentCompression = httpServerEnableContentCompression(); + options.contentCompressionLevel = httpServerContentCompressionLevel(); + options.contentCompressionMinimumSize = + httpServerContentCompressionMinimumSize(); + options.enableZstdCompression = httpServerEnableZstdCompression(); + options.zstdContentCompressionLevel = httpServerZstdContentCompressionLevel(); + options.enableGzipCompression = httpServerEnableGzipCompression(); + return options; +} + std::string SystemConfig::httpsSupportedCiphers() const { return optionalProperty(kHttpsSupportedCiphers).value(); } @@ -583,6 +606,16 @@ std::string SystemConfig::spillerFileCreateConfig() const { return optionalProperty(kSpillerFileCreateConfig).value(); } +std::string SystemConfig::spillerAggregationFileCreateConfig() const { + return optionalProperty(kSpillerAggregationFileCreateConfig) + .value(); +} + +std::string SystemConfig::spillerHashJoinFileCreateConfig() const { + return optionalProperty(kSpillerHashJoinFileCreateConfig) + .value(); +} + std::string SystemConfig::spillerDirectoryCreateConfig() const { return optionalProperty(kSpillerDirectoryCreateConfig).value(); } @@ -695,6 +728,14 @@ int32_t SystemConfig::asyncCacheMinSsdSavableBytes() const { return optionalProperty(kAsyncCacheMinSsdSavableBytes).value(); } +int32_t SystemConfig::asyncCacheNumShards() const { + return optionalProperty(kAsyncCacheNumShards).value(); +} + +uint64_t SystemConfig::asyncCacheSsdFlushThresholdBytes() const { + return optionalProperty(kAsyncCacheSsdFlushThresholdBytes).value(); +} + std::chrono::duration SystemConfig::asyncCachePersistenceInterval() const { return velox::config::toDuration( @@ -958,6 +999,20 @@ bool SystemConfig::httpClientConnectionReuseCounterEnabled() const { .value(); } +http::HttpClientOptions SystemConfig::httpClientOptions() const { + http::HttpClientOptions options; + options.http2Enabled = httpClientHttp2Enabled(); + options.http2MaxStreamsPerConnection = + httpClientHttp2MaxStreamsPerConnection(); + options.http2InitialStreamWindow = httpClientHttp2InitialStreamWindow(); + options.http2StreamWindow = httpClientHttp2StreamWindow(); + options.http2SessionWindow = httpClientHttp2SessionWindow(); + options.maxAllocateBytes = httpMaxAllocateBytes(); + options.connectionReuseCounterEnabled = + httpClientConnectionReuseCounterEnabled(); + return options; +} + std::chrono::duration SystemConfig::exchangeMaxErrorDuration() const { return velox::config::toDuration( optionalProperty(kExchangeMaxErrorDuration).value()); @@ -1022,6 +1077,17 @@ int32_t SystemConfig::internalCommunicationJwtExpirationSeconds() const { .value(); } +http::JwtOptions SystemConfig::jwtOptions() const { + http::JwtOptions options; + options.jwtEnabled = internalCommunicationJwtEnabled(); + if (options.jwtEnabled) { + options.sharedSecret = internalCommunicationSharedSecret(); + options.jwtExpirationSeconds = internalCommunicationJwtExpirationSeconds(); + options.nodeId = NodeConfig::instance()->nodeId(); + } + return options; +} + bool SystemConfig::useLegacyArrayAgg() const { return optionalProperty(kUseLegacyArrayAgg).value(); } diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index 2c44c3e845427..3b11127eab708 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -17,6 +17,9 @@ #include #include #include +#include "presto_cpp/main/http/HttpClientOptions.h" // @manual +#include "presto_cpp/main/http/HttpServerStartupOptions.h" // @manual +#include "presto_cpp/main/http/JwtOptions.h" // @manual #include "velox/common/config/Config.h" namespace facebook::presto { @@ -320,6 +323,18 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kSpillerFileCreateConfig{ "spiller.file-create-config"}; + /// Config used to create aggregation spill files. This config is provided to + /// underlying file system and the config is free form. The form should be + /// defined by the underlying file system. + static constexpr std::string_view kSpillerAggregationFileCreateConfig{ + "spiller.aggregation-file-create-config"}; + + /// Config used to create hash join spill files. This config is provided to + /// underlying file system and the config is free form. The form should be + /// defined by the underlying file system. + static constexpr std::string_view kSpillerHashJoinFileCreateConfig{ + "spiller.hash-join-file-create-config"}; + /// Config used to create spill directories. This config is provided to /// underlying file system and the config is free form. The form should be /// defined by the underlying file system. @@ -429,6 +444,18 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kAsyncCacheMinSsdSavableBytes{ "async-cache-min-ssd-savable-bytes"}; + /// The number of shards for the async data cache. The cache is divided into + /// shards to decrease contention on the mutex for the key to entry mapping + /// and other housekeeping. Must be a power of 2. + static constexpr std::string_view kAsyncCacheNumShards{ + "async-cache-num-shards"}; + + /// The maximum threshold in bytes for triggering SSD flush. When the + /// accumulated SSD-savable bytes exceed this value, a flush to SSD is + /// triggered. Set to 0 to disable this threshold (default). + static constexpr std::string_view kAsyncCacheSsdFlushThresholdBytes{ + "async-cache-ssd-flush-threshold-bytes"}; + /// The interval for persisting in-memory cache to SSD. Setting this config /// to a non-zero value will activate periodic cache persistence. static constexpr std::string_view kAsyncCachePersistenceInterval{ @@ -930,6 +957,8 @@ class SystemConfig : public ConfigBase { bool httpServerEnableGzipCompression() const; + http::HttpServerStartupOptions httpServerStartupOptions() const; + /// A list of ciphers (comma separated) that are supported by /// server and client. Note Java and folly::SSLContext use different names to /// refer to the same cipher. For e.g. TLS_RSA_WITH_AES_256_GCM_SHA384 in Java @@ -1012,6 +1041,10 @@ class SystemConfig : public ConfigBase { std::string spillerFileCreateConfig() const; + std::string spillerAggregationFileCreateConfig() const; + + std::string spillerHashJoinFileCreateConfig() const; + std::string spillerDirectoryCreateConfig() const; folly::Optional spillerSpillPath() const; @@ -1066,6 +1099,10 @@ class SystemConfig : public ConfigBase { int32_t asyncCacheMinSsdSavableBytes() const; + int32_t asyncCacheNumShards() const; + + uint64_t asyncCacheSsdFlushThresholdBytes() const; + std::chrono::duration asyncCachePersistenceInterval() const; bool asyncCacheSsdDisableFileCow() const; @@ -1160,6 +1197,8 @@ class SystemConfig : public ConfigBase { bool httpClientConnectionReuseCounterEnabled() const; + http::HttpClientOptions httpClientOptions() const; + std::chrono::duration exchangeMaxErrorDuration() const; std::chrono::duration exchangeRequestTimeoutMs() const; @@ -1188,6 +1227,8 @@ class SystemConfig : public ConfigBase { int32_t internalCommunicationJwtExpirationSeconds() const; + http::JwtOptions jwtOptions() const; + bool useLegacyArrayAgg() const; bool cacheVeloxTtlEnabled() const; diff --git a/presto-native-execution/presto_cpp/main/common/Counters.cpp b/presto-native-execution/presto_cpp/main/common/Counters.cpp index da56b7c9df91c..2704568c28c19 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.cpp +++ b/presto-native-execution/presto_cpp/main/common/Counters.cpp @@ -112,6 +112,8 @@ void registerPrestoMetrics() { DEFINE_METRIC( kCounterPartitionedOutputBufferGetDataLatencyMs, facebook::velox::StatType::AVG); + DEFINE_METRIC( + kCounterWorkerRuntimeUptimeSecs, facebook::velox::StatType::AVG); DEFINE_METRIC(kCounterOsUserCpuTimeMicros, facebook::velox::StatType::AVG); DEFINE_METRIC(kCounterOsSystemCpuTimeMicros, facebook::velox::StatType::AVG); DEFINE_METRIC(kCounterOsNumSoftPageFaults, facebook::velox::StatType::AVG); diff --git a/presto-native-execution/presto_cpp/main/common/Counters.h b/presto-native-execution/presto_cpp/main/common/Counters.h index 52ecc1356e06b..c6379c575f011 100644 --- a/presto-native-execution/presto_cpp/main/common/Counters.h +++ b/presto-native-execution/presto_cpp/main/common/Counters.h @@ -157,6 +157,9 @@ constexpr std::string_view kCounterTotalPartitionedOutputBuffer{ /// OutputBufferManager. constexpr std::string_view kCounterPartitionedOutputBufferGetDataLatencyMs{ "presto_cpp.partitioned_output_buffer_get_data_latency_ms"}; +/// Worker runtime uptime in seconds after the worker process started. +constexpr std::string_view kCounterWorkerRuntimeUptimeSecs{ + "presto_cpp.worker_runtime_uptime_secs"}; /// ================== OS Counters ================= diff --git a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp index 76096dad1145a..8eca5a8de78ff 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp @@ -230,6 +230,31 @@ TEST_F(ConfigTest, optionalSystemConfigsWithDefault) { ASSERT_EQ(config.maxDriversPerTask(), 1024); } +TEST_F(ConfigTest, asyncCacheNumShards) { + SystemConfig config; + init(config, {}); + // Test default value is 4 + ASSERT_EQ(config.asyncCacheNumShards(), 4); + + // Test custom value + init(config, {{std::string(SystemConfig::kAsyncCacheNumShards), "8"}}); + ASSERT_EQ(config.asyncCacheNumShards(), 8); +} + +TEST_F(ConfigTest, asyncCacheSsdFlushThresholdBytes) { + SystemConfig config; + init(config, {}); + // Test default value is 0 + ASSERT_EQ(config.asyncCacheSsdFlushThresholdBytes(), 0); + + // Test custom value + init( + config, + {{std::string(SystemConfig::kAsyncCacheSsdFlushThresholdBytes), + "134217728"}}); + ASSERT_EQ(config.asyncCacheSsdFlushThresholdBytes(), 134217728); +} + TEST_F(ConfigTest, remoteFunctionServer) { SystemConfig config; init(config, {}); diff --git a/presto-native-execution/presto_cpp/main/connectors/HivePrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/HivePrestoToVeloxConnector.cpp index e464614703fcf..cbb4f5c7d40b7 100644 --- a/presto-native-execution/presto_cpp/main/connectors/HivePrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/HivePrestoToVeloxConnector.cpp @@ -380,7 +380,6 @@ HivePrestoToVeloxConnector::toVeloxTableHandle( return toHiveTableHandle( hiveLayout->domainPredicate, hiveLayout->remainingPredicate, - hiveLayout->pushdownFilterEnabled, tableName, hiveLayout->dataColumns, tableHandle, diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp index b6ad329f0e5ae..a03586008da7e 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.cpp @@ -47,7 +47,6 @@ velox::dwio::common::FileFormat toVeloxFileFormat( std::unique_ptr toIcebergTableHandle( const protocol::TupleDomain& domainPredicate, const std::shared_ptr& remainingPredicate, - bool isPushdownFilterEnabled, const std::string& tableName, const protocol::List& dataColumns, const protocol::TableHandle& tableHandle, @@ -100,7 +99,6 @@ std::unique_ptr toIcebergTableHandle( return std::make_unique( tableHandle.connectorId, tableName, - isPushdownFilterEnabled, std::move(subfieldFilters), remainingFilter, finalDataColumns, @@ -150,6 +148,22 @@ toVeloxIcebergPartitionSpec( spec.specId, fields); } +velox::parquet::ParquetFieldId toParquetField( + const protocol::iceberg::ColumnIdentity& column) { + std::vector children; + if (!column.children.empty()) { + children.reserve(column.children.size()); + for (const auto& child : column.children) { + children.push_back(toParquetField(child)); + } + } + // ParquetFieldId does not declare a constructor that takes fieldId and + // children, so we use aggregate initialization to make it work for compilers + // that don't create the necessary constructors by default (e.g clang-15). + velox::parquet::ParquetFieldId pf{.fieldId = column.id, .children = children}; + return pf; +} + } // namespace std::unique_ptr @@ -234,13 +248,13 @@ IcebergPrestoToVeloxConnector::toVeloxColumnHandle( columnParseParameters.partitionDateValueFormat = velox::connector::hive:: HiveColumnHandle::ColumnParseParameters::kDaysSinceEpoch; } - return std::make_unique( + + return std::make_unique( icebergColumn->columnIdentity.name, toHiveColumnType(icebergColumn->columnType), type, - type, - toRequiredSubfields(icebergColumn->requiredSubfields), - columnParseParameters); + toParquetField(icebergColumn->columnIdentity), + toRequiredSubfields(icebergColumn->requiredSubfields)); } std::unique_ptr @@ -296,7 +310,6 @@ IcebergPrestoToVeloxConnector::toVeloxTableHandle( return toIcebergTableHandle( icebergLayout->domainPredicate, icebergLayout->remainingPredicate, - icebergLayout->pushdownFilterEnabled, tableName, icebergLayout->dataColumns, tableHandle, @@ -324,7 +337,7 @@ IcebergPrestoToVeloxConnector::toVeloxInsertTableHandle( createHandle->handle.connectorHandle->_type); const auto inputColumns = - toHiveColumns(icebergOutputTableHandle->inputColumns, typeParser); + toIcebergColumns(icebergOutputTableHandle->inputColumns, typeParser); return std::make_unique< velox::connector::hive::iceberg::IcebergInsertTableHandle>( @@ -354,7 +367,7 @@ IcebergPrestoToVeloxConnector::toVeloxInsertTableHandle( insertHandle->handle.connectorHandle->_type); const auto inputColumns = - toHiveColumns(icebergInsertTableHandle->inputColumns, typeParser); + toIcebergColumns(icebergInsertTableHandle->inputColumns, typeParser); return std::make_unique< velox::connector::hive::iceberg::IcebergInsertTableHandle>( @@ -370,18 +383,20 @@ IcebergPrestoToVeloxConnector::toVeloxInsertTableHandle( toFileCompressionKind(icebergInsertTableHandle->compressionCodec))); } -std::vector -IcebergPrestoToVeloxConnector::toHiveColumns( +std::vector +IcebergPrestoToVeloxConnector::toIcebergColumns( const protocol::List& inputColumns, const TypeParser& typeParser) const { - std::vector hiveColumns; - hiveColumns.reserve(inputColumns.size()); + std::vector + icebergColumns; + icebergColumns.reserve(inputColumns.size()); for (const auto& columnHandle : inputColumns) { - hiveColumns.emplace_back( - std::dynamic_pointer_cast( + icebergColumns.emplace_back( + std::dynamic_pointer_cast< + velox::connector::hive::iceberg::IcebergColumnHandle>( std::shared_ptr(toVeloxColumnHandle(&columnHandle, typeParser)))); } - return hiveColumns; + return icebergColumns; } } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h index c9336ba6c9bc4..7d9c3cac00fbb 100644 --- a/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/IcebergPrestoToVeloxConnector.h @@ -17,6 +17,8 @@ #include "presto_cpp/main/connectors/PrestoToVeloxConnector.h" #include "presto_cpp/presto_protocol/connector/iceberg/presto_protocol_iceberg.h" +#include "velox/connectors/hive/iceberg/IcebergColumnHandle.h" + namespace facebook::presto { class IcebergPrestoToVeloxConnector final : public PrestoToVeloxConnector { @@ -52,7 +54,8 @@ class IcebergPrestoToVeloxConnector final : public PrestoToVeloxConnector { const TypeParser& typeParser) const final; private: - std::vector toHiveColumns( + std::vector + toIcebergColumns( const protocol::List& inputColumns, const TypeParser& typeParser) const; diff --git a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.cpp b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.cpp index f95b7c9f27a13..0cf8624456f35 100644 --- a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.cpp @@ -16,6 +16,7 @@ #include #include "presto_cpp/main/types/TypeParser.h" +#include "velox/common/base/Exceptions.h" #include "velox/connectors/hive/TableHandle.h" #include "velox/type/fbhive/HiveTypeParser.h" @@ -389,7 +390,8 @@ std::unique_ptr combineIntegerRanges( if (bigintFilters.size() == 2 && bigintFilters[0]->lower() == std::numeric_limits::min() && bigintFilters[1]->upper() == std::numeric_limits::max()) { - assert(bigintFilters[0]->upper() + 1 <= bigintFilters[1]->lower() - 1); + VELOX_CHECK_LE( + bigintFilters[0]->upper() + 1, bigintFilters[1]->lower() - 1); return std::make_unique( bigintFilters[0]->upper() + 1, bigintFilters[1]->lower() - 1, @@ -398,7 +400,7 @@ std::unique_ptr combineIntegerRanges( bool allNegatedValues = true; bool foundMaximum = false; - assert(bigintFilters.size() > 1); // true by size checks on ranges + VELOX_CHECK_GT(bigintFilters.size(), 1); std::vector rejectedValues; // check if int64 min is a rejected value @@ -746,7 +748,6 @@ connector::hive::HiveColumnHandle::ColumnType toHiveColumnType( std::unique_ptr toHiveTableHandle( const protocol::TupleDomain& domainPredicate, const std::shared_ptr& remainingPredicate, - bool isPushdownFilterEnabled, const std::string& tableName, const protocol::List& dataColumns, const protocol::TableHandle& tableHandle, @@ -799,7 +800,6 @@ std::unique_ptr toHiveTableHandle( return std::make_unique( tableHandle.connectorId, tableName, - isPushdownFilterEnabled, std::move(subfieldFilters), remainingFilter, finalDataColumns, diff --git a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h index aec1f169ffad1..15e31c0a0f7eb 100644 --- a/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h +++ b/presto-native-execution/presto_cpp/main/connectors/PrestoToVeloxConnectorUtils.h @@ -53,7 +53,6 @@ velox::connector::hive::HiveColumnHandle::ColumnType toHiveColumnType( std::unique_ptr toHiveTableHandle( const protocol::TupleDomain& domainPredicate, const std::shared_ptr& remainingPredicate, - bool isPushdownFilterEnabled, const std::string& tableName, const protocol::List& dataColumns, const protocol::TableHandle& tableHandle, diff --git a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp index 9251c70f1a257..50b1546235016 100644 --- a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.cpp @@ -29,43 +29,39 @@ static const std::string kTasksTable = "tasks"; } // namespace const velox::RowTypePtr SystemTableHandle::taskSchema() const { - static std::vector kTaskColumnNames = { - "node_id", - "task_id", - "stage_execution_id", - "stage_id", - "query_id", - "state", - "splits", - "queued_splits", - "running_splits", - "completed_splits", - "split_scheduled_time_ms", - "split_cpu_time_ms", - "split_blocked_time_ms", - "raw_input_bytes", - "raw_input_rows", - "processed_input_bytes", - "processed_input_rows", - "output_bytes", - "output_rows", - "physical_written_bytes", - "created", - "start", - "last_heartbeat", - "end"}; - - static std::vector kTaskColumnTypes = { - velox::VARCHAR(), velox::VARCHAR(), velox::VARCHAR(), - velox::VARCHAR(), velox::VARCHAR(), velox::VARCHAR(), - velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), - velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), - velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), - velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), - velox::BIGINT(), velox::BIGINT(), velox::TIMESTAMP(), - velox::TIMESTAMP(), velox::TIMESTAMP(), velox::TIMESTAMP()}; static const RowTypePtr kTaskSchema = - ROW(std::move(kTaskColumnNames), std::move(kTaskColumnTypes)); + ROW({"node_id", + "task_id", + "stage_execution_id", + "stage_id", + "query_id", + "state", + "splits", + "queued_splits", + "running_splits", + "completed_splits", + "split_scheduled_time_ms", + "split_cpu_time_ms", + "split_blocked_time_ms", + "raw_input_bytes", + "raw_input_rows", + "processed_input_bytes", + "processed_input_rows", + "output_bytes", + "output_rows", + "physical_written_bytes", + "created", + "start", + "last_heartbeat", + "end"}, + {velox::VARCHAR(), velox::VARCHAR(), velox::VARCHAR(), + velox::VARCHAR(), velox::VARCHAR(), velox::VARCHAR(), + velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), + velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), + velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), + velox::BIGINT(), velox::BIGINT(), velox::BIGINT(), + velox::BIGINT(), velox::BIGINT(), velox::TIMESTAMP(), + velox::TIMESTAMP(), velox::TIMESTAMP(), velox::TIMESTAMP()}); return kTaskSchema; } @@ -140,24 +136,24 @@ void SystemDataSource::addSplit( VELOX_CHECK(currentSplit_, "Wrong type of split for SystemDataSource."); } -#define SET_TASK_COLUMN(value) \ - int j = 0; \ - for (const auto& taskEntry : taskMap) { \ - auto task = taskEntry.second; \ - auto taskInfo = taskInfos[j]; \ - flat->set(j, value); \ - j++; \ +#define SET_TASK_COLUMN(value) \ + int j = 0; \ + for (const auto& taskEntry : taskMap) { \ + [[maybe_unused]] const auto& task = taskEntry.second; \ + [[maybe_unused]] const auto& taskInfo = taskInfos[j]; \ + flat->set(j, value); \ + j++; \ } -#define SET_TASK_FMT_COLUMN(value) \ - int j = 0; \ - std::string temp; \ - for (const auto& taskEntry : taskMap) { \ - auto task = taskEntry.second; \ - auto taskInfo = taskInfos[j]; \ - temp = fmt::format("{}", value); \ - flat->set(j, StringView(temp)); \ - j++; \ +#define SET_TASK_FMT_COLUMN(value) \ + int j = 0; \ + std::string temp; \ + for (const auto& taskEntry : taskMap) { \ + [[maybe_unused]] const auto& task = taskEntry.second; \ + [[maybe_unused]] const auto& taskInfo = taskInfos[j]; \ + temp = fmt::format("{}", value); \ + flat->set(j, StringView(temp)); \ + j++; \ } RowVectorPtr SystemDataSource::getTaskResults() { @@ -326,11 +322,17 @@ RowVectorPtr SystemDataSource::getTaskResults() { SET_TASK_COLUMN(velox::Timestamp::fromMillis(task->lastEndTimeMs)); break; } + + default: + VELOX_UNREACHABLE(); } } return result; } +#undef SET_TASK_COLUMN +#undef SET_TASK_FMT_COLUMN + std::optional SystemDataSource::next( uint64_t size, velox::ContinueFuture& /*future*/) { diff --git a/presto-native-execution/presto_cpp/main/connectors/SystemSplit.h b/presto-native-execution/presto_cpp/main/connectors/SystemSplit.h index fab7a00422704..49096fe47fa3c 100644 --- a/presto-native-execution/presto_cpp/main/connectors/SystemSplit.h +++ b/presto-native-execution/presto_cpp/main/connectors/SystemSplit.h @@ -27,11 +27,11 @@ struct SystemSplit : public velox::connector::ConnectorSplit { schemaName_(schemaName), tableName_(tableName) {} - const std::string& schemaName() { + const std::string& schemaName() const { return schemaName_; } - const std::string& tableName() { + const std::string& tableName() const { return tableName_; } diff --git a/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt index 20020ea182e5d..7ac8ebcd7b1aa 100644 --- a/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt @@ -14,6 +14,7 @@ add_library(presto_function_metadata OBJECT FunctionMetadata.cpp) target_link_libraries(presto_function_metadata presto_common velox_function_registry) add_subdirectory(dynamic_registry) +add_subdirectory(theta_sketch) if(PRESTO_ENABLE_REMOTE_FUNCTIONS) add_subdirectory(remote) diff --git a/presto-native-execution/presto_cpp/main/functions/remote/client/RestRemoteClient.cpp b/presto-native-execution/presto_cpp/main/functions/remote/client/RestRemoteClient.cpp index b39483fb20d4b..68e2a651eff48 100644 --- a/presto-native-execution/presto_cpp/main/functions/remote/client/RestRemoteClient.cpp +++ b/presto-native-execution/presto_cpp/main/functions/remote/client/RestRemoteClient.cpp @@ -17,6 +17,7 @@ #include #include +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/functions/remote/utils/ContentTypes.h" #include "velox/common/base/Exceptions.h" #include "velox/common/memory/Memory.h" @@ -39,6 +40,8 @@ RestRemoteClient::RestRemoteClient(const std::string& url) : url_(url) { folly::SocketAddress addr(uri.host().c_str(), uri.port(), true); evbThread_ = std::make_unique("rest-client"); + auto systemConfig = SystemConfig::instance(); + auto httpClientOptions = systemConfig->httpClientOptions(); httpClient_ = std::make_shared( evbThread_->getEventBase(), nullptr, @@ -47,7 +50,8 @@ RestRemoteClient::RestRemoteClient(const std::string& url) : url_(url) { requestTimeoutMs, connectTimeoutMs, memPool_, - nullptr); + nullptr, + std::move(httpClientOptions)); } RestRemoteClient::~RestRemoteClient() { diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/theta_sketch/CMakeLists.txt new file mode 100644 index 0000000000000..905f6ae7de67d --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/CMakeLists.txt @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(presto_theta_sketch_functions ThetaSketchAggregate.cpp ThetaSketchFunctions.cpp) + +target_link_libraries(presto_theta_sketch_functions velox_common_base Folly::folly) + +if(PRESTO_ENABLE_TESTING) + add_subdirectory(tests) +endif() diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchAggregate.cpp b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchAggregate.cpp new file mode 100644 index 0000000000000..14d6a00c59846 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchAggregate.cpp @@ -0,0 +1,246 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h" + +#include "DataSketches/theta_sketch.hpp" +#include "DataSketches/theta_union.hpp" + +#include "velox/exec/Aggregate.h" +#include "velox/exec/SimpleAggregateAdapter.h" +#include "velox/functions/prestosql/aggregates/AggregateNames.h" +#include "velox/type/HugeInt.h" + +namespace facebook::presto::functions::aggregate { + +namespace { + +const char* const kThetaSketch = "sketch_theta"; + +template +class ThetaSketchAggregate { + public: + // Type(s) of input vector(s) wrapped in Row. + using InputType = velox::Row; + + // Type of intermediate result + using IntermediateType = velox::Varbinary; + + // Type of output vector. + using OutputType = velox::Varbinary; + + static constexpr bool default_null_behavior_ = false; + + static bool toIntermediate( + velox::exec::out_type& out, + velox::exec::optional_arg_type in) { + if (in.has_value()) { + auto updateSketch = datasketches::update_theta_sketch::builder().build(); + if constexpr (std::is_same_v) { + updateSketch.update(std::to_string(in.value())); + } else if constexpr ( + std::is_same_v || + std::is_same_v) { + const auto& strView = in.value(); + updateSketch.update(std::string(strView.data(), strView.size())); + } else { + updateSketch.update(in.value()); + } + datasketches::theta_union thetaUnion = + datasketches::theta_union::builder().build(); + thetaUnion.update(updateSketch); + auto compactSketch = thetaUnion.get_result(); + out.resize(compactSketch.get_serialized_size_bytes()); + auto serializedBytes = compactSketch.serialize(); + std::memcpy(out.data(), serializedBytes.data(), out.size()); + } + return true; + } + + struct AccumulatorType { + datasketches::theta_union thetaUnion = + datasketches::theta_union::builder().build(); + datasketches::update_theta_sketch updateSketch = + datasketches::update_theta_sketch::builder().build(); + + AccumulatorType() = delete; + + // Constructor used in initializeNewGroups(). + explicit AccumulatorType( + velox::HashStringAllocator* /*allocator*/, + ThetaSketchAggregate* /*fn*/) {} + + void updateUnion() { + thetaUnion.update(updateSketch); + updateSketch.reset(); + } + + // addInput expects one parameter of exec::arg_type for each child-type T + // wrapped in InputType. + bool addInput( + velox::HashStringAllocator* /*allocator*/, + velox::exec::optional_arg_type data) { + if (data.has_value()) { + if constexpr (std::is_same_v) { + updateSketch.update(std::to_string(data.value())); + } else if constexpr ( + std::is_same_v || + std::is_same_v) { + const auto& strView = data.value(); + updateSketch.update(std::string(strView.data(), strView.size())); + } else { + updateSketch.update(data.value()); + } + } + return true; + } + + // combine expects one parameter of exec::arg_type. + bool combine( + velox::HashStringAllocator* /*allocator*/, + velox::exec::optional_arg_type other) { + if (other.has_value()) { + updateUnion(); + auto compactSketch = datasketches::wrapped_compact_theta_sketch::wrap( + other->data(), other->size()); + thetaUnion.update(compactSketch); + } + return true; + } + + bool getResult(velox::exec::out_type& out) { + updateUnion(); + auto compactSketch = thetaUnion.get_result(); + out.resize(compactSketch.get_serialized_size_bytes()); + auto serializedBytes = compactSketch.serialize(); + std::memcpy(out.data(), serializedBytes.data(), out.size()); + return true; + } + + bool writeFinalResult( + bool nonNullGroup, + velox::exec::out_type& out) { + return getResult(out); + } + + bool writeIntermediateResult( + bool nonNullGroup, + velox::exec::out_type& out) { + return getResult(out); + } + }; +}; + +} // namespace + +velox::exec::AggregateRegistrationResult registerThetaSketchAggregate( + const std::string& prefix, + bool withCompanionFunctions, + bool overwrite) { + std::vector> + signatures; + std::string returnType = "varbinary"; + std::string intermediateType = "varbinary"; + + for (const auto& inputType : + {"tinyint", + "smallint", + "integer", + "bigint", + "hugeint", + "real", + "double", + "varchar", + "date", + "timestamp"}) { + signatures.push_back( + velox::exec::AggregateFunctionSignatureBuilder() + .returnType(returnType) + .intermediateType(intermediateType) + .argumentType(inputType) + .build()); + } + signatures.push_back( + velox::exec::AggregateFunctionSignatureBuilder() + .integerVariable("a_precision") + .integerVariable("a_scale") + .returnType(returnType) + .intermediateType(intermediateType) + .argumentType("DECIMAL(a_precision, a_scale)") + .build()); + + auto name = prefix + kThetaSketch; + + return velox::exec::registerAggregateFunction( + name, + std::move(signatures), + [name]( + velox::core::AggregationNode::Step step, + const std::vector& argTypes, + const velox::TypePtr& resultType, + const velox::core::QueryConfig& /*config*/) + -> std::unique_ptr { + VELOX_CHECK_LE( + argTypes.size(), 1, "{} takes at most one argument", name); + auto inputType = argTypes[0]; + if (velox::exec::isRawInput(step)) { + switch (inputType->kind()) { + case velox::TypeKind::TINYINT: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::SMALLINT: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::INTEGER: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::BIGINT: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::HUGEINT: + return std::make_unique>>( + step, argTypes, resultType); + case velox::TypeKind::REAL: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::DOUBLE: + return std::make_unique>>(step, argTypes, resultType); + case velox::TypeKind::VARCHAR: + return std::make_unique>>( + step, argTypes, resultType); + case velox::TypeKind::TIMESTAMP: + return std::make_unique::NativeType>>>( + step, argTypes, resultType); + default: + VELOX_FAIL( + "Unknown input type for {} aggregation {}", + name, + inputType->kindName()); + } + } else { + return std::make_unique>>( + step, argTypes, resultType); + } + }, + withCompanionFunctions, + overwrite); +} + +} // namespace facebook::presto::functions::aggregate diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchFunctions.cpp b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchFunctions.cpp new file mode 100644 index 0000000000000..dab48b74dfff6 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchFunctions.cpp @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h" + +#include "DataSketches/theta_sketch.hpp" + +#include "velox/velox/functions/Macros.h" +#include "velox/velox/functions/Registerer.h" + +namespace facebook::presto::functions { + +namespace { + +template +struct ThetaSketchEstimateFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + FOLLY_ALWAYS_INLINE void call( + out_type& result, + const arg_type& in) { + auto compactSketch = + datasketches::wrapped_compact_theta_sketch::wrap(in.data(), in.size()); + result = compactSketch.get_estimate(); + } +}; + +template +struct ThetaSketchSummaryFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + FOLLY_ALWAYS_INLINE void call( + out_type>& result, + const arg_type& in) { + auto compactSketch = + datasketches::wrapped_compact_theta_sketch::wrap(in.data(), in.size()); + result.copy_from( + std::make_tuple( + compactSketch.get_estimate(), + compactSketch.get_theta(), + compactSketch.get_upper_bound(1), + compactSketch.get_lower_bound(1), + compactSketch.get_num_retained())); + } +}; +} // namespace + +void registerThetaSketchFunctions(const std::string& prefix) { + velox:: + registerFunction( + {prefix + "sketch_theta_estimate"}); + velox::registerFunction< + ThetaSketchSummaryFunction, + velox::Row, + velox::Varbinary>({prefix + "sketch_theta_summary"}); +} +} // namespace facebook::presto::functions diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h new file mode 100644 index 0000000000000..be706b5a24b79 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/exec/Aggregate.h" + +namespace facebook::presto::functions::aggregate { + +velox::exec::AggregateRegistrationResult registerThetaSketchAggregate( + const std::string& prefix, + bool withCompanionFunctions = true, + bool overwrite = false); +} // namespace facebook::presto::functions::aggregate + +namespace facebook::presto::functions { + +void registerThetaSketchFunctions(const std::string& prefix = ""); +} + +namespace facebook::presto::functions::aggregate::theta_sketch { +namespace { +void registerAllThetaSketchFunctions(const std::string& prefix = "") { + facebook::presto::functions::aggregate::registerThetaSketchAggregate(prefix); + facebook::presto::functions::registerThetaSketchFunctions(prefix); +} +} // namespace +} // namespace facebook::presto::functions::aggregate::theta_sketch diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/CMakeLists.txt new file mode 100644 index 0000000000000..c9ea6d535d7a1 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(presto_theta_sketch_functions_test ThetaSketchAggregationTest.cpp) + +add_test( + NAME presto_theta_sketch_functions_test + COMMAND presto_theta_sketch_functions_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries( + presto_theta_sketch_functions_test + presto_theta_sketch_functions + velox_exec + velox_exec_test_lib + velox_functions_aggregates_test_lib + GTest::gtest + GTest::gtest_main +) diff --git a/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/ThetaSketchAggregationTest.cpp b/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/ThetaSketchAggregationTest.cpp new file mode 100644 index 0000000000000..3ef1b843d2e16 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/theta_sketch/tests/ThetaSketchAggregationTest.cpp @@ -0,0 +1,353 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DataSketches/theta_sketch.hpp" +#include "DataSketches/theta_union.hpp" + +#include "presto_cpp/main/functions/theta_sketch/ThetaSketchRegistration.h" +#include "velox/common/hyperloglog/HllUtils.h" +#include "velox/exec/PlanNodeStats.h" +#include "velox/exec/tests/utils/PlanBuilder.h" +#include "velox/functions/lib/aggregates/tests/utils/AggregationTestBase.h" + +using namespace facebook::velox; +using namespace facebook::velox::exec; +using namespace facebook::velox::exec::test; +using namespace facebook::velox::functions::aggregate::test; +using namespace datasketches; + +namespace facebook::presto::functions::aggregate::test { +namespace { +class ThetaSketchAggregationTest : public AggregationTestBase { + protected: + static const std::vector kFruits; + static const std::vector kVegetables; + + void SetUp() override { + folly::SingletonVault::singleton()->registrationComplete(); + AggregationTestBase::SetUp(); + presto::functions::aggregate::theta_sketch::registerAllThetaSketchFunctions( + ""); + } + + template + void testGlobalAgg(const VectorPtr& values) { + auto vectors = makeRowVector({values}); + auto expected = makeRowVector({makeFlatVector( + {getExpectedResult(values)}, VARBINARY())}); + + testAggregations({vectors}, {}, {"sketch_theta(c0)"}, {expected}); + } + + template + const std::string getExpectedResult(const VectorPtr& values) { + auto updateSketch = update_theta_sketch::builder().build(); + FlatVector* flatVector = values->asFlatVector(); + for (auto i = 0; i < flatVector->size(); i++) { + if (!flatVector->isNullAt(i)) { + if constexpr ( + std::is_same_v || std::is_same_v) { + const auto& strView = flatVector->valueAt(i); + updateSketch.update(std::string(strView.data(), strView.size())); + } else if constexpr (std::is_same_v) { + updateSketch.update(std::to_string(flatVector->valueAt(i))); + } else { + updateSketch.update(flatVector->valueAt(i)); + } + } + } + auto thetaUnion = theta_union::builder().build(); + thetaUnion.update(updateSketch); + + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + thetaUnion.get_result().serialize(s); + return s.str(); + } + + template + const RowVectorPtr getExpectedResultForGroupBy( + const VectorPtr& keys, + const VectorPtr& values) { + VELOX_CHECK_EQ(keys->size(), values->size()); + struct thetaUnionStruct { + theta_union thetaUnion = theta_union::builder().build(); + update_theta_sketch updateSketch = update_theta_sketch::builder().build(); + bool hasNull = false; + }; + + std::unordered_map groupedTheta; + FlatVector* keysVector = keys->asFlatVector(); + FlatVector* valuesVector = values->asFlatVector(); + + for (auto i = 0; i < keysVector->size(); ++i) { + auto key = keysVector->valueAt(i); + if (!valuesVector->isNullAt(i)) { + auto value = valuesVector->valueAt(i); + if constexpr ( + std::is_same_v || std::is_same_v) { + groupedTheta[key].updateSketch.update( + std::string(value.data(), value.size())); + } else if constexpr (std::is_same_v) { + groupedTheta[key].updateSketch.update(std::to_string(value)); + } else { + groupedTheta[key].updateSketch.update(value); + } + } else { + groupedTheta[keysVector->valueAt(i)].hasNull = true; + } + } + + std::unordered_map results; + + for (auto& iter : groupedTheta) { + groupedTheta[iter.first].thetaUnion.update( + groupedTheta[iter.first].updateSketch); + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + groupedTheta[iter.first].thetaUnion.get_result().serialize(s); + results[iter.first] = s.str(); + } + + return toRowVector(results); + } + + template + RowVectorPtr toRowVector(const std::unordered_map& data) { + std::vector keys(data.size()); + transform(data.begin(), data.end(), keys.begin(), [](auto pair) { + return pair.first; + }); + + std::vector values(data.size()); + transform(data.begin(), data.end(), values.begin(), [](auto pair) { + return pair.second; + }); + + return makeRowVector( + {makeFlatVector(keys), makeFlatVector(values, VARBINARY())}); + } + + template + void testGroupByAgg(const VectorPtr& keys, const VectorPtr& values) { + auto vectors = makeRowVector({keys, values}); + auto expectedResults = getExpectedResultForGroupBy(keys, values); + + testAggregations( + {vectors}, {"c0"}, {"sketch_theta(c1)"}, {expectedResults}); + } + + template + void runNumericTest(int64_t minValue) { + vector_size_t size = 50'000; + auto keys = makeFlatVector( + size, [&minValue](auto row) { return (minValue + row) % 2; }); + auto values = makeFlatVector( + size, [&minValue](auto row) { return minValue + row; }); + testGroupByAgg(keys, values); + testGlobalAgg(values); + } +}; + +TEST_F(ThetaSketchAggregationTest, numericTest) { + runNumericTest(0); + runNumericTest(std::numeric_limits::max()); + runNumericTest(std::numeric_limits::max()); + runNumericTest(std::numeric_limits::max()); + runNumericTest(std::numeric_limits::max()); +} + +const std::vector ThetaSketchAggregationTest::kFruits = { + "apple", + "banana", + "cherry", + "dragonfruit", + "grapefruit", + "melon", + "orange", + "pear", + "pineapple", + "unknown fruit with a very long name", + "watermelon"}; + +const std::vector ThetaSketchAggregationTest::kVegetables = { + "cucumber", + "tomato", + "potato", + "squash", + "unknown vegetable with a very long name"}; + +TEST_F(ThetaSketchAggregationTest, varcharTest) { + vector_size_t size = 50'000; + + auto keys = makeFlatVector(size, [](auto row) { return row % 2; }); + auto values = makeFlatVector(size, [&](auto row) { + return StringView( + row % 2 == 0 ? kFruits[row % kFruits.size()] + : kVegetables[row % kVegetables.size()]); + }); + + testGroupByAgg(keys, values); + testGlobalAgg(values); +} + +TEST_F(ThetaSketchAggregationTest, floatingPointTest) { + vector_size_t size = 50'000; + auto keys = makeFlatVector(size, [](auto row) { return row % 2; }); + + { + auto values = makeFlatVector(size, [](auto row) { + return static_cast(rand()) / + (static_cast(RAND_MAX / 50000)); + }); + testGroupByAgg(keys, values); + testGlobalAgg(values); + } + + { + auto values = makeFlatVector(50000, [](auto row) { + return static_cast(rand()) / + (static_cast(RAND_MAX / 50000)); + }); + testGroupByAgg(keys, values); + testGlobalAgg(values); + } +} + +TEST_F(ThetaSketchAggregationTest, timestampTest) { + vector_size_t size = 50'000; + auto keys = makeFlatVector(size, [](auto row) { return row % 2; }); + auto values = makeFlatVector( + size, [](auto row) { return Timestamp(row, row); }); + testGroupByAgg(keys, values); + testGlobalAgg(values); +} + +TEST_F(ThetaSketchAggregationTest, allNullsTest) { + vector_size_t size = 5000; + auto keys = makeFlatVector(size, [](auto row) { return row % 2; }); + auto values = + makeFlatVector(size, [](auto row) { return row; }, nullEvery(1)); + testGroupByAgg(keys, values); + testGlobalAgg(values); +} + +TEST_F(ThetaSketchAggregationTest, mixedNullsTest) { + vector_size_t size = 5000; + auto keys = makeFlatVector(size, [](auto row) { return row % 2; }); + auto values = + makeFlatVector(size, [](auto row) { return row; }, nullEvery(2)); + testGroupByAgg(keys, values); + testGlobalAgg(values); +} + +TEST_F(ThetaSketchAggregationTest, streaming) { + auto rawInput1 = makeFlatVector({1, 2, 3}); + auto rawInput2 = makeFlatVector(1000, folly::identity); + auto combinedInput = makeFlatVector({1, 2, 3}); + combinedInput->append(rawInput2->wrappedVector()); + auto result = testStreaming("sketch_theta", true, {rawInput1}, {rawInput2}); + auto expectedResult = getExpectedResult(combinedInput); + ASSERT_EQ(result->size(), 1); + ASSERT_EQ(result->asFlatVector()->valueAt(0), expectedResult); + + result = testStreaming("sketch_theta", false, {rawInput1}, {rawInput2}); + ASSERT_EQ(result->size(), 1); + ASSERT_EQ(result->asFlatVector()->valueAt(0), expectedResult); +} + +TEST_F(ThetaSketchAggregationTest, testSketchThetaEstimate) { + auto sketch_theta_estimate = [this](auto input) { + auto op = PlanBuilder() + .values({makeRowVector({input})}) + .singleAggregation({}, {"sketch_theta(c0)"}) + .project({"sketch_theta_estimate(a0)"}) + .planNode(); + + return readSingleValue(op); + }; + + // Empty sketch + auto input = makeFlatVector({}); + ASSERT_EQ(sketch_theta_estimate(input).value(), 0.0); + + // Single value sketch + input = makeFlatVector(1); + ASSERT_EQ(sketch_theta_estimate(input).value(), 1.0); + + // Many value sketch + input = makeFlatVector(100, [](auto row) { return row; }); + update_theta_sketch updateSketch = update_theta_sketch::builder().build(); + for (auto i = 0; i < input->size(); ++i) { + updateSketch.update(input->valueAt(i)); + } + theta_union thetaUnion = theta_union::builder().build(); + thetaUnion.update(updateSketch); + + ASSERT_EQ( + sketch_theta_estimate(input).value(), + thetaUnion.get_result().get_estimate()); +} + +void assertSummaryMatches( + compact_theta_sketch compactSketch, + variant sketchSummary) { + auto row = + sketchSummary.value>(); + ASSERT_EQ(row.at(0).value(), compactSketch.get_estimate()); + ASSERT_EQ(row.at(1).value(), compactSketch.get_theta()); + ASSERT_EQ( + row.at(2).value(), compactSketch.get_upper_bound(1)); + ASSERT_EQ( + row.at(3).value(), compactSketch.get_lower_bound(1)); + ASSERT_EQ( + row.at(4).value(), compactSketch.get_num_retained()); +} + +TEST_F(ThetaSketchAggregationTest, testSketchThetaSummary) { + auto sketch_theta_summary = [this](auto input) { + auto op = PlanBuilder() + .values({makeRowVector({input})}) + .singleAggregation({}, {"sketch_theta(c0)"}) + .project({"sketch_theta_summary(a0)"}) + .planNode(); + + return readSingleValue(op); + }; + + // Empty sketch + auto input = makeFlatVector({}); + update_theta_sketch updateSketch = update_theta_sketch::builder().build(); + theta_union thetaUnion = theta_union::builder().build(); + thetaUnion.update(updateSketch); + assertSummaryMatches(thetaUnion.get_result(), sketch_theta_summary(input)); + + // Single value sketch + input = makeFlatVector(1); + updateSketch = update_theta_sketch::builder().build(); + updateSketch.update(1); + thetaUnion = theta_union::builder().build(); + thetaUnion.update(updateSketch); + assertSummaryMatches(thetaUnion.get_result(), sketch_theta_summary(input)); + + // Many value sketch + input = makeFlatVector(100, [](auto row) { return row; }); + updateSketch = update_theta_sketch::builder().build(); + for (auto i = 0; i < input->size(); ++i) { + updateSketch.update(input->valueAt(i)); + } + thetaUnion = theta_union::builder().build(); + thetaUnion.update(updateSketch); + assertSummaryMatches(thetaUnion.get_result(), sketch_theta_summary(input)); +} +} // namespace +} // namespace facebook::presto::functions::aggregate::test diff --git a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp index 0b5c559803a8f..ce547538fc353 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp +++ b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp @@ -20,7 +20,6 @@ #include #include #include -#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/common/Counters.h" #include "presto_cpp/main/common/Utils.h" #include "presto_cpp/main/http/HttpClient.h" @@ -37,6 +36,7 @@ HttpClient::HttpClient( std::chrono::milliseconds connectTimeout, std::shared_ptr pool, folly::SSLContextPtr sslContext, + HttpClientOptions options, std::function&& reportOnBodyStatsFunc) : eventBase_(eventBase), connPool_(connPool), @@ -44,20 +44,10 @@ HttpClient::HttpClient( address_(address), transactionTimeout_(transactionTimeout), connectTimeout_(connectTimeout), - http2Enabled_(SystemConfig::instance()->httpClientHttp2Enabled()), - maxConcurrentStreams_( - SystemConfig::instance()->httpClientHttp2MaxStreamsPerConnection()), - http2InitialStreamWindow_( - SystemConfig::instance()->httpClientHttp2InitialStreamWindow()), - http2StreamWindow_( - SystemConfig::instance()->httpClientHttp2StreamWindow()), - http2SessionWindow_( - SystemConfig::instance()->httpClientHttp2SessionWindow()), + options_(std::move(options)), pool_(std::move(pool)), - sslContext_(sslContext), - reportOnBodyStatsFunc_(std::move(reportOnBodyStatsFunc)), - maxResponseAllocBytes_(SystemConfig::instance()->httpMaxAllocateBytes()) { -} + sslContext_(std::move(sslContext)), + reportOnBodyStatsFunc_(std::move(reportOnBodyStatsFunc)) {} HttpClient::~HttpClient() { if (sessionPoolHolder_) { @@ -149,7 +139,7 @@ std::unique_ptr HttpResponse::consumeBody( for (auto& iobuf : bodyChain_) { const auto length = iobuf->length(); ::memcpy(curr, iobuf->data(), length); - curr = (char*)curr + length; + curr = static_cast(curr) + length; iobuf.reset(); } bodyChain_.clear(); @@ -207,7 +197,7 @@ class ResponseHandler : public proxygen::HTTPTransactionHandler { folly::SemiFuture> initialize( std::shared_ptr self) { - self_ = self; + self_ = std::move(self); return promise_.getSemiFuture(); } @@ -218,7 +208,7 @@ class ResponseHandler : public proxygen::HTTPTransactionHandler { // - seqNo == 0: First request on this connection // - seqNo > 0: Connection is being reused for subsequent requests // Reuse rate = connection_reuse / (connection_first_use + connection_reuse) - if (SystemConfig::instance()->httpClientConnectionReuseCounterEnabled()) { + if (client_->connectionReuseCounterEnabled()) { const uint32_t seqNo = txn_->getSequenceNumber(); if (seqNo > 0) { RECORD_METRIC_VALUE(kCounterHttpClientConnectionReuse); @@ -342,7 +332,7 @@ class ConnectionHandler : public proxygen::HTTPConnector::Callback { folly::SSLContextPtr sslContext) : responseHandler_(responseHandler), sessionPool_(sessionPool), - transactionTimer_(transactionTimeout), + transactionTimer_(std::move(transactionTimeout)), connectTimeout_(connectTimeout), http2Enabled_(http2Enabled), maxConcurrentStreams_(maxConcurrentStreams), @@ -567,11 +557,11 @@ void HttpClient::sendRequest(std::shared_ptr responseHandler) { sessionPool_, proxygen::WheelTimerInstance(transactionTimeout_, eventBase_), connectTimeout_, - http2Enabled_, - maxConcurrentStreams_, - http2InitialStreamWindow_, - http2StreamWindow_, - http2SessionWindow_, + options_.http2Enabled, + options_.http2MaxStreamsPerConnection, + options_.http2InitialStreamWindow, + options_.http2StreamWindow, + options_.http2SessionWindow, eventBase_, address_, sslContext_); @@ -592,7 +582,7 @@ folly::SemiFuture> HttpClient::sendRequest( request.ensureHostHeader(); auto responseHandler = std::make_shared( request, - maxResponseAllocBytes_, + options_.maxAllocateBytes, body, reportOnBodyStatsFunc_, shared_from_this()); @@ -618,26 +608,20 @@ folly::SemiFuture> HttpClient::sendRequest( void RequestBuilder::addJwtIfConfigured() { #ifdef PRESTO_ENABLE_JWT - if (SystemConfig::instance()->internalCommunicationJwtEnabled()) { + if (jwtOptions_.jwtEnabled) { // If JWT was enabled the secret cannot be empty. auto secretHash = std::vector(SHA256_DIGEST_LENGTH); folly::ssl::OpenSSLHash::sha256( folly::range(secretHash), - folly::ByteRange( - folly::StringPiece( - SystemConfig::instance() - ->internalCommunicationSharedSecret()))); + folly::ByteRange(folly::StringPiece(jwtOptions_.sharedSecret))); const auto time = std::chrono::system_clock::now(); const auto token = jwt::create() - .set_subject(NodeConfig::instance()->nodeId()) + .set_subject(jwtOptions_.nodeId) .set_issued_at(time) .set_expires_at( - time + - std::chrono::seconds{ - SystemConfig::instance() - ->internalCommunicationJwtExpirationSeconds()}) + time + std::chrono::seconds{jwtOptions_.jwtExpirationSeconds}) .sign( jwt::algorithm::hs256{std::string( reinterpret_cast(secretHash.data()), diff --git a/presto-native-execution/presto_cpp/main/http/HttpClient.h b/presto-native-execution/presto_cpp/main/http/HttpClient.h index c6dc84ad0d86a..76ad101f2e465 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpClient.h +++ b/presto-native-execution/presto_cpp/main/http/HttpClient.h @@ -18,7 +18,9 @@ #include #include #include +#include "presto_cpp/main/http/HttpClientOptions.h" // @manual #include "presto_cpp/main/http/HttpConstants.h" +#include "presto_cpp/main/http/JwtOptions.h" // @manual #include "velox/common/base/Exceptions.h" namespace facebook::presto::http { @@ -151,10 +153,6 @@ class HttpClientConnectionPool { class ResponseHandler; -// HttpClient uses proxygen::SessionPool which must be destructed on the -// EventBase thread. Hence, the destructor of HttpClient must run on the -// EventBase thread as well. Consider running HttpClient's destructor -// via EventBase::runOnDestruction. class HttpClient : public std::enable_shared_from_this { public: HttpClient( @@ -166,6 +164,7 @@ class HttpClient : public std::enable_shared_from_this { std::chrono::milliseconds connectTimeout, std::shared_ptr pool, folly::SSLContextPtr sslContext, + HttpClientOptions options = {}, std::function&& reportOnBodyStatsFunc = nullptr); ~HttpClient(); @@ -180,6 +179,10 @@ class HttpClient : public std::enable_shared_from_this { return pool_; } + bool connectionReuseCounterEnabled() const { + return options_.connectionReuseCounterEnabled; + } + static int64_t numConnectionsCreated() { return numConnectionsCreated_; } @@ -200,15 +203,10 @@ class HttpClient : public std::enable_shared_from_this { const folly::SocketAddress address_; const std::chrono::milliseconds transactionTimeout_; const std::chrono::milliseconds connectTimeout_; - const bool http2Enabled_; - const uint32_t maxConcurrentStreams_; - const uint32_t http2InitialStreamWindow_; - const uint32_t http2StreamWindow_; - const uint32_t http2SessionWindow_; + const HttpClientOptions options_; const std::shared_ptr pool_; const folly::SSLContextPtr sslContext_; const std::function reportOnBodyStatsFunc_; - const uint64_t maxResponseAllocBytes_; proxygen::SessionPool* sessionPool_ = nullptr; proxygen::ServerIdleSessionController* idleSessions_ = nullptr; @@ -221,6 +219,11 @@ class RequestBuilder { public: RequestBuilder() {} + RequestBuilder& jwtOptions(JwtOptions options) { + jwtOptions_ = std::move(options); + return *this; + } + RequestBuilder& method(proxygen::HTTPMethod method) { headers_.setMethod(method); return *this; @@ -253,6 +256,7 @@ class RequestBuilder { private: void addJwtIfConfigured(); + JwtOptions jwtOptions_; proxygen::HTTPMessage headers_; }; diff --git a/presto-native-execution/presto_cpp/main/http/HttpClientOptions.h b/presto-native-execution/presto_cpp/main/http/HttpClientOptions.h new file mode 100644 index 0000000000000..4f5b698389057 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/http/HttpClientOptions.h @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +namespace facebook::presto::http { + +struct HttpClientOptions { + // HTTP/2 transport settings (used in constructor) + bool http2Enabled{false}; + uint32_t http2MaxStreamsPerConnection{8}; + uint32_t http2InitialStreamWindow{1 << 23}; + uint32_t http2StreamWindow{1 << 23}; + uint32_t http2SessionWindow{1 << 26}; + uint64_t maxAllocateBytes{65536}; + + // Metrics settings (used in ResponseHandler) + bool connectionReuseCounterEnabled{true}; +}; + +} // namespace facebook::presto::http diff --git a/presto-native-execution/presto_cpp/main/http/HttpServer.cpp b/presto-native-execution/presto_cpp/main/http/HttpServer.cpp index df11c0bdafe8c..a987174722799 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpServer.cpp +++ b/presto-native-execution/presto_cpp/main/http/HttpServer.cpp @@ -14,9 +14,9 @@ #include -#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/common/Utils.h" #include "presto_cpp/main/http/HttpServer.h" +#include "velox/common/base/Exceptions.h" namespace facebook::presto::http { @@ -218,7 +218,7 @@ proxygen::RequestHandler* DispatchingRequestHandlerFactory::onRequest( message->getURL())); } - auto path = message->getPath(); + const auto& path = message->getPath(); // Allocate vector outside of loop to avoid repeated alloc/free. std::vector matches(4); @@ -279,14 +279,13 @@ HttpServer::endpoints() const { } void HttpServer::start( + HttpServerStartupOptions startupOptions, std::vector> filters, std::function onSuccess, std::function onError) { proxygen::HTTPServerOptions options; - auto systemConfig = SystemConfig::instance(); - options.idleTimeout = - std::chrono::milliseconds(systemConfig->httpServerIdleTimeoutMs()); + options.idleTimeout = std::chrono::milliseconds(startupOptions.idleTimeoutMs); proxygen::RequestHandlerChain handlerFactories; @@ -301,30 +300,24 @@ void HttpServer::start( options.handlerFactories = handlerFactories.build(); // HTTP/2 flow control window sizes (configurable) - options.initialReceiveWindow = - systemConfig->httpServerHttp2InitialReceiveWindow(); - options.receiveStreamWindowSize = - systemConfig->httpServerHttp2ReceiveStreamWindowSize(); + options.initialReceiveWindow = startupOptions.http2InitialReceiveWindow; + options.receiveStreamWindowSize = startupOptions.http2ReceiveStreamWindowSize; options.receiveSessionWindowSize = - systemConfig->httpServerHttp2ReceiveSessionWindowSize(); + startupOptions.http2ReceiveSessionWindowSize; options.maxConcurrentIncomingStreams = - systemConfig->httpServerHttp2MaxConcurrentStreams(); + startupOptions.http2MaxConcurrentStreams; options.h2cEnabled = true; // Enable HTTP/2 responses compression for better performance // Supports both gzip and zstd (zstd preferred when client supports it) - options.enableContentCompression = - systemConfig->httpServerEnableContentCompression(); - options.contentCompressionLevel = - systemConfig->httpServerContentCompressionLevel(); + options.enableContentCompression = startupOptions.enableContentCompression; + options.contentCompressionLevel = startupOptions.contentCompressionLevel; options.contentCompressionMinimumSize = - systemConfig->httpServerContentCompressionMinimumSize(); - options.enableZstdCompression = - systemConfig->httpServerEnableZstdCompression(); + startupOptions.contentCompressionMinimumSize; + options.enableZstdCompression = startupOptions.enableZstdCompression; options.zstdContentCompressionLevel = - systemConfig->httpServerZstdContentCompressionLevel(); - options.enableGzipCompression = - systemConfig->httpServerEnableGzipCompression(); + startupOptions.zstdContentCompressionLevel; + options.enableGzipCompression = startupOptions.enableGzipCompression; // CRITICAL: Add Thrift content-types for Presto task updates // By default, proxygen only compresses text/* and some application/* types diff --git a/presto-native-execution/presto_cpp/main/http/HttpServer.h b/presto-native-execution/presto_cpp/main/http/HttpServer.h index b5c89129cf6b5..c8b56dbbb0f2d 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpServer.h +++ b/presto-native-execution/presto_cpp/main/http/HttpServer.h @@ -20,6 +20,7 @@ #include #include "presto_cpp/external/json/nlohmann/json.hpp" #include "presto_cpp/main/http/HttpConstants.h" +#include "presto_cpp/main/http/HttpServerStartupOptions.h" // @manual namespace facebook::presto::http { @@ -63,13 +64,13 @@ class AbstractRequestHandler : public proxygen::RequestHandler { body_.emplace_back(std::move(body)); } - void onUpgrade(proxygen::UpgradeProtocol proto) noexcept override {} + void onUpgrade(proxygen::UpgradeProtocol /*proto*/) noexcept override {} void requestComplete() noexcept override { delete this; } - void onError(proxygen::ProxygenError err) noexcept override { + void onError(proxygen::ProxygenError /*err*/) noexcept override { delete this; } @@ -106,7 +107,7 @@ class CallbackRequestHandlerState { // The function 'fn' will run on the thread that invoked onEOM() void runOnFinalization(std::function callback) { - onFinalizationCallback_ = callback; + onFinalizationCallback_ = std::move(callback); } bool requestExpired() const { @@ -136,10 +137,11 @@ using AsyncRequestHandlerCallback = std::function httpsConfig = nullptr); void start( + HttpServerStartupOptions startupOptions = {}, std::vector> filters = {}, std::function onSuccess = nullptr, diff --git a/presto-native-execution/presto_cpp/main/http/HttpServerStartupOptions.h b/presto-native-execution/presto_cpp/main/http/HttpServerStartupOptions.h new file mode 100644 index 0000000000000..55ae878fbe356 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/http/HttpServerStartupOptions.h @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +namespace facebook::presto::http { + +struct HttpServerStartupOptions { + uint32_t idleTimeoutMs{60'000}; + uint32_t http2InitialReceiveWindow{1 << 20}; + uint32_t http2ReceiveStreamWindowSize{1 << 20}; + uint32_t http2ReceiveSessionWindowSize{10 * (1 << 20)}; + uint32_t http2MaxConcurrentStreams{100}; + bool enableContentCompression{false}; + uint32_t contentCompressionLevel{4}; + uint32_t contentCompressionMinimumSize{3584}; + bool enableZstdCompression{false}; + uint32_t zstdContentCompressionLevel{8}; + bool enableGzipCompression{false}; +}; + +} // namespace facebook::presto::http diff --git a/presto-native-execution/presto_cpp/main/http/JwtOptions.h b/presto-native-execution/presto_cpp/main/http/JwtOptions.h new file mode 100644 index 0000000000000..0eb349c12c891 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/http/JwtOptions.h @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +namespace facebook::presto::http { + +struct JwtOptions { + bool jwtEnabled{false}; + std::string sharedSecret; + int32_t jwtExpirationSeconds{300}; + std::string nodeId; +}; + +} // namespace facebook::presto::http diff --git a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp index bea48739e0af1..460e158d754ec 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp +++ b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.cpp @@ -103,7 +103,7 @@ void InternalAuthenticationFilter::onError( delete this; } -void InternalAuthenticationFilter::sendGenericErrorResponse(void) { +void InternalAuthenticationFilter::sendGenericErrorResponse() { /// Indicate to upstream an error occurred and make sure /// no further forwarding occurs. upstream_->onError(proxygen::kErrorUnsupportedExpectation); @@ -154,11 +154,11 @@ void InternalAuthenticationFilter::processAndVerifyJwt( } // Passed the verification, move the message along. Filter::onRequest(std::move(msg)); - } catch (const jwt::error::token_verification_exception& e) { + } catch (const jwt::error::token_verification_exception&) { sendUnauthorizedResponse(); - } catch (const jwt::error::signature_verification_exception& e) { + } catch (const jwt::error::signature_verification_exception&) { sendUnauthorizedResponse(); - } catch (const std::system_error& e) { + } catch (const std::system_error&) { sendGenericErrorResponse(); } #endif // PRESTO_ENABLE_JWT diff --git a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.h b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.h index 5002b285589b8..c403cdfa7dcc7 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.h +++ b/presto-native-execution/presto_cpp/main/http/filters/InternalAuthenticationFilter.h @@ -47,9 +47,9 @@ class InternalAuthenticationFilter : public proxygen::Filter { void onError(proxygen::ProxygenError err) noexcept override; private: - void sendGenericErrorResponse(void); + void sendGenericErrorResponse(); - void sendUnauthorizedResponse(void); + void sendUnauthorizedResponse(); void processAndVerifyJwt( const std::string& token, diff --git a/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.h b/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.h index 7696b040b93e8..69b15c5a9f2a7 100644 --- a/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.h +++ b/presto-native-execution/presto_cpp/main/http/filters/StatsFilter.h @@ -46,7 +46,7 @@ class StatsFilterFactory : public proxygen::RequestHandlerFactory { proxygen::RequestHandler* onRequest( proxygen::RequestHandler* handler, - proxygen::HTTPMessage* msg) noexcept override { + proxygen::HTTPMessage* /*msg*/) noexcept override { return new StatsFilter(handler); } }; diff --git a/presto-native-execution/presto_cpp/main/http/tests/HttpJwtTest.cpp b/presto-native-execution/presto_cpp/main/http/tests/HttpJwtTest.cpp index 08d3135b593d0..22a4036eba9ba 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/HttpJwtTest.cpp +++ b/presto-native-execution/presto_cpp/main/http/tests/HttpJwtTest.cpp @@ -104,8 +104,10 @@ class HttpJwtTestSuite : public ::testing::TestWithParam { auto [reqPromise, reqFuture] = folly::makePromiseContract(); request->requestPromise = std::move(reqPromise); + auto jwtOpts = systemConfig->jwtOptions(); + auto responseFuture = - sendGet(client.get(), "/async/msg", sendDelayMs, "TestBody"); + sendGet(client.get(), "/async/msg", sendDelayMs, "TestBody", jwtOpts); auto serverConfig = jwtSystemConfig(serverSystemConfigOverride); auto valuesMap = serverConfig->rawConfigsCopy(); @@ -133,7 +135,7 @@ class HttpJwtTestSuite : public ::testing::TestWithParam { TEST_P(HttpJwtTestSuite, basicJwtTest) { const bool useHttps = GetParam(); - auto response = std::move(produceHttpResponse(useHttps)); + auto response = produceHttpResponse(useHttps); EXPECT_EQ(response->headers()->getStatusCode(), http::kHttpOk); } @@ -145,8 +147,7 @@ TEST_P(HttpJwtTestSuite, jwtSecretMismatch) { const bool useHttps = GetParam(); - auto response = - std::move(produceHttpResponse(useHttps, {}, serverConfigOverride)); + auto response = produceHttpResponse(useHttps, {}, serverConfigOverride); EXPECT_EQ(response->headers()->getStatusCode(), http::kHttpUnauthorized); } @@ -162,8 +163,8 @@ TEST_P(HttpJwtTestSuite, jwtExpiredToken) { const bool useHttps = GetParam(); - auto response = std::move( - produceHttpResponse(useHttps, clientConfigOverride, {}, kSendDelay)); + auto response = + produceHttpResponse(useHttps, clientConfigOverride, {}, kSendDelay); EXPECT_EQ(response->headers()->getStatusCode(), http::kHttpUnauthorized); } @@ -176,8 +177,7 @@ TEST_P(HttpJwtTestSuite, jwtServerVerificationDisabled) { const bool useHttps = GetParam(); - auto response = - std::move(produceHttpResponse(useHttps, {}, serverConfigOverride)); + auto response = produceHttpResponse(useHttps, {}, serverConfigOverride); EXPECT_EQ(response->headers()->getStatusCode(), http::kHttpUnauthorized); } @@ -190,8 +190,7 @@ TEST_P(HttpJwtTestSuite, jwtClientMissingJwt) { const bool useHttps = GetParam(); - auto response = - std::move(produceHttpResponse(useHttps, clientConfigOverride)); + auto response = produceHttpResponse(useHttps, clientConfigOverride); EXPECT_EQ(response->headers()->getStatusCode(), http::kHttpUnauthorized); } diff --git a/presto-native-execution/presto_cpp/main/http/tests/HttpTest.cpp b/presto-native-execution/presto_cpp/main/http/tests/HttpTest.cpp index c72617f01dfc0..3b25551c58b98 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/HttpTest.cpp +++ b/presto-native-execution/presto_cpp/main/http/tests/HttpTest.cpp @@ -160,7 +160,8 @@ TEST_P(HttpTestSuite, clientIdleSessions) { std::chrono::seconds(1), std::chrono::milliseconds(0), memoryPool, - useHttps ? makeSslContext() : nullptr); + useHttps ? makeSslContext() : nullptr, + http::HttpClientOptions{}); auto response = sendGet(client.get(), "/ping").get(std::chrono::seconds(3)); ASSERT_EQ(response->headers()->getStatusCode(), http::kHttpOk); } diff --git a/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h b/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h index f67a2ff0451c5..4f70a2cd9d997 100644 --- a/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h +++ b/presto-native-execution/presto_cpp/main/http/tests/HttpTestBase.h @@ -75,7 +75,7 @@ class HttpServerWrapper { promise_ = std::move(promise); serverThread_ = std::make_unique([this]() { server_->start( - std::move(filters_), [&](proxygen::HTTPServer* httpServer) { + {}, std::move(filters_), [&](proxygen::HTTPServer* httpServer) { ASSERT_EQ(httpServer->addresses().size(), 1); promise_.setValue(httpServer->addresses()[0].address); }); @@ -208,6 +208,7 @@ class HttpClientFactory { connectTimeout, pool, useHttps ? makeSslContext() : nullptr, + facebook::presto::http::HttpClientOptions{}, std::move(reportOnBodyStatsFunc)); } @@ -221,8 +222,10 @@ sendGet( facebook::presto::http::HttpClient* client, const std::string& url, const uint64_t sendDelay = 0, - const std::string body = "") { + const std::string body = "", + facebook::presto::http::JwtOptions jwtOptions = {}) { return facebook::presto::http::RequestBuilder() + .jwtOptions(std::move(jwtOptions)) .method(proxygen::HTTPMethod::GET) .url(url) .send(client, body, sendDelay); diff --git a/presto-native-execution/presto_cpp/main/operators/BroadcastFile.cpp b/presto-native-execution/presto_cpp/main/operators/BroadcastFile.cpp index bf6a7cda80080..d6b3ba45a1e7c 100644 --- a/presto-native-execution/presto_cpp/main/operators/BroadcastFile.cpp +++ b/presto-native-execution/presto_cpp/main/operators/BroadcastFile.cpp @@ -118,7 +118,7 @@ BroadcastFileWriter::BroadcastFileWriter( writeBufferSize, "", std::move(serdeOptions), - getNamedVectorSerde(VectorSerde::Kind::kPresto), + getNamedVectorSerde("Presto"), pool), maxBroadcastBytes_(maxBroadcastBytes) {} diff --git a/presto-native-execution/presto_cpp/main/operators/BroadcastWrite.cpp b/presto-native-execution/presto_cpp/main/operators/BroadcastWrite.cpp index 10a74815ff002..301d91cecb60b 100644 --- a/presto-native-execution/presto_cpp/main/operators/BroadcastWrite.cpp +++ b/presto-native-execution/presto_cpp/main/operators/BroadcastWrite.cpp @@ -62,7 +62,7 @@ class BroadcastWriteOperator : public Operator { getVectorSerdeOptions( common::stringToCompressionKind( ctx->queryConfig().shuffleCompressionKind()), - VectorSerde::Kind::kPresto), + "Presto"), operatorCtx_->pool()); } diff --git a/presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp b/presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp index c3a17902d9780..0212918556e1a 100644 --- a/presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp +++ b/presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp @@ -112,7 +112,7 @@ class LocalShuffleSerializedPage : public ShuffleSerializedPage { velox::BufferPtr buffer) : rows_{std::move(rows)}, buffer_{std::move(buffer)} {} - const std::vector& rows() override { + const std::vector& rows(int32_t /*driverId*/) override { return rows_; } diff --git a/presto-native-execution/presto_cpp/main/operators/ShuffleInterface.h b/presto-native-execution/presto_cpp/main/operators/ShuffleInterface.h index 456dcaf27099f..32e00f72ea013 100644 --- a/presto-native-execution/presto_cpp/main/operators/ShuffleInterface.h +++ b/presto-native-execution/presto_cpp/main/operators/ShuffleInterface.h @@ -63,7 +63,14 @@ class ShuffleSerializedPage : public velox::exec::SerializedPageBase { VELOX_UNSUPPORTED(); } - virtual const std::vector& rows() = 0; + /// Legacy single-consumer path that delegates to rows(0).. + /// retained for backward compatibility. + virtual const std::vector& rows() { + return rows(0); + } + + /// @param driverId Driver ID for per-consumer checksum tracking. + virtual const std::vector& rows(int32_t driverId) = 0; }; class ShuffleReader { diff --git a/presto-native-execution/presto_cpp/main/operators/ShuffleRead.cpp b/presto-native-execution/presto_cpp/main/operators/ShuffleRead.cpp index aa4f2e9df43d1..96f0e9d47a40e 100644 --- a/presto-native-execution/presto_cpp/main/operators/ShuffleRead.cpp +++ b/presto-native-execution/presto_cpp/main/operators/ShuffleRead.cpp @@ -35,7 +35,7 @@ ShuffleRead::ShuffleRead( std::make_shared( shuffleReadNode->id(), shuffleReadNode->outputType(), - VectorSerde::Kind::kCompactRow), + "CompactRow"), exchangeClient, "ShuffleRead") { initStats(); @@ -84,9 +84,10 @@ RowVectorPtr ShuffleRead::getOutput() { numRows += pageRows; } rows_.reserve(numRows); + const int32_t driverId = operatorCtx()->driverCtx()->driverId; for (const auto& page : currentPages_) { auto* batch = checkedPointerCast(page.get()); - const auto& rows = batch->rows(); + const auto& rows = batch->rows(driverId); for (const auto& row : rows) { rows_.emplace_back(row); } diff --git a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp index 7899de068ed21..987900b1dc0a6 100644 --- a/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp +++ b/presto-native-execution/presto_cpp/main/operators/tests/BroadcastTest.cpp @@ -117,9 +117,8 @@ class BroadcastTest : public exec::test::OperatorTestBase, const std::string& basePath, const std::vector& broadcastFilePaths) { // Create plan for read node using file path. - auto readerPlan = exec::test::PlanBuilder() - .exchange(dataType, velox::VectorSerde::Kind::kPresto) - .planNode(); + auto readerPlan = + exec::test::PlanBuilder().exchange(dataType, "Presto").planNode(); exec::CursorParameters broadcastReadParams; broadcastReadParams.planNode = readerPlan; @@ -348,9 +347,8 @@ TEST_P(BroadcastTest, malformedBroadcastInfoJson) { std::string basePath = "/tmp"; std::string invalidBroadcastFilePath = "/tmp/file.bin"; - auto readerPlan = exec::test::PlanBuilder() - .exchange(dataType, velox::VectorSerde::Kind::kPresto) - .planNode(); + auto readerPlan = + exec::test::PlanBuilder().exchange(dataType, "Presto").planNode(); exec::CursorParameters broadcastReadParams; broadcastReadParams.planNode = readerPlan; diff --git a/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt index 459cbc9e623b9..4deeaeb1f9111 100644 --- a/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/tests/CMakeLists.txt @@ -90,7 +90,6 @@ if(PRESTO_ENABLE_REMOTE_FUNCTIONS) presto_server_remote_function_test presto_server_remote_function velox_expression - velox_temp_path GTest::gmock GTest::gtest GTest::gtest_main diff --git a/presto-native-execution/presto_cpp/main/tests/HttpServerWrapper.cpp b/presto-native-execution/presto_cpp/main/tests/HttpServerWrapper.cpp index d1a9c64a29761..48e0d73e5daeb 100644 --- a/presto-native-execution/presto_cpp/main/tests/HttpServerWrapper.cpp +++ b/presto-native-execution/presto_cpp/main/tests/HttpServerWrapper.cpp @@ -21,7 +21,7 @@ folly::SemiFuture HttpServerWrapper::start() { auto [promise, future] = folly::makePromiseContract(); promise_ = std::move(promise); serverThread_ = std::make_unique([this]() { - server_->start({}, [&](proxygen::HTTPServer* httpServer) { + server_->start({}, {}, [&](proxygen::HTTPServer* httpServer) { ASSERT_EQ(httpServer->addresses().size(), 1); promise_.setValue(httpServer->addresses()[0].address); }); diff --git a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp index f37fed70b3389..1d7b770558918 100644 --- a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp @@ -13,6 +13,8 @@ */ #include +#include + #include "presto_cpp/main/PrestoToVeloxQueryConfig.h" #include "presto_cpp/main/SessionProperties.h" #include "presto_cpp/main/common/Configs.h" @@ -92,6 +94,33 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionPropertiesOverrideSystemConfigs) { EXPECT_EQ(expectedValue, config.spillFileCreateConfig()); }}, + {.veloxConfigKey = core::QueryConfig::kAggregationSpillFileCreateConfig, + .sessionPropertyKey = std::make_optional( + SessionProperties::kAggregationSpillFileCreateConfig), + .systemConfigKey = + std::string(SystemConfig::kSpillerAggregationFileCreateConfig), + .sessionValue = "test_agg_config_1", + .differentSessionValue = "test_agg_config_2", + .validator = + [](const core::QueryConfig& config, + const std::string& expectedValue) { + EXPECT_EQ( + expectedValue, config.aggregationSpillFileCreateConfig()); + }}, + + {.veloxConfigKey = core::QueryConfig::kHashJoinSpillFileCreateConfig, + .sessionPropertyKey = std::make_optional( + SessionProperties::kHashJoinSpillFileCreateConfig), + .systemConfigKey = + std::string(SystemConfig::kSpillerHashJoinFileCreateConfig), + .sessionValue = "test_join_config_1", + .differentSessionValue = "test_join_config_2", + .validator = + [](const core::QueryConfig& config, + const std::string& expectedValue) { + EXPECT_EQ(expectedValue, config.hashJoinSpillFileCreateConfig()); + }}, + {.veloxConfigKey = core::QueryConfig::kSpillEnabled, .sessionPropertyKey = std::make_optional(core::QueryConfig::kSpillEnabled), @@ -141,6 +170,18 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionPropertiesOverrideSystemConfigs) { expectedValue == "true", config.aggregationSpillEnabled()); }}, + {.veloxConfigKey = core::QueryConfig::kMaxSpillBytes, + .sessionPropertyKey = + std::make_optional(SessionProperties::kMaxSpillBytes), + .systemConfigKey = std::string(SystemConfig::kMaxSpillBytes), + .sessionValue = "214748364800", + .differentSessionValue = "107374182400", + .validator = + [](const core::QueryConfig& config, + const std::string& expectedValue) noexcept { + EXPECT_EQ(std::stoull(expectedValue), config.maxSpillBytes()); + }}, + {.veloxConfigKey = core::QueryConfig::kRequestDataSizesMaxWaitSec, .sessionPropertyKey = std::make_optional( SessionProperties::kRequestDataSizesMaxWaitSec), @@ -280,7 +321,7 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionPropertiesOverrideSystemConfigs) { // CRITICAL: This count MUST match the exact number of entries in // veloxToPrestoConfigMapping If this assertion fails, it means a new // mapping was added and this test needs to be updated - const size_t kExpectedMappingCount = 14; + const size_t kExpectedMappingCount = 17; EXPECT_EQ(kExpectedMappingCount, testCases.size()); // Test each mapping to ensure session properties override system configs @@ -651,6 +692,12 @@ TEST_F(PrestoToVeloxQueryConfigTest, systemConfigsWithoutSessionOverride) { .systemConfigKey = std::string(SystemConfig::kQueryMaxMemoryPerNode)}, {.veloxConfigKey = core::QueryConfig::kSpillFileCreateConfig, .systemConfigKey = std::string(SystemConfig::kSpillerFileCreateConfig)}, + {.veloxConfigKey = core::QueryConfig::kAggregationSpillFileCreateConfig, + .systemConfigKey = + std::string(SystemConfig::kSpillerAggregationFileCreateConfig)}, + {.veloxConfigKey = core::QueryConfig::kHashJoinSpillFileCreateConfig, + .systemConfigKey = + std::string(SystemConfig::kSpillerHashJoinFileCreateConfig)}, {.veloxConfigKey = core::QueryConfig::kSpillEnabled, .systemConfigKey = std::string(SystemConfig::kSpillEnabled)}, {.veloxConfigKey = core::QueryConfig::kJoinSpillEnabled, @@ -659,6 +706,8 @@ TEST_F(PrestoToVeloxQueryConfigTest, systemConfigsWithoutSessionOverride) { .systemConfigKey = std::string(SystemConfig::kOrderBySpillEnabled)}, {.veloxConfigKey = core::QueryConfig::kAggregationSpillEnabled, .systemConfigKey = std::string(SystemConfig::kAggregationSpillEnabled)}, + {.veloxConfigKey = core::QueryConfig::kMaxSpillBytes, + .systemConfigKey = std::string(SystemConfig::kMaxSpillBytes)}, {.veloxConfigKey = core::QueryConfig::kRequestDataSizesMaxWaitSec, .systemConfigKey = std::string(SystemConfig::kRequestDataSizesMaxWaitSec)}, @@ -700,7 +749,7 @@ TEST_F(PrestoToVeloxQueryConfigTest, systemConfigsWithoutSessionOverride) { std::string(SystemConfig::kExchangeLazyFetchingEnabled)}, }; - const size_t kExpectedSystemConfigMappingCount = 20; + const size_t kExpectedSystemConfigMappingCount = 23; EXPECT_EQ(kExpectedSystemConfigMappingCount, expectedMappings.size()) << "Update expectedMappings to match veloxToPrestoConfigMapping"; diff --git a/presto-native-execution/presto_cpp/main/tests/RemoteFunctionRegistererTest.cpp b/presto-native-execution/presto_cpp/main/tests/RemoteFunctionRegistererTest.cpp index 7788cc22c38e1..c93973b394c94 100644 --- a/presto-native-execution/presto_cpp/main/tests/RemoteFunctionRegistererTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/RemoteFunctionRegistererTest.cpp @@ -16,8 +16,8 @@ #include #include #include "velox/common/base/Fs.h" -#include "velox/exec/tests/utils/TempDirectoryPath.h" -#include "velox/exec/tests/utils/TempFilePath.h" +#include "velox/common/testutil/TempDirectoryPath.h" +#include "velox/common/testutil/TempFilePath.h" #include "velox/expression/VectorFunction.h" using namespace facebook::velox; @@ -56,7 +56,7 @@ TEST_F(RemoteFunctionRegistererTest, singleFile) { })"; // Write to a single output file. - auto path = exec::test::TempFilePath::create(); + auto path = common::testutil::TempFilePath::create(); writeToFile(path->getPath(), json); // Check functions do not exist first. @@ -86,7 +86,7 @@ TEST_F(RemoteFunctionRegistererTest, prefixes) { })"; // Write to a single output file. - auto path = exec::test::TempFilePath::create(); + auto path = common::testutil::TempFilePath::create(); writeToFile(path->getPath(), json); EXPECT_TRUE(exec::getVectorFunctionSignatures("mock3") == std::nullopt); @@ -124,7 +124,7 @@ std::string getJson(const std::string& functionName) { } TEST_F(RemoteFunctionRegistererTest, directory) { - auto tempDir = exec::test::TempDirectoryPath::create(); + auto tempDir = common::testutil::TempDirectoryPath::create(); // Create the following structure: // diff --git a/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp b/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp index 5d64e66cf6c52..cc1b3b5a26f49 100644 --- a/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp @@ -15,7 +15,9 @@ #include #include #include "presto_cpp/main/PrestoServerOperations.h" +#include "presto_cpp/main/common/tests/MutableConfigs.h" #include "velox/common/base/tests/GTestUtils.h" +#include "velox/common/file/FileSystems.h" #include "velox/common/memory/Memory.h" #include "velox/connectors/hive/HiveConnector.h" #include "velox/exec/tests/utils/OperatorTestBase.h" @@ -122,6 +124,16 @@ TEST_F(ServerOperationTest, buildServerOp) { EXPECT_EQ(ServerOperation::Target::kSystemConfig, op.target); EXPECT_EQ(ServerOperation::Action::kSetProperty, op.action); + op = buildServerOpFromHttpMsgPath( + "/v1/operation/veloxQueryConfig/setProperty"); + EXPECT_EQ(ServerOperation::Target::kVeloxQueryConfig, op.target); + EXPECT_EQ(ServerOperation::Action::kSetProperty, op.action); + + op = buildServerOpFromHttpMsgPath( + "/v1/operation/veloxQueryConfig/getProperty"); + EXPECT_EQ(ServerOperation::Target::kVeloxQueryConfig, op.target); + EXPECT_EQ(ServerOperation::Action::kGetProperty, op.action); + op = buildServerOpFromHttpMsgPath("/v1/operation/task/getDetail"); EXPECT_EQ(ServerOperation::Target::kTask, op.target); EXPECT_EQ(ServerOperation::Action::kGetDetail, op.action); @@ -237,4 +249,66 @@ TEST_F(ServerOperationTest, systemConfigEndpoint) { EXPECT_EQ(std::stoi(getPropertyResponse), folly::hardware_concurrency()); } +TEST_F(ServerOperationTest, veloxQueryConfigEndpoint) { + filesystems::registerLocalFileSystem(); + test::setupMutableSystemConfig(); + + PrestoServerOperations serverOperation(nullptr, nullptr); + proxygen::HTTPMessage httpMessage; + + // Getting an unknown property returns "" (not an error). + httpMessage.setQueryParam("name", "nonexistent.property"); + auto getPropertyResponse = serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kGetProperty}, + &httpMessage); + EXPECT_EQ(getPropertyResponse, "\n"); + + // Getting a known system config property returns its value. + httpMessage.setQueryParam("name", "task.max-drivers-per-task"); + getPropertyResponse = serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kGetProperty}, + &httpMessage); + EXPECT_EQ(std::stoi(getPropertyResponse), folly::hardware_concurrency()); + + // Setting a registered property returns a message with "velox query config" + // wording (verifying the copy-paste bug fix from systemConfigOperation). + httpMessage.setQueryParam("name", "shutdown-onset-sec"); + httpMessage.setQueryParam("value", "42"); + auto setPropertyResponse = serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kSetProperty}, + &httpMessage); + EXPECT_NE(setPropertyResponse.find("velox query config"), std::string::npos); + EXPECT_NE(setPropertyResponse.find("shutdown-onset-sec"), std::string::npos); + EXPECT_NE(setPropertyResponse.find("42"), std::string::npos); + + // Verify the property was set by reading it back. + httpMessage.setQueryParam("name", "shutdown-onset-sec"); + getPropertyResponse = serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kGetProperty}, + &httpMessage); + EXPECT_EQ(getPropertyResponse, "42\n"); + + // Missing 'name' parameter should throw. + proxygen::HTTPMessage emptyMessage; + VELOX_ASSERT_THROW( + serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kGetProperty}, + &emptyMessage), + "Missing 'name' parameter"); + + // Missing 'value' parameter for set should throw. + emptyMessage.setQueryParam("name", "some.prop"); + VELOX_ASSERT_THROW( + serverOperation.veloxQueryConfigOperation( + {.target = ServerOperation::Target::kVeloxQueryConfig, + .action = ServerOperation::Action::kSetProperty}, + &emptyMessage), + "Missing 'name' or 'value' parameter"); +} + } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp index 016d61e52ce26..546c79c866597 100644 --- a/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/SessionPropertiesTest.cpp @@ -43,6 +43,10 @@ TEST_F(SessionPropertiesTest, validateMapping) { core::QueryConfig::kSpillWriteBufferSize}, {SessionProperties::kSpillFileCreateConfig, core::QueryConfig::kSpillFileCreateConfig}, + {SessionProperties::kAggregationSpillFileCreateConfig, + core::QueryConfig::kAggregationSpillFileCreateConfig}, + {SessionProperties::kHashJoinSpillFileCreateConfig, + core::QueryConfig::kHashJoinSpillFileCreateConfig}, {SessionProperties::kJoinSpillEnabled, core::QueryConfig::kJoinSpillEnabled}, {SessionProperties::kWindowSpillEnabled, @@ -86,6 +90,8 @@ TEST_F(SessionPropertiesTest, validateMapping) { core::QueryConfig::kMaxOutputBufferSize}, {SessionProperties::kMaxPartitionedOutputBufferSize, core::QueryConfig::kMaxPartitionedOutputBufferSize}, + {SessionProperties::kPartitionedOutputEagerFlush, + core::QueryConfig::kPartitionedOutputEagerFlush}, {SessionProperties::kLegacyTimestamp, core::QueryConfig::kAdjustTimestampToTimezone}, {SessionProperties::kDriverCpuTimeSliceLimitMs, @@ -131,7 +137,11 @@ TEST_F(SessionPropertiesTest, validateMapping) { {SessionProperties::kAggregationCompactionBytesThreshold, core::QueryConfig::kAggregationCompactionBytesThreshold}, {SessionProperties::kAggregationCompactionUnusedMemoryRatio, - core::QueryConfig::kAggregationCompactionUnusedMemoryRatio}}; + core::QueryConfig::kAggregationCompactionUnusedMemoryRatio}, + {SessionProperties::kAggregationMemoryCompactionReclaimEnabled, + core::QueryConfig::kAggregationMemoryCompactionReclaimEnabled}, + {SessionProperties::kMergeJoinOutputBatchStartSize, + core::QueryConfig::kMergeJoinOutputBatchStartSize}}; const auto sessionProperties = SessionProperties::instance(); for (const auto& [sessionProperty, expectedVeloxConfig] : expectedMappings) { diff --git a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp index c4cae5c9eaf05..ef0fc134c724f 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp @@ -112,7 +112,7 @@ class Cursor { TaskManager* taskManager, const protocol::TaskId& taskId, const RowTypePtr& rowType, - velox::VectorSerde::Kind serdeKind, + const std::string& serdeKind, memory::MemoryPool* pool) : pool_(pool), taskManager_(taskManager), @@ -178,7 +178,7 @@ class Cursor { TaskManager* const taskManager_; const protocol::TaskId taskId_; const RowTypePtr rowType_; - const velox::VectorSerde::Kind serdeKind_; + const std::string serdeKind_; bool atEnd_{false}; uint64_t sequence_{0}; }; @@ -190,14 +190,10 @@ void setAggregationSpillConfig( } class TaskManagerTest : public exec::test::OperatorTestBase, - public testing::WithParamInterface { + public testing::WithParamInterface { public: - static std::vector getTestParams() { - const std::vector kinds( - {VectorSerde::Kind::kPresto, - VectorSerde::Kind::kCompactRow, - VectorSerde::Kind::kUnsafeRow}); - return kinds; + static std::vector getTestParams() { + return {"Presto", "CompactRow", "UnsafeRow"}; } static void SetUpTestCase() { diff --git a/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp index 3fbbea4c4f4e3..6550684e57642 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskStatusTest.cpp @@ -37,7 +37,8 @@ TEST_F(TaskStatusTest, errorCode) { "code": 1234, "name": "name", "type": "INTERNAL_ERROR", - "retriable": false + "retriable": false, + "catchableByTry": false })"; json j = json::parse(str); @@ -65,7 +66,8 @@ TEST_F(TaskStatusTest, executionFailureInfoOptionalFieldsEmpty) { "code": 1234, "name": "name", "type": "INTERNAL_ERROR", - "retriable": false + "retriable": false, + "catchableByTry": false }, "remoteHost": "localhost:8080", "errorCause": "EXCEEDS_BROADCAST_MEMORY_LIMIT" diff --git a/presto-native-execution/presto_cpp/main/tests/data/ExecutionFailureInfo.json b/presto-native-execution/presto_cpp/main/tests/data/ExecutionFailureInfo.json index 3e2f60ec1da12..c1663e6d2bfed 100644 --- a/presto-native-execution/presto_cpp/main/tests/data/ExecutionFailureInfo.json +++ b/presto-native-execution/presto_cpp/main/tests/data/ExecutionFailureInfo.json @@ -14,7 +14,8 @@ "code": 1234, "name": "name", "type": "INSUFFICIENT_RESOURCES", - "retriable": true + "retriable": true, + "catchableByTry": false }, "remoteHost": "localhost:8080", "errorCause": "UNKNOWN" @@ -32,7 +33,8 @@ "code": 1234, "name": "name", "type": "EXTERNAL", - "retriable": true + "retriable": true, + "catchableByTry": false }, "remoteHost": "localhost:8080", "errorCause": "LOW_PARTITION_COUNT" @@ -50,7 +52,8 @@ "code": 1234, "name": "name", "type": "INTERNAL_ERROR", - "retriable": true + "retriable": true, + "catchableByTry": false }, "remoteHost": "localhost:8080", "errorCause": "EXCEEDS_BROADCAST_MEMORY_LIMIT" @@ -64,7 +67,8 @@ "code": 1234, "name": "name", "type": "INTERNAL_ERROR", - "retriable": false + "retriable": false, + "catchableByTry": false }, "remoteHost": "localhost:8080", "errorCause": "EXCEEDS_BROADCAST_MEMORY_LIMIT" diff --git a/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt b/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt index 9deb725d94d06..a7aba85cdbd6e 100644 --- a/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/thrift/CMakeLists.txt @@ -16,8 +16,30 @@ find_library(THRIFT_CORE thrift-core) find_library(THRIFT_PROTOCOL thriftprotocol) find_library(THRIFT_METADATA thriftmetadata) find_library(THRIFT_TRANSPORT transport) +find_library(THRIFT_ASYNC async) +find_library(THRIFT_RPC_METADATA rpcmetadata) +find_library(THRIFT_TYPEREP thrifttyperep) + find_path(THRIFT_INCLUDES thrift/lib/cpp2/gen/module_data_h.h PATH_SUFFIXES include REQUIRED) +# In the CI /opt/homebrew/include is not included and it leads to missing +# event.h. +if(APPLE AND EXISTS "/opt/homebrew") + include_directories(SYSTEM /opt/homebrew/include) +endif() + +set( + presto_thrift_library_dependencies + ${THRIFT_PROTOCOL} + ${THRIFT_METADATA} + ${THRIFT_CORE} + ${THRIFT_TYPEREP} + ${THRIFT_ASYNC} + ${THRIFT_TRANSPORT} + ${THRIFT_RPC_METADATA} + ${FOLLY_WITH_DEPENDENCIES} +) + include(ThriftLibrary.cmake) thrift_library( @@ -28,14 +50,10 @@ thrift_library( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/presto_cpp/main/thrift ".." + THRIFT_INCLUDE_DIRECTORIES + ${THRIFT_INCLUDES} ) -target_link_libraries( - presto_thrift-cpp2 - ${THRIFT_PROTOCOL} - ${THRIFT_METADATA} - ${THRIFT_CORE} - ${THRIFT_TRANSPORT} -) +target_link_libraries(presto_thrift-cpp2 ${presto_thrift_library_dependencies}) set(presto_thrift_INCLUDES ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(presto_thrift-cpp2 PUBLIC ${presto_thrift_INCLUDES} ${GLOG_INCLUDE_DIR}) target_include_directories(presto_thrift-cpp2-obj PUBLIC ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR}) @@ -49,13 +67,7 @@ thrift_library( ${CMAKE_CURRENT_BINARY_DIR}/presto_cpp/main/thrift ".." ) -target_link_libraries( - presto_native-cpp2 - ${THRIFT_PROTOCOL} - ${THRIFT_METADATA} - ${THRIFT_CORE} - ${THRIFT_TRANSPORT} -) +target_link_libraries(presto_native-cpp2 ${presto_thrift_library_dependencies}) set(presto_native_INCLUDES ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(presto_native-cpp2 PUBLIC ${presto_native_INCLUDES} ${GLOG_INCLUDE_DIR}) target_include_directories(presto_native-cpp2-obj PUBLIC ${THRIFT_INCLUDES} ${GLOG_INCLUDE_DIR}) diff --git a/presto-native-execution/presto_cpp/main/thrift/ThriftLibrary.cmake b/presto-native-execution/presto_cpp/main/thrift/ThriftLibrary.cmake index e63a7556d2cf6..fb547388c8b49 100644 --- a/presto-native-execution/presto_cpp/main/thrift/ThriftLibrary.cmake +++ b/presto-native-execution/presto_cpp/main/thrift/ThriftLibrary.cmake @@ -1,4 +1,4 @@ -# Copyright (c) Facebook, Inc. and its affiliates. +# Copyright (c) Meta Platforms, Inc. and affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -117,7 +117,7 @@ endmacro() # #include_prefix # ) # add_library(somelib ...) -# target_link_libraries(somelibe ${file_name}-${language} ...) +# target_link_libraries(somelib ${file_name}-${language} ...) # macro(thrift_library @@ -167,15 +167,21 @@ endmacro() # # thrift_generate # This is used to codegen thrift files using the thrift compiler +# Supports library names that differ from the file name (to handle two libraries +# with the same filename on disk (in different folders)) # Params: -# @file_name - The name of tge thrift file +# @file_name - Input file name. Will be used for naming the CMake +# target if TARGET_NAME_BASE is not specified. # @services - A list of services that are declared in the thrift file # @language - The generator to use (cpp, cpp2 or py3) # @options - Extra options to pass to the generator # @output_path - The directory where the thrift file lives +# @include_prefix - Prefix to use for thrift includes in generated sources +# @TARGET_NAME_BASE (optional) - name used for target instead of real filename +# @THRIFT_INCLUDE_DIRECTORIES (optional) path to thrift include directories # # Output: -# file-language-target - A custom target to add a dependenct +# file-language-target - A custom target to add a dependency # ${file-language-HEADERS} - The generated Header Files. # ${file-language-SOURCES} - The generated Source Files. # @@ -186,7 +192,6 @@ endmacro() # bypass_source_check(${file_language-SOURCES}) # This will prevent cmake from complaining about missing source files # - macro(thrift_generate file_name services @@ -198,43 +203,59 @@ macro(thrift_generate ) cmake_parse_arguments(THRIFT_GENERATE # Prefix "" # Options - "" # One Value args + "TARGET_NAME_BASE" # One Value args "THRIFT_INCLUDE_DIRECTORIES" # Multi-value args "${ARGN}") + set(source_file_name ${file_name}) + set(target_file_name ${file_name}) set(thrift_include_directories) foreach(dir ${THRIFT_GENERATE_THRIFT_INCLUDE_DIRECTORIES}) list(APPEND thrift_include_directories "-I" "${dir}") endforeach() + if(DEFINED THRIFT_GENERATE_TARGET_NAME_BASE + AND NOT THRIFT_GENERATE_TARGET_NAME_BASE STREQUAL "") + set(target_file_name ${THRIFT_GENERATE_TARGET_NAME_BASE}) + endif() - set("${file_name}-${language}-HEADERS" - ${output_path}/gen-${language}/${file_name}_constants.h - ${output_path}/gen-${language}/${file_name}_data.h - ${output_path}/gen-${language}/${file_name}_metadata.h - ${output_path}/gen-${language}/${file_name}_types.h - ${output_path}/gen-${language}/${file_name}_types.tcc + set("${target_file_name}-${language}-HEADERS" + ${output_path}/gen-${language}/${source_file_name}_constants.h + ${output_path}/gen-${language}/${source_file_name}_data.h + ${output_path}/gen-${language}/${source_file_name}_metadata.h + ${output_path}/gen-${language}/${source_file_name}_types.h + ${output_path}/gen-${language}/${source_file_name}_types.tcch + ${output_path}/gen-${language}/${source_file_name}_types_custom_protocol.h ) - set("${file_name}-${language}-SOURCES" - ${output_path}/gen-${language}/${file_name}_constants.cpp - ${output_path}/gen-${language}/${file_name}_data.cpp - ${output_path}/gen-${language}/${file_name}_types.cpp + set("${target_file_name}-${language}-SOURCES" + ${output_path}/gen-${language}/${source_file_name}_constants.cpp + ${output_path}/gen-${language}/${source_file_name}_data.cpp + ${output_path}/gen-${language}/${source_file_name}_types.cpp + ${output_path}/gen-${language}/${source_file_name}_types_binary.cpp + ${output_path}/gen-${language}/${source_file_name}_types_compact.cpp + ${output_path}/gen-${language}/${source_file_name}_types_serialization.cpp ) + if("${options}" MATCHES "layouts") + set("${target_file_name}-${language}-SOURCES" + ${${target_file_name}-${language}-SOURCES} + ${output_path}/gen-${language}/${source_file_name}_layouts.cpp + ) + endif() if(NOT "${options}" MATCHES "no_metadata") - set("${file_name}-${language}-SOURCES" - ${${file_name}-${language}-SOURCES} - ${output_path}/gen-${language}/${file_name}_metadata.cpp + set("${target_file_name}-${language}-SOURCES" + ${${target_file_name}-${language}-SOURCES} + ${output_path}/gen-${language}/${source_file_name}_metadata.cpp ) endif() foreach(service ${services}) - set("${file_name}-${language}-HEADERS" - ${${file_name}-${language}-HEADERS} + set("${target_file_name}-${language}-HEADERS" + ${${source_file_name}-${language}-HEADERS} ${output_path}/gen-${language}/${service}.h ${output_path}/gen-${language}/${service}.tcc ${output_path}/gen-${language}/${service}AsyncClient.h ${output_path}/gen-${language}/${service}_custom_protocol.h ) - set("${file_name}-${language}-SOURCES" - ${${file_name}-${language}-SOURCES} + set("${target_file_name}-${language}-SOURCES" + ${${source_file_name}-${language}-SOURCES} ${output_path}/gen-${language}/${service}.cpp ${output_path}/gen-${language}/${service}AsyncClient.cpp ) @@ -252,26 +273,27 @@ macro(thrift_generate set(gen_language "mstch_cpp2") elseif("${language}" STREQUAL "py3") set(gen_language "mstch_py3") - file(WRITE "${output_path}/gen-${language}/${file_name}/__init__.py") + file(WRITE "${output_path}/gen-${language}/${source_file_name}/__init__.py") endif() + message(STATUS "Thrift command:") + message(STATUS "${THRIFT1} -v --gen \"${gen_language}:${options}${include_prefix_text}\" -o ${output_path} ${thrift_include_directories} \"${file_path}/${source_file_name}.thrift\"") add_custom_command( - OUTPUT ${${file_name}-${language}-HEADERS} - ${${file_name}-${language}-SOURCES} - COMMAND mkdir -p ${output_path} + OUTPUT ${${target_file_name}-${language}-HEADERS} + ${${target_file_name}-${language}-SOURCES} COMMAND ${THRIFT1} --gen "${gen_language}:${options}${include_prefix_text}" -o ${output_path} ${thrift_include_directories} - "${file_path}/${file_name}.thrift" + "${file_path}/${source_file_name}.thrift" DEPENDS ${THRIFT1} - "${file_path}/${file_name}.thrift" - COMMENT "Generating ${file_name} files. Output: ${output_path}" + "${file_path}/${source_file_name}.thrift" + COMMENT "Generating ${target_file_name} files. Output: ${output_path}. Command: ${THRIFT1} -v --gen \"${gen_language}:${options}${include_prefix_text}\" -o ${output_path} ${thrift_include_directories} \"${file_path}/${source_file_name}.thrift\"" ) add_custom_target( - ${file_name}-${language}-target ALL + ${target_file_name}-${language}-target ALL DEPENDS ${${language}-${language}-HEADERS} - ${${file_name}-${language}-SOURCES} + ${${target_file_name}-${language}-SOURCES} ) install( DIRECTORY gen-${language} diff --git a/presto-native-execution/presto_cpp/main/thrift/presto_thrift.thrift b/presto-native-execution/presto_cpp/main/thrift/presto_thrift.thrift index 035e40d5b2823..a8b1b5178344d 100644 --- a/presto-native-execution/presto_cpp/main/thrift/presto_thrift.thrift +++ b/presto-native-execution/presto_cpp/main/thrift/presto_thrift.thrift @@ -12,6 +12,9 @@ * limitations under the License. */ +include "thrift/annotation/cpp.thrift" +include "thrift/annotation/thrift.thrift" + namespace cpp2 facebook.presto.thrift enum TaskState { @@ -579,10 +582,9 @@ struct UpdateHandle { struct ExecutionFailureInfo { 1: string type; 2: string message; - 3: optional ExecutionFailureInfo cause ( - cpp.ref_type = "shared", - drift.recursive_reference = true, - ); + @cpp.Ref{type = cpp.RefType.SharedMutable} + @thrift.DeprecatedUnvalidatedAnnotations{items = {"drift.recursive_reference": "true"}} + 3: optional ExecutionFailureInfo cause; 4: list suppressed; 5: list stack; 6: ErrorLocation errorLocation; diff --git a/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift b/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift index 035e40d5b2823..a8b1b5178344d 100644 --- a/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift +++ b/presto-native-execution/presto_cpp/main/thrift/temp_presto_thrift.thrift @@ -12,6 +12,9 @@ * limitations under the License. */ +include "thrift/annotation/cpp.thrift" +include "thrift/annotation/thrift.thrift" + namespace cpp2 facebook.presto.thrift enum TaskState { @@ -579,10 +582,9 @@ struct UpdateHandle { struct ExecutionFailureInfo { 1: string type; 2: string message; - 3: optional ExecutionFailureInfo cause ( - cpp.ref_type = "shared", - drift.recursive_reference = true, - ); + @cpp.Ref{type = cpp.RefType.SharedMutable} + @thrift.DeprecatedUnvalidatedAnnotations{items = {"drift.recursive_reference": "true"}} + 3: optional ExecutionFailureInfo cause; 4: list suppressed; 5: list stack; 6: ErrorLocation errorLocation; diff --git a/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt index f75598ff1d4d7..2927edf96d82b 100644 --- a/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/thrift/tests/CMakeLists.txt @@ -24,12 +24,6 @@ target_link_libraries( presto_thrift_extra presto_thrift-cpp2 presto_native-cpp2 - ${THRIFTCPP2} - ${THRIFT_PROTOCOL} - ${THRIFT_METADATA} - ${THRIFT_CORE} - ${THRIFT_TRANSPORT} - ${FOLLY_WITH_DEPENDENCIES} ${RE2} GTest::gmock GTest::gtest diff --git a/presto-native-execution/presto_cpp/main/tool/trace/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/tool/trace/tests/CMakeLists.txt index e108c349ebdff..416a6ed75b3a0 100644 --- a/presto-native-execution/presto_cpp/main/tool/trace/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/tool/trace/tests/CMakeLists.txt @@ -21,6 +21,7 @@ target_link_libraries( velox_exec velox_exec_test_lib velox_hive_connector + velox_hive_iceberg_splitreader velox_query_trace_replayer_base gtest gtest_main diff --git a/presto-native-execution/presto_cpp/main/types/PrestoTaskId.h b/presto-native-execution/presto_cpp/main/types/PrestoTaskId.h index 6b0a767e27f48..e7e687b6a4e1d 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoTaskId.h +++ b/presto-native-execution/presto_cpp/main/types/PrestoTaskId.h @@ -12,8 +12,13 @@ * limitations under the License. */ #pragma once -#include + +#include #include +#include + +#include + #include "velox/common/base/Exceptions.h" namespace facebook::presto { diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp index 4b314d613925c..5f7cb4b62883b 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp @@ -128,8 +128,7 @@ velox::variant VeloxExprConverter::getConstantValue( std::string(valueVector->as>() ->valueAt(0))); default: - throw std::invalid_argument( - fmt::format("Unexpected Block type: {}", typeKind)); + VELOX_UNSUPPORTED("Unexpected Block type: {}", typeKind); } } @@ -520,24 +519,32 @@ TypedExprPtr VeloxExprConverter::toVeloxExpr( auto returnType = typeParser_->parse(pexpr.returnType); return std::make_shared( returnType, args, getFunctionName(signature)); + } - } else if ( - auto sqlFunctionHandle = + // Parse args and returnType once for all remaining branches + auto args = toVeloxExpr(pexpr.arguments); + auto returnType = typeParser_->parse(pexpr.returnType); + + if (auto sqlFunctionHandle = std::dynamic_pointer_cast( pexpr.functionHandle)) { - auto args = toVeloxExpr(pexpr.arguments); - auto returnType = typeParser_->parse(pexpr.returnType); return std::make_shared( returnType, args, getFunctionName(sqlFunctionHandle->functionId)); } + + else if ( + auto nativeFunctionHandle = + std::dynamic_pointer_cast( + pexpr.functionHandle)) { + auto signature = nativeFunctionHandle->signature; + return std::make_shared( + returnType, args, getFunctionName(signature)); + } #ifdef PRESTO_ENABLE_REMOTE_FUNCTIONS else if ( auto restFunctionHandle = std::dynamic_pointer_cast( pexpr.functionHandle)) { - auto args = toVeloxExpr(pexpr.arguments); - auto returnType = typeParser_->parse(pexpr.returnType); - functions::remote::rest::PrestoRestFunctionRegistration::getInstance() .registerFunction(*restFunctionHandle); return std::make_shared( @@ -894,8 +901,7 @@ TypedExprPtr VeloxExprConverter::toVeloxExpr( return toVeloxExpr(lambda); } - throw std::invalid_argument( - "Unsupported RowExpression type: " + pexpr->_type); + VELOX_UNSUPPORTED("Unsupported RowExpression type: {}", pexpr->_type); } } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp index c69089a759366..a869d0e3a26dd 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp @@ -65,6 +65,21 @@ bool useCachedHashTable(const protocol::PlanNode& node) { return false; } +const protocol::Signature* getSignatureFromFunctionHandle( + const std::shared_ptr& functionHandle) { + if (const auto builtin = + std::dynamic_pointer_cast( + functionHandle)) { + return &builtin->signature; + } else if ( + const auto native = + std::dynamic_pointer_cast( + functionHandle)) { + return &native->signature; + } + return nullptr; +} + std::vector getNames(const protocol::Assignments& assignments) { std::vector names; names.reserve(assignments.assignments.size()); @@ -345,12 +360,12 @@ core::LocalPartitionNode::Type toLocalExchangeType( } } -VectorSerde::Kind toVeloxSerdeKind(protocol::ExchangeEncoding encoding) { +std::string toVeloxSerdeKind(protocol::ExchangeEncoding encoding) { switch (encoding) { case protocol::ExchangeEncoding::COLUMNAR: - return VectorSerde::Kind::kPresto; + return "Presto"; case protocol::ExchangeEncoding::ROW_WISE: - return VectorSerde::Kind::kCompactRow; + return "CompactRow"; } VELOX_UNSUPPORTED("Unsupported encoding: {}.", fmt::underlying(encoding)); } @@ -903,7 +918,8 @@ VeloxQueryPlanConverterBase::generateOutputVariables( if (statisticsAggregation == nullptr) { return outputVariables; } - const auto statisticsOutputVariables = statisticsAggregation->outputVariables; + const auto& statisticsOutputVariables = + statisticsAggregation->outputVariables; auto statisticsGroupingVariables = statisticsAggregation->groupingVariables; outputVariables.insert( outputVariables.end(), @@ -933,12 +949,10 @@ void VeloxQueryPlanConverterBase::toAggregations( aggregate.call = std::dynamic_pointer_cast( exprConverter_.toVeloxExpr(prestoAggregation.call)); - if (const auto builtin = - std::dynamic_pointer_cast( - prestoAggregation.functionHandle)) { - const auto& signature = builtin->signature; - aggregate.rawInputTypes.reserve(signature.argumentTypes.size()); - for (const auto& argumentType : signature.argumentTypes) { + if (const auto signature = + getSignatureFromFunctionHandle(prestoAggregation.functionHandle)) { + aggregate.rawInputTypes.reserve(signature->argumentTypes.size()); + for (const auto& argumentType : signature->argumentTypes) { aggregate.rawInputTypes.push_back( stringToType(argumentType, typeParser_)); } @@ -1739,7 +1753,7 @@ toSortFieldsAndOrders( std::vector sortFields; std::vector sortOrders; if (orderingScheme != nullptr) { - auto nodeSpecOrdering = orderingScheme->orderBy; + const auto& nodeSpecOrdering = orderingScheme->orderBy; sortFields.reserve(nodeSpecOrdering.size()); sortOrders.reserve(nodeSpecOrdering.size()); for (const auto& [variable, sortOrder] : nodeSpecOrdering) { @@ -1823,6 +1837,22 @@ VeloxQueryPlanConverterBase::toVeloxQueryPlan( toVeloxQueryPlan(node->source, tableWriteInfo, taskId)); } +namespace { +core::TopNRowNumberNode::RankFunction prestoToVeloxRankFunction( + protocol::RankingFunction rankingFunction) { + switch (rankingFunction) { + case protocol::RankingFunction::ROW_NUMBER: + return core::TopNRowNumberNode::RankFunction::kRowNumber; + case protocol::RankingFunction::RANK: + return core::TopNRowNumberNode::RankFunction::kRank; + case protocol::RankingFunction::DENSE_RANK: + return core::TopNRowNumberNode::RankFunction::kDenseRank; + default: + VELOX_UNREACHABLE(); + } +} +}; // namespace + std::shared_ptr VeloxQueryPlanConverterBase::toVeloxQueryPlan( const std::shared_ptr& node, @@ -1858,7 +1888,7 @@ VeloxQueryPlanConverterBase::toVeloxQueryPlan( return std::make_shared( node->id, - core::TopNRowNumberNode::RankFunction::kRowNumber, + prestoToVeloxRankFunction(node->rankingType), partitionFields, sortFields, sortOrders, @@ -2231,7 +2261,7 @@ core::PlanNodePtr VeloxQueryPlanConverterBase::toVeloxQueryPlan( return core::PartitionedOutputNode::single( node->id, toRowType(node->outputVariables, typeParser_), - VectorSerde::Kind::kPresto, + "Presto", toVeloxQueryPlan(node->source, tableWriteInfo, taskId)); } @@ -2301,7 +2331,7 @@ core::PlanFragment VeloxBatchQueryPlanConverter::toVeloxQueryPlan( partitionedOutputNode->id(), 1, broadcastWriteNode->outputType(), - VectorSerde::Kind::kPresto, + "Presto", {broadcastWriteNode}); return planFragment; } @@ -2372,8 +2402,7 @@ core::PlanNodePtr VeloxBatchQueryPlanConverter::toVeloxQueryPlan( auto rowType = toRowType(node->outputVariables, typeParser_); // Broadcast exchange source. if (node->exchangeType == protocol::ExchangeNodeType::REPLICATE) { - return std::make_shared( - node->id, rowType, VectorSerde::Kind::kPresto); + return std::make_shared(node->id, rowType, "Presto"); } // Partitioned shuffle exchange source. return std::make_shared(node->id, rowType); diff --git a/presto-native-execution/presto_cpp/main/types/TypeParser.cpp b/presto-native-execution/presto_cpp/main/types/TypeParser.cpp index 5a8ee896dc027..3370642607401 100644 --- a/presto-native-execution/presto_cpp/main/types/TypeParser.cpp +++ b/presto-native-execution/presto_cpp/main/types/TypeParser.cpp @@ -11,19 +11,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include - #include "presto_cpp/main/types/TypeParser.h" -#include "velox/functions/prestosql/types/parser/TypeParser.h" #include "presto_cpp/main/common/Configs.h" +#include "velox/functions/prestosql/types/parser/TypeParser.h" namespace facebook::presto { velox::TypePtr TypeParser::parse(const std::string& text) const { if (SystemConfig::instance()->charNToVarcharImplicitCast()) { - if (text.find("char(") == 0 || text.find("CHAR(") == 0) { + if (text.starts_with("char(") || text.starts_with("CHAR(")) { return velox::VARCHAR(); } } diff --git a/presto-native-execution/presto_cpp/main/types/TypeParser.h b/presto-native-execution/presto_cpp/main/types/TypeParser.h index 0328f69d3665a..81c8bf008733e 100644 --- a/presto-native-execution/presto_cpp/main/types/TypeParser.h +++ b/presto-native-execution/presto_cpp/main/types/TypeParser.h @@ -14,6 +14,9 @@ #pragma once +#include +#include + #include "velox/type/Type.h" namespace facebook::presto { diff --git a/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.cpp b/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.cpp index f2b7653d65745..fba82342384e8 100644 --- a/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.cpp @@ -13,9 +13,11 @@ */ #include "presto_cpp/main/types/VeloxToPrestoExpr.h" #include +#include "presto_cpp/main/common/Utils.h" #include "presto_cpp/main/types/PrestoToVeloxExpr.h" #include "velox/core/ITypedExpr.h" #include "velox/expression/ExprConstants.h" +#include "velox/vector/BaseVector.h" #include "velox/vector/ConstantVector.h" using namespace facebook::presto; @@ -96,6 +98,30 @@ const std::unordered_map& veloxToPrestoOperatorMap() { } return veloxToPrestoOperatorMap; } + +// If the function name prefix starts from "presto.default", then it is a built +// in function handle. Otherwise, it is a native function handle. +std::shared_ptr getFunctionHandle( + const std::string& name, + const protocol::Signature& signature) { + static constexpr char const* kStatic = "$static"; + static constexpr char const* kNativeFunctionHandle = "native"; + static constexpr char const* builtInCatalog = "presto"; + static constexpr char const* builtInSchema = "default"; + + const auto parts = util::getFunctionNameParts(name); + if ((parts[0] == builtInCatalog) && (parts[1] == builtInSchema)) { + auto handle = std::make_shared(); + handle->_type = kStatic; + handle->signature = signature; + return handle; + } else { + auto handle = std::make_shared(); + handle->_type = kNativeFunctionHandle; + handle->signature = signature; + return handle; + } +} } // namespace std::string VeloxToPrestoExprConverter::getValueBlock( @@ -136,6 +162,78 @@ VeloxToPrestoExprConverter::getSwitchSpecialFormExpressionArgs( return result; } +void VeloxToPrestoExprConverter::getArgsFromConstantInList( + const velox::core::ConstantTypedExpr* inList, + std::vector& result) const { + const auto inListVector = inList->toConstantVector(pool_); + auto* constantVector = + inListVector->as>(); + VELOX_CHECK_NOT_NULL( + constantVector, "Expected ConstantVector of Array type for IN-list."); + const auto* arrayVector = + constantVector->wrappedVector()->as(); + VELOX_CHECK_NOT_NULL( + arrayVector, + "Expected constant IN-list to be of Array type, but got {}.", + constantVector->wrappedVector()->type()->toString()); + + auto wrappedIdx = constantVector->wrappedIndex(0); + auto size = arrayVector->sizeAt(wrappedIdx); + auto offset = arrayVector->offsetAt(wrappedIdx); + auto elementsVector = arrayVector->elements(); + + for (velox::vector_size_t i = 0; i < size; i++) { + auto elementIndex = offset + i; + auto elementConstant = + velox::BaseVector::wrapInConstant(1, elementIndex, elementsVector); + // Construct a core::ConstantTypedExpr from the constant value at this + // index in array vector, then convert it to a protocol::RowExpression. + const auto constantExpr = + std::make_shared(elementConstant); + result.push_back(getConstantExpression(constantExpr.get())); + } +} + +// IN expression in Presto is of form `expr0 IN [expr1, expr2, ..., exprN]`. +// The Velox representation of IN expression has the same form as Presto when +// any of the expressions in the IN list is non-constant; when the IN list only +// has constant expressions, it is of form `expr0 IN constantExpr(ARRAY[ +// expr1.constantValue(), expr2.constantValue(), ..., exprN.constantValue()])`. +// This function retrieves the arguments to Presto IN expression from Velox IN +// expression in both of these forms. +std::vector +VeloxToPrestoExprConverter::getInSpecialFormExpressionArgs( + const velox::core::CallTypedExpr* inExpr) const { + std::vector result; + const auto& inputs = inExpr->inputs(); + const auto numInputs = inputs.size(); + VELOX_CHECK_GE(numInputs, 2, "IN expression should have at least 2 inputs"); + + // Value being searched for with this `IN` expression is always the first + // input, convert it to a Presto expression. + result.push_back(getRowExpression(inputs.at(0))); + const auto& inList = inputs.at(1); + if (numInputs == 2 && inList->isConstantKind()) { + // Converts inputs from constant Velox IN-list to arguments in the Presto + // `IN` expression. Eg: For expression `col0 IN ['apple', 'foo', `bar`]`, + // `apple`, `foo`, and `bar` from the IN-list are converted to equivalent + // Presto constant expressions. + const auto* constantInList = + inList->asUnchecked(); + getArgsFromConstantInList(constantInList, result); + } else { + // Converts inputs from the Velox IN-list to arguments in the Presto `IN` + // expression when the Velox IN-list has at least one non-constant + // expression. Eg: For expression `col0 IN ['apple', col1, 'foo']`, `apple`, + // col1, and `foo` from the IN-list are converted to equivalent + // Presto expressions. + for (auto i = 1; i < numInputs; i++) { + result.push_back(getRowExpression(inputs[i])); + } + } + return result; +} + SpecialFormExpressionPtr VeloxToPrestoExprConverter::getSpecialFormExpression( const velox::core::CallTypedExpr* expr) const { VELOX_CHECK( @@ -156,11 +254,14 @@ SpecialFormExpressionPtr VeloxToPrestoExprConverter::getSpecialFormExpression( // Arguments for switch expression include 'WHEN' special form expression(s) // so they are constructed separately. static constexpr char const* kSwitch = "SWITCH"; + static constexpr char const* kIn = "IN"; if (name == kSwitch) { result.arguments = getSwitchSpecialFormExpressionArgs(expr); + } else if (name == kIn) { + result.arguments = getInSpecialFormExpressionArgs(expr); } else { - // Presto special form expressions that are not of type `SWITCH`, such as - // `IN`, `AND`, `OR` etc,. are handled in this clause. The list of Presto + // Presto special form expressions that are not of type `SWITCH` and `IN`, + // such as `AND`, `OR`, are handled in this clause. The list of Presto // special form expressions can be found in `kPrestoSpecialForms` in the // helper function `isPrestoSpecialForm`. auto exprInputs = expr->inputs(); @@ -218,7 +319,7 @@ SpecialFormExpressionPtr VeloxToPrestoExprConverter::getDereferenceExpression( const auto dereferenceInputs = std::vector{ dereferenceExpr->inputs().at(0), std::make_shared( - velox::BIGINT(), static_cast(dereferenceExpr->index()))}; + velox::INTEGER(), static_cast(dereferenceExpr->index()))}; for (const auto& input : dereferenceInputs) { const auto rowExpr = getRowExpression(input); protocol::to_json(j, rowExpr); @@ -255,7 +356,6 @@ LambdaDefinitionExpressionPtr VeloxToPrestoExprConverter::getLambdaExpression( CallExpressionPtr VeloxToPrestoExprConverter::getCallExpression( const velox::core::CallTypedExpr* expr) const { static constexpr char const* kCall = "call"; - static constexpr char const* kStatic = "$static"; json result; result["@type"] = kCall; @@ -281,10 +381,7 @@ CallExpressionPtr VeloxToPrestoExprConverter::getCallExpression( signature.argumentTypes = argumentTypes; signature.variableArity = false; - protocol::BuiltInFunctionHandle builtInFunctionHandle; - builtInFunctionHandle._type = kStatic; - builtInFunctionHandle.signature = signature; - result["functionHandle"] = builtInFunctionHandle; + result["functionHandle"] = getFunctionHandle(exprName, signature); result["returnType"] = getTypeSignature(expr->type()); result["arguments"] = json::array(); for (const auto& exprInput : exprInputs) { diff --git a/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.h b/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.h index 08e3de660e0ae..26369c7ec382e 100644 --- a/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.h +++ b/presto-native-execution/presto_cpp/main/types/VeloxToPrestoExpr.h @@ -81,6 +81,17 @@ class VeloxToPrestoExprConverter { std::vector getSwitchSpecialFormExpressionArgs( const velox::core::CallTypedExpr* switchExpr) const; + /// Helper function to convert values from a constant `IN` list in Velox + /// expression to equivalent Presto expressions. + void getArgsFromConstantInList( + const velox::core::ConstantTypedExpr* inList, + std::vector& result) const; + + /// Helper function to get the arguments for Presto `IN` expression from + /// Velox `IN` expression. + std::vector getInSpecialFormExpressionArgs( + const velox::core::CallTypedExpr* inExpr) const; + /// Helper function to construct a Presto `protocol::SpecialFormExpression` /// from a Velox call expression. This function should be called only on call /// expressions that map to a Presto `SpecialFormExpression`. This can be diff --git a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt index 25ca93ca91bec..8f9483e78cead 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries( velox_dwio_common velox_dwio_orc_reader velox_hive_connector + velox_hive_iceberg_splitreader velox_tpch_connector velox_tpcds_connector velox_exec @@ -70,6 +71,7 @@ target_link_libraries( velox_functions_prestosql velox_presto_type_parser velox_hive_connector + velox_hive_iceberg_splitreader velox_tpch_connector velox_type Boost::filesystem @@ -94,13 +96,18 @@ target_link_libraries( presto_types velox_dwio_common velox_hive_connector + velox_hive_iceberg_splitreader velox_tpch_connector velox_tpcds_connector GTest::gtest GTest::gtest_main ) -add_executable(presto_to_velox_query_plan_test PrestoToVeloxQueryPlanTest.cpp) +add_executable( + presto_to_velox_query_plan_test + PrestoToVeloxQueryPlanTest.cpp + NativeFunctionHandleTest.cpp +) add_test( NAME presto_to_velox_query_plan_test @@ -118,6 +125,7 @@ target_link_libraries( velox_dwio_common velox_exec_test_lib velox_hive_connector + velox_hive_iceberg_splitreader velox_tpch_connector GTest::gtest GTest::gtest_main diff --git a/presto-native-execution/presto_cpp/main/types/tests/NativeFunctionHandleTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/NativeFunctionHandleTest.cpp new file mode 100644 index 0000000000000..39caa7f663abe --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/NativeFunctionHandleTest.cpp @@ -0,0 +1,308 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "presto_cpp/main/types/PrestoToVeloxExpr.h" + +using namespace facebook::presto; +using namespace facebook::velox; + +class NativeFunctionHandleTest : public ::testing::Test { + protected: + TypeParser typeParser_; +}; + +TEST_F(NativeFunctionHandleTest, basic) { + try { + const std::string str = R"JSON( + { + "@type": "native", + "signature": { + "name": "native.default.test", + "kind": "SCALAR", + "typeVariableConstraints": [], + "longVariableConstraints": [], + "returnType": "integer", + "argumentTypes": ["integer", "integer"], + "variableArity": true + } + } + )JSON"; + const json j = json::parse(str); + const std::shared_ptr nativeFunctionHandle = + j; + + // Verify the signature parsing + ASSERT_NE(nativeFunctionHandle, nullptr); + EXPECT_EQ(nativeFunctionHandle->signature.name, "native.default.test"); + EXPECT_EQ( + nativeFunctionHandle->signature.kind, protocol::FunctionKind::SCALAR); + EXPECT_EQ(nativeFunctionHandle->signature.returnType, "integer"); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes.size(), 2); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes[0], "integer"); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes[1], "integer"); + EXPECT_EQ(nativeFunctionHandle->signature.variableArity, true); + + // Verify type parsing + auto returnType = + typeParser_.parse(nativeFunctionHandle->signature.returnType); + EXPECT_EQ(returnType->kind(), TypeKind::INTEGER); + + auto argType0 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[0]); + EXPECT_EQ(argType0->kind(), TypeKind::INTEGER); + + auto argType1 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[1]); + EXPECT_EQ(argType1->kind(), TypeKind::INTEGER); + + } catch (const std::exception& e) { + FAIL() << "Exception: " << e.what(); + } +} + +TEST_F(NativeFunctionHandleTest, basicArray) { + try { + const std::string str = R"JSON( + { + "@type": "native", + "signature": { + "name": "native.default.array_test", + "kind": "SCALAR", + "returnType": "array(bigint)", + "argumentTypes": ["array(bigint)", "array(varchar)"], + "typeVariableConstraints": [], + "longVariableConstraints": [], + "variableArity": false + } + } + )JSON"; + const json j = json::parse(str); + const std::shared_ptr nativeFunctionHandle = + j; + + // Verify the signature parsing + ASSERT_NE(nativeFunctionHandle, nullptr); + EXPECT_EQ( + nativeFunctionHandle->signature.name, "native.default.array_test"); + EXPECT_EQ(nativeFunctionHandle->signature.returnType, "array(bigint)"); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes.size(), 2); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[0], "array(bigint)"); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[1], "array(varchar)"); + + // Verify type parsing + auto returnType = + typeParser_.parse(nativeFunctionHandle->signature.returnType); + EXPECT_EQ(returnType->kind(), TypeKind::ARRAY); + auto returnArrayType = + std::dynamic_pointer_cast(returnType); + ASSERT_NE(returnArrayType, nullptr); + EXPECT_EQ(returnArrayType->elementType()->kind(), TypeKind::BIGINT); + + auto argType0 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[0]); + EXPECT_EQ(argType0->kind(), TypeKind::ARRAY); + auto argArrayType0 = std::dynamic_pointer_cast(argType0); + ASSERT_NE(argArrayType0, nullptr); + EXPECT_EQ(argArrayType0->elementType()->kind(), TypeKind::BIGINT); + + auto argType1 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[1]); + EXPECT_EQ(argType1->kind(), TypeKind::ARRAY); + auto argArrayType1 = std::dynamic_pointer_cast(argType1); + ASSERT_NE(argArrayType1, nullptr); + EXPECT_EQ(argArrayType1->elementType()->kind(), TypeKind::VARCHAR); + + } catch (const std::exception& e) { + FAIL() << "Exception: " << e.what(); + } +} + +TEST_F(NativeFunctionHandleTest, nestedComplexTypes) { + try { + const std::string str = R"JSON( + { + "@type": "native", + "signature": { + "name": "native.default.nested_function", + "kind": "SCALAR", + "typeVariableConstraints": [], + "longVariableConstraints": [], + "returnType": "map(varchar,array(bigint))", + "argumentTypes": ["array(map(varchar,bigint))", "row(array(decimal(10,2)),map(bigint,varchar))"], + "variableArity": false + } + } + )JSON"; + const json j = json::parse(str); + const std::shared_ptr nativeFunctionHandle = + j; + + // Verify the signature parsing + ASSERT_NE(nativeFunctionHandle, nullptr); + EXPECT_EQ( + nativeFunctionHandle->signature.name, "native.default.nested_function"); + EXPECT_EQ( + nativeFunctionHandle->signature.returnType, + "map(varchar,array(bigint))"); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes.size(), 2); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[0], + "array(map(varchar,bigint))"); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[1], + "row(array(decimal(10,2)),map(bigint,varchar))"); + + // Verify return type: map(varchar,array(bigint)) + auto returnType = + typeParser_.parse(nativeFunctionHandle->signature.returnType); + EXPECT_EQ(returnType->kind(), TypeKind::MAP); + auto returnMapType = std::dynamic_pointer_cast(returnType); + ASSERT_NE(returnMapType, nullptr); + EXPECT_EQ(returnMapType->keyType()->kind(), TypeKind::VARCHAR); + EXPECT_EQ(returnMapType->valueType()->kind(), TypeKind::ARRAY); + auto valueArrayType = + std::dynamic_pointer_cast(returnMapType->valueType()); + ASSERT_NE(valueArrayType, nullptr); + EXPECT_EQ(valueArrayType->elementType()->kind(), TypeKind::BIGINT); + + // Verify arg0 type: array(map(varchar,bigint)) + auto argType0 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[0]); + EXPECT_EQ(argType0->kind(), TypeKind::ARRAY); + auto argArrayType = std::dynamic_pointer_cast(argType0); + ASSERT_NE(argArrayType, nullptr); + EXPECT_EQ(argArrayType->elementType()->kind(), TypeKind::MAP); + auto elementMapType = + std::dynamic_pointer_cast(argArrayType->elementType()); + ASSERT_NE(elementMapType, nullptr); + EXPECT_EQ(elementMapType->keyType()->kind(), TypeKind::VARCHAR); + EXPECT_EQ(elementMapType->valueType()->kind(), TypeKind::BIGINT); + + // Verify arg1 type: row(array(decimal(10,2)),map(bigint,varchar)) + auto argType1 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[1]); + EXPECT_EQ(argType1->kind(), TypeKind::ROW); + auto argRowType = std::dynamic_pointer_cast(argType1); + ASSERT_NE(argRowType, nullptr); + EXPECT_EQ(argRowType->size(), 2); + + // First child: array(decimal(10,2)) + EXPECT_EQ(argRowType->childAt(0)->kind(), TypeKind::ARRAY); + auto childArrayType = + std::dynamic_pointer_cast(argRowType->childAt(0)); + ASSERT_NE(childArrayType, nullptr); + EXPECT_EQ(childArrayType->elementType()->kind(), TypeKind::BIGINT); + + // Second child: map(bigint,varchar) + EXPECT_EQ(argRowType->childAt(1)->kind(), TypeKind::MAP); + auto childMapType = + std::dynamic_pointer_cast(argRowType->childAt(1)); + ASSERT_NE(childMapType, nullptr); + EXPECT_EQ(childMapType->keyType()->kind(), TypeKind::BIGINT); + EXPECT_EQ(childMapType->valueType()->kind(), TypeKind::VARCHAR); + + } catch (const std::exception& e) { + FAIL() << "Exception: " << e.what(); + } +} + +TEST_F(NativeFunctionHandleTest, mixedConstraints) { + try { + const std::string str = R"JSON( + { + "@type": "native", + "signature": { + "name": "native.default.mod", + "kind": "SCALAR", + "typeVariableConstraints": [ + { + "name": "T", + "comparableRequired": false, + "orderableRequired": false, + "variadicBound": "", + "nonDecimalNumericRequired": false + } + ], + "longVariableConstraints":[ + { + "expression":"min(i2 - i6, i1 - i5) + max(i5, i6)", + "name":"i3" + } + ], + "returnType":"decimal(10, 2)", + "argumentTypes":["decimal(10, 2)","decimal(10, 2)"], + "variableArity":false + } + } + )JSON"; + const json j = json::parse(str); + const std::shared_ptr nativeFunctionHandle = + j; + + // Verify the signature parsing + ASSERT_NE(nativeFunctionHandle, nullptr); + EXPECT_EQ(nativeFunctionHandle->signature.name, "native.default.mod"); + EXPECT_EQ( + nativeFunctionHandle->signature.kind, protocol::FunctionKind::SCALAR); + EXPECT_EQ(nativeFunctionHandle->signature.returnType, "decimal(10, 2)"); + EXPECT_EQ(nativeFunctionHandle->signature.argumentTypes.size(), 2); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[0], "decimal(10, 2)"); + EXPECT_EQ( + nativeFunctionHandle->signature.argumentTypes[1], "decimal(10, 2)"); + + // Verify type variable constraints + EXPECT_EQ( + nativeFunctionHandle->signature.typeVariableConstraints.size(), 1); + EXPECT_EQ( + nativeFunctionHandle->signature.typeVariableConstraints[0].name, "T"); + EXPECT_EQ( + nativeFunctionHandle->signature.typeVariableConstraints[0] + .comparableRequired, + false); + EXPECT_EQ( + nativeFunctionHandle->signature.typeVariableConstraints[0] + .orderableRequired, + false); + + // Verify long variable constraints + EXPECT_EQ( + nativeFunctionHandle->signature.longVariableConstraints.size(), 1); + EXPECT_EQ( + nativeFunctionHandle->signature.longVariableConstraints[0].name, "i3"); + EXPECT_EQ( + nativeFunctionHandle->signature.longVariableConstraints[0].expression, + "min(i2 - i6, i1 - i5) + max(i5, i6)"); + + // Verify type parsing for return type + auto returnType = + typeParser_.parse(nativeFunctionHandle->signature.returnType); + EXPECT_EQ(returnType->kind(), TypeKind::BIGINT); + + // Verify arg0 type: decimal(10, 2) + auto argType0 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[0]); + EXPECT_EQ(argType0->kind(), TypeKind::BIGINT); + + // Verify arg1 type: decimal(10, 2) + auto argType1 = + typeParser_.parse(nativeFunctionHandle->signature.argumentTypes[1]); + EXPECT_EQ(argType1->kind(), TypeKind::BIGINT); + } catch (const std::exception& e) { + FAIL() << "Exception: " << e.what(); + } +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp index f139db590a889..2ecd12aa2caf7 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp @@ -144,7 +144,7 @@ TEST_F(PlanConverterTest, partitionedOutput) { ASSERT_EQ(keys.size(), 2); ASSERT_EQ(keys[0]->toString(), "{cluster_label_v2}"); ASSERT_EQ(keys[1]->toString(), "\"expr_181\""); - ASSERT_EQ(partitionedOutput->serdeKind(), VectorSerde::Kind::kCompactRow); + ASSERT_EQ(partitionedOutput->serdeKind(), "CompactRow"); } // Final Agg stage plan for select regionkey, sum(1) from nation group by 1 diff --git a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp index 6f3725faadfb3..69458fa279eb5 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PrestoToVeloxConnectorTest.cpp @@ -21,6 +21,7 @@ #include "velox/common/base/tests/GTestUtils.h" #include "velox/connectors/hive/HiveConnector.h" #include "velox/connectors/hive/TableHandle.h" +#include "velox/connectors/hive/iceberg/IcebergColumnHandle.h" using namespace facebook::presto; using namespace facebook::velox; @@ -185,3 +186,110 @@ TEST_F(PrestoToVeloxConnectorTest, hiveLowercasesColumnNames) { EXPECT_EQ(dataColumnsType->nameOf(0), "mixedcasecol1"); EXPECT_EQ(dataColumnsType->nameOf(1), "uppercasecol2"); } + +namespace { + +protocol::iceberg::IcebergColumnHandle createIcebergColumnHandle( + const std::string& name, + int32_t fieldId, + const std::string& type, + protocol::iceberg::TypeCategory typeCategory = + protocol::iceberg::TypeCategory::PRIMITIVE, + const std::vector& children = {}) { + protocol::iceberg::IcebergColumnHandle column; + column.columnIdentity.name = name; + column.columnIdentity.id = fieldId; + column.columnIdentity.typeCategory = typeCategory; + column.columnIdentity.children = children; + column.type = type; + column.columnType = protocol::hive::ColumnType::REGULAR; + return column; +} + +} // namespace + +TEST_F(PrestoToVeloxConnectorTest, icebergColumnHandleSimple) { + auto icebergColumn = createIcebergColumnHandle("col1", 1, "integer"); + + IcebergPrestoToVeloxConnector icebergConnector("iceberg"); + auto handle = + icebergConnector.toVeloxColumnHandle(&icebergColumn, *typeParser_); + auto* icebergHandle = + dynamic_cast( + handle.get()); + ASSERT_NE(icebergHandle, nullptr); + + EXPECT_EQ(icebergHandle->name(), "col1"); + EXPECT_EQ(icebergHandle->dataType()->kind(), TypeKind::INTEGER); + EXPECT_EQ(icebergHandle->field().fieldId, 1); + EXPECT_TRUE(icebergHandle->field().children.empty()); +} + +TEST_F(PrestoToVeloxConnectorTest, icebergColumnHandleNested) { + protocol::iceberg::ColumnIdentity child1; + child1.name = "child1"; + child1.id = 2; + child1.typeCategory = protocol::iceberg::TypeCategory::PRIMITIVE; + + protocol::iceberg::ColumnIdentity child2; + child2.name = "child2"; + child2.id = 3; + child2.typeCategory = protocol::iceberg::TypeCategory::PRIMITIVE; + + auto icebergColumn = createIcebergColumnHandle( + "struct_col", + 1, + "row(child1 integer, child2 varchar)", + protocol::iceberg::TypeCategory::STRUCT, + {child1, child2}); + + IcebergPrestoToVeloxConnector icebergConnector("iceberg"); + auto handle = + icebergConnector.toVeloxColumnHandle(&icebergColumn, *typeParser_); + auto* icebergHandle = + dynamic_cast( + handle.get()); + ASSERT_NE(icebergHandle, nullptr); + + EXPECT_EQ(icebergHandle->name(), "struct_col"); + EXPECT_EQ(icebergHandle->dataType()->kind(), TypeKind::ROW); + EXPECT_EQ(icebergHandle->field().fieldId, 1); + ASSERT_EQ(icebergHandle->field().children.size(), 2); + EXPECT_EQ(icebergHandle->field().children[0].fieldId, 2); + EXPECT_EQ(icebergHandle->field().children[1].fieldId, 3); +} + +TEST_F(PrestoToVeloxConnectorTest, icebergColumnHandleDeeplyNested) { + protocol::iceberg::ColumnIdentity inner; + inner.name = "inner"; + inner.id = 3; + inner.typeCategory = protocol::iceberg::TypeCategory::PRIMITIVE; + + protocol::iceberg::ColumnIdentity middle; + middle.name = "middle"; + middle.id = 2; + middle.typeCategory = protocol::iceberg::TypeCategory::STRUCT; + middle.children = {inner}; + + auto icebergColumn = createIcebergColumnHandle( + "outer", + 1, + "row(middle row(inner bigint))", + protocol::iceberg::TypeCategory::STRUCT, + {middle}); + + IcebergPrestoToVeloxConnector icebergConnector("iceberg"); + auto handle = + icebergConnector.toVeloxColumnHandle(&icebergColumn, *typeParser_); + auto* icebergHandle = + dynamic_cast( + handle.get()); + ASSERT_NE(icebergHandle, nullptr); + + EXPECT_EQ(icebergHandle->name(), "outer"); + EXPECT_EQ(icebergHandle->field().fieldId, 1); + ASSERT_EQ(icebergHandle->field().children.size(), 1); + EXPECT_EQ(icebergHandle->field().children[0].fieldId, 2); + ASSERT_EQ(icebergHandle->field().children[0].children.size(), 1); + EXPECT_EQ(icebergHandle->field().children[0].children[0].fieldId, 3); +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp index b341789aab7b3..601b1e70a1a5a 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp @@ -11,17 +11,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include +#include -using namespace std; +#include +#include +#include namespace facebook::presto::test::utils { namespace { -const std::string applyClionDirFix( +std::string applyClionDirFix( std::string& currentPath, - const std::string& fileName) { + const std::string& fileName) noexcept { // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ // directory. Hard-coded json files are not copied there and test fails with // file not found. Fixing the path so that we can trigger these tests from @@ -33,8 +34,8 @@ const std::string applyClionDirFix( } } // namespace -const std::string getDataPath(const std::string& fileName) { - std::string currentPath = boost::filesystem::current_path().c_str(); +std::string getDataPath(const std::string& fileName) noexcept { + std::string currentPath = boost::filesystem::current_path().string(); if (boost::algorithm::ends_with(currentPath, "fbcode")) { return currentPath + "/github/presto-trunk/presto-native-execution/presto_cpp/main/types/tests/data/" + @@ -48,10 +49,10 @@ const std::string getDataPath(const std::string& fileName) { return applyClionDirFix(currentPath, fileName); } -const std::string getDataPath( +std::string getDataPath( const std::string& testDataDir, - const std::string& fileName) { - std::string currentPath = boost::filesystem::current_path().c_str(); + const std::string& fileName) noexcept { + std::string currentPath = boost::filesystem::current_path().string(); if (boost::algorithm::ends_with(currentPath, "fbcode")) { return currentPath + testDataDir + fileName; } diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h index 660c43f03784c..76501275abfaa 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h @@ -1,3 +1,4 @@ +#pragma once /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -11,9 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include + namespace facebook::presto::test::utils { -const std::string getDataPath(const std::string& fileName); -const std::string getDataPath( +std::string getDataPath(const std::string& fileName) noexcept; +std::string getDataPath( const std::string& testDataDir, - const std::string& fileName); + const std::string& fileName) noexcept; } // namespace facebook::presto::test::utils diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 56e35b97348de..39b1686cf0692 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -114,7 +114,7 @@ void to_json(json& j, const std::shared_ptr& p) { return; } if (type == "native") { - j = *std::static_pointer_cast(p); + j = *std::static_pointer_cast(p); return; } if (type == "json_file") { @@ -149,8 +149,8 @@ void from_json(const json& j, std::shared_ptr& p) { return; } if (type == "native") { - std::shared_ptr k = - std::make_shared(); + std::shared_ptr k = + std::make_shared(); j.get_to(*k); p = std::static_pointer_cast(k); return; @@ -752,7 +752,7 @@ void to_json(json& j, const std::shared_ptr& p) { j = *std::static_pointer_cast(p); return; } - if (type == "com.facebook.presto.sql.planner.plan.TopNRowNumberNode") { + if (type == ".TopNRowNumberNode") { j = *std::static_pointer_cast(p); return; } @@ -940,7 +940,7 @@ void from_json(const json& j, std::shared_ptr& p) { p = std::static_pointer_cast(k); return; } - if (type == "com.facebook.presto.sql.planner.plan.TopNRowNumberNode") { + if (type == ".TopNRowNumberNode") { std::shared_ptr k = std::make_shared(); j.get_to(*k); @@ -5206,6 +5206,13 @@ void to_json(json& j, const ErrorCode& p) { to_json_key(j, "name", p.name, "ErrorCode", "String", "name"); to_json_key(j, "type", p.type, "ErrorCode", "ErrorType", "type"); to_json_key(j, "retriable", p.retriable, "ErrorCode", "bool", "retriable"); + to_json_key( + j, + "catchableByTry", + p.catchableByTry, + "ErrorCode", + "bool", + "catchableByTry"); } void from_json(const json& j, ErrorCode& p) { @@ -5213,6 +5220,13 @@ void from_json(const json& j, ErrorCode& p) { from_json_key(j, "name", p.name, "ErrorCode", "String", "name"); from_json_key(j, "type", p.type, "ErrorCode", "ErrorType", "type"); from_json_key(j, "retriable", p.retriable, "ErrorCode", "bool", "retriable"); + from_json_key( + j, + "catchableByTry", + p.catchableByTry, + "ErrorCode", + "bool", + "catchableByTry"); } } // namespace facebook::presto::protocol namespace facebook::presto::protocol { @@ -7558,6 +7572,34 @@ void from_json(const json& j, MergeTarget& p) { } } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +NativeFunctionHandle::NativeFunctionHandle() noexcept { + _type = "native"; +} + +void to_json(json& j, const NativeFunctionHandle& p) { + j = json::object(); + j["@type"] = "native"; + to_json_key( + j, + "signature", + p.signature, + "NativeFunctionHandle", + "Signature", + "signature"); +} + +void from_json(const json& j, NativeFunctionHandle& p) { + p._type = j["@type"]; + from_json_key( + j, + "signature", + p.signature, + "NativeFunctionHandle", + "Signature", + "signature"); +} +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { void to_json(json& j, const NativeSidecarFailureInfo& p) { j = json::object(); @@ -12149,13 +12191,51 @@ void from_json(const json& j, TopNNode& p) { } } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +// Loosely copied this here from NLOHMANN_JSON_SERIALIZE_ENUM() + +// NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays +static const std::pair RankingFunction_enum_table[] = + { // NOLINT: cert-err58-cpp + {RankingFunction::ROW_NUMBER, "ROW_NUMBER"}, + {RankingFunction::RANK, "RANK"}, + {RankingFunction::DENSE_RANK, "DENSE_RANK"}}; +void to_json(json& j, const RankingFunction& e) { + static_assert( + std::is_enum::value, "RankingFunction must be an enum!"); + const auto* it = std::find_if( + std::begin(RankingFunction_enum_table), + std::end(RankingFunction_enum_table), + [e](const std::pair& ej_pair) -> bool { + return ej_pair.first == e; + }); + j = ((it != std::end(RankingFunction_enum_table)) + ? it + : std::begin(RankingFunction_enum_table)) + ->second; +} +void from_json(const json& j, RankingFunction& e) { + static_assert( + std::is_enum::value, "RankingFunction must be an enum!"); + const auto* it = std::find_if( + std::begin(RankingFunction_enum_table), + std::end(RankingFunction_enum_table), + [&j](const std::pair& ej_pair) -> bool { + return ej_pair.second == j; + }); + e = ((it != std::end(RankingFunction_enum_table)) + ? it + : std::begin(RankingFunction_enum_table)) + ->first; +} +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { TopNRowNumberNode::TopNRowNumberNode() noexcept { - _type = "com.facebook.presto.sql.planner.plan.TopNRowNumberNode"; + _type = ".TopNRowNumberNode"; } void to_json(json& j, const TopNRowNumberNode& p) { j = json::object(); - j["@type"] = "com.facebook.presto.sql.planner.plan.TopNRowNumberNode"; + j["@type"] = ".TopNRowNumberNode"; to_json_key(j, "id", p.id, "TopNRowNumberNode", "PlanNodeId", "id"); to_json_key(j, "source", p.source, "TopNRowNumberNode", "PlanNode", "source"); to_json_key( @@ -12165,6 +12245,13 @@ void to_json(json& j, const TopNRowNumberNode& p) { "TopNRowNumberNode", "DataOrganizationSpecification", "specification"); + to_json_key( + j, + "rankingType", + p.rankingType, + "TopNRowNumberNode", + "RankingFunction", + "rankingType"); to_json_key( j, "rowNumberVariable", @@ -12201,6 +12288,13 @@ void from_json(const json& j, TopNRowNumberNode& p) { "TopNRowNumberNode", "DataOrganizationSpecification", "specification"); + from_json_key( + j, + "rankingType", + p.rankingType, + "TopNRowNumberNode", + "RankingFunction", + "rankingType"); from_json_key( j, "rowNumberVariable", diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index a189d8bfa18b9..a20589ae571ff 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -1271,6 +1271,7 @@ struct ErrorCode { String name = {}; ErrorType type = {}; bool retriable = {}; + bool catchableByTry = {}; }; void to_json(json& j, const ErrorCode& p); void from_json(const json& j, ErrorCode& p); @@ -1849,6 +1850,15 @@ void to_json(json& j, const MergeTarget& p); void from_json(const json& j, MergeTarget& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +struct NativeFunctionHandle : public FunctionHandle { + Signature signature = {}; + + NativeFunctionHandle() noexcept; +}; +void to_json(json& j, const NativeFunctionHandle& p); +void from_json(const json& j, NativeFunctionHandle& p); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { struct NativeSidecarFailureInfo { String type = {}; String message = {}; @@ -2618,9 +2628,15 @@ void to_json(json& j, const TopNNode& p); void from_json(const json& j, TopNNode& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +enum class RankingFunction { ROW_NUMBER, RANK, DENSE_RANK }; +extern void to_json(json& j, const RankingFunction& e); +extern void from_json(const json& j, RankingFunction& e); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { struct TopNRowNumberNode : public PlanNode { std::shared_ptr source = {}; DataOrganizationSpecification specification = {}; + RankingFunction rankingType = {}; VariableReferenceExpression rowNumberVariable = {}; int maxRowCountPerPartition = {}; bool partial = {}; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml index c02a4825f550c..2036ea37a8d4b 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml @@ -172,7 +172,7 @@ AbstractClasses: - { name: TableWriterNode, key: .TableWriterNode } - { name: TableWriterMergeNode, key: com.facebook.presto.sql.planner.plan.TableWriterMergeNode } - { name: TopNNode, key: .TopNNode } - - { name: TopNRowNumberNode, key: com.facebook.presto.sql.planner.plan.TopNRowNumberNode } + - { name: TopNRowNumberNode, key: .TopNRowNumberNode } - { name: UnnestNode, key: .UnnestNode } - { name: ValuesNode, key: .ValuesNode } - { name: AssignUniqueId, key: com.facebook.presto.sql.planner.plan.AssignUniqueId } @@ -200,7 +200,7 @@ AbstractClasses: super: JsonEncodedSubclass subclasses: - { name: BuiltInFunctionHandle, key: $static } - - { name: SqlFunctionHandle, key: native } + - { name: NativeFunctionHandle, key: native } - { name: SqlFunctionHandle, key: json_file } - { name: SqlFunctionHandle, key: sql_function_handle } - { name: RestFunctionHandle, key: rest } @@ -249,7 +249,7 @@ JavaClasses: - presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/GroupIdNode.java - presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/RowNumberNode.java - presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/SampleNode.java - - presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNRowNumberNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java - presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/UnnestNode.java @@ -363,3 +363,4 @@ JavaClasses: - presto-spi/src/main/java/com/facebook/presto/spi/NodeStats.java - presto-spi/src/main/java/com/facebook/presto/spi/NodeLoadMetrics.java - presto-spi/src/main/java/com/facebook/presto/spi/session/SessionPropertyMetadata.java + - presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml index adf58e95f1ced..6b561fa93c7c6 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml @@ -161,7 +161,7 @@ AbstractClasses: - { name: TableWriterNode, key: .TableWriterNode } - { name: TableWriterMergeNode, key: com.facebook.presto.sql.planner.plan.TableWriterMergeNode } - { name: TopNNode, key: .TopNNode } - - { name: TopNRowNumberNode, key: com.facebook.presto.sql.planner.plan.TopNRowNumberNode } + - { name: TopNRowNumberNode, key: .TopNRowNumberNode } - { name: UnnestNode, key: .UnnestNode } - { name: ValuesNode, key: .ValuesNode } - { name: AssignUniqueId, key: com.facebook.presto.sql.planner.plan.AssignUniqueId } @@ -238,7 +238,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/GroupIdNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RowNumberNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SampleNode.java - - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNRowNumberNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/UnnestNode.java diff --git a/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile b/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile index fc9c0c810f265..928692df345c4 100644 --- a/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/centos-dependency.dockerfile @@ -16,10 +16,15 @@ FROM quay.io/centos/centos:stream9 # from https://github.com/facebookincubator/velox/pull/14366 ARG ARM_BUILD_TARGET +# This defaults to 12.9 but can be overridden with a build arg +ARG CUDA_VERSION + ENV PROMPT_ALWAYS_RESPOND=y ENV CC=/opt/rh/gcc-toolset-12/root/bin/gcc ENV CXX=/opt/rh/gcc-toolset-12/root/bin/g++ ENV ARM_BUILD_TARGET=${ARM_BUILD_TARGET} +ENV CUDA_VERSION=${CUDA_VERSION:-12.9} +ENV UCX_VERSION="1.19.0" RUN mkdir -p /scripts /velox/scripts COPY scripts /scripts @@ -27,7 +32,9 @@ COPY velox/scripts /velox/scripts # Copy extra script called during setup. # from https://github.com/facebookincubator/velox/pull/14016 COPY velox/CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch /velox +COPY CMake/arrow/arrow-flight.patch /scripts ENV VELOX_ARROW_CMAKE_PATCH=/velox/cmake-compatibility.patch +ENV EXTRA_ARROW_PATCH=/scripts/arrow-flight.patch RUN bash -c "mkdir build && \ (cd build && ../scripts/setup-centos.sh && \ ../scripts/setup-adapters.sh && \ @@ -35,7 +42,8 @@ RUN bash -c "mkdir build && \ source ../velox/scripts/setup-centos-adapters.sh && \ install_adapters && \ install_clang15 && \ - install_cuda 12.8) && \ + install_cuda ${CUDA_VERSION} && \ + install_ucx) && \ rm -rf build" # put CUDA binaries on the PATH diff --git a/presto-native-execution/scripts/setup-adapters.sh b/presto-native-execution/scripts/setup-adapters.sh index d91845532ab9d..0cceb12835b73 100755 --- a/presto-native-execution/scripts/setup-adapters.sh +++ b/presto-native-execution/scripts/setup-adapters.sh @@ -14,14 +14,16 @@ # Propagate errors and improve debugging. set -eufx -o pipefail +EXTRA_ARROW_PATCH=${EXTRA_ARROW_PATCH:-""} + JWT_VERSION="v0.6.0" PROMETHEUS_VERSION="v1.2.4" -SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") -if [ -f "${SCRIPT_DIR}/setup-common.sh" ]; then - source "${SCRIPT_DIR}/setup-common.sh" +PRESTO_SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +if [ -f "${PRESTO_SCRIPT_DIR}/setup-common.sh" ]; then + source "${PRESTO_SCRIPT_DIR}/setup-common.sh" else - source "${SCRIPT_DIR}/../velox/scripts/setup-common.sh" + source "${PRESTO_SCRIPT_DIR}/../velox/scripts/setup-common.sh" fi DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} @@ -47,7 +49,10 @@ function install_arrow_flight { # Arrow Flight enabled. The Velox version of Arrow is used. # NOTE: benchmarks are on due to a compilation error with v15.0.0, once updated that can be removed # see https://github.com/apache/arrow/issues/41617 - EXTRA_ARROW_OPTIONS=" -DARROW_FLIGHT=ON -DARROW_BUILD_BENCHMARKS=ON -DgRPC_SOURCE=BUNDLED -DProtobuf_SOURCE=BUNDLED " + if [ -z "$EXTRA_ARROW_PATCH" ]; then + EXTRA_ARROW_PATCH="${PRESTO_SCRIPT_DIR}/../CMake/arrow/arrow-flight.patch" + fi + EXTRA_ARROW_OPTIONS=" -DARROW_FLIGHT=ON -DARROW_BUILD_BENCHMARKS=ON -Dabsl_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED -DProtobuf_SOURCE=BUNDLED " install_arrow } diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 4e246ab33b6aa..f6226ff2d9cd7 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -32,7 +32,8 @@ fi export NPROC=${NPROC:-$(getconf _NPROCESSORS_ONLN)} function install_presto_deps_from_package_managers { - dnf install -y maven java clang-tools-extra jq perl-XML-XPath + # proxygen requires c-ares-devel + dnf install -y maven java clang-tools-extra jq perl-XML-XPath c-ares-devel # This python version is installed by the Velox setup scripts pip install regex pyyaml chevron black ptsd-jbroll } @@ -54,9 +55,6 @@ function install_gperf { function install_proxygen { wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen - # Folly Portability.h being used to decide whether or not support coroutines - # causes issues (build, lin) if the selection is not consistent across users of folly. - EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" cmake_install_dir proxygen -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON } diff --git a/presto-native-execution/scripts/setup-macos.sh b/presto-native-execution/scripts/setup-macos.sh index e773d4210438c..ad2bee2288b4a 100755 --- a/presto-native-execution/scripts/setup-macos.sh +++ b/presto-native-execution/scripts/setup-macos.sh @@ -25,12 +25,19 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-false}" source "$(dirname "${BASH_SOURCE[0]}")/../velox/scripts/setup-macos.sh" GPERF_VERSION="3.1" DATASKETCHES_VERSION="5.2.0" +# c-ares is required for proxygen +MACOS_PRESTO_DEPS="c-ares" + +function install_presto_deps_from_brew { + local pkg + + for pkg in ${MACOS_PRESTO_DEPS}; do + install_from_brew "${pkg}" + done +} function install_proxygen { wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen - # Folly Portability.h being used to decide whether or not support coroutines - # causes issues (build, lin) if the selection is not consistent across users of folly. - EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" cmake_install_dir proxygen -DBUILD_TESTS=OFF } @@ -49,6 +56,7 @@ function install_datasketches { } function install_presto_deps { + run_and_time install_presto_deps_from_brew run_and_time install_gperf run_and_time install_proxygen run_and_time install_datasketches diff --git a/presto-native-execution/scripts/setup-ubuntu.sh b/presto-native-execution/scripts/setup-ubuntu.sh index 70c172b50a0c7..5332933908b48 100755 --- a/presto-native-execution/scripts/setup-ubuntu.sh +++ b/presto-native-execution/scripts/setup-ubuntu.sh @@ -22,13 +22,10 @@ SUDO="${SUDO:-"sudo --preserve-env"}" DATASKETCHES_VERSION="5.2.0" function install_proxygen { - # proxygen requires python and gperf + # proxygen requires python, gperf, and libc-ares-dev ${SUDO} apt update - ${SUDO} apt install -y gperf python3 + ${SUDO} apt install -y gperf python3 libc-ares-dev wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen - # Folly Portability.h being used to decide whether or not support coroutines - # causes issues (build, lin) if the selection is not consistent across users of folly. - EXTRA_PKG_CXXFLAGS=" -DFOLLY_CFG_NO_COROUTINES" cmake_install_dir proxygen -DBUILD_TESTS=OFF } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java index fde2a56d6f96a..fa5c331b84727 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeGeneralQueries.java @@ -403,8 +403,8 @@ public void testDateFilter() .build(); try { - computeExpected(String.format("CREATE TABLE %s (c0 DATE) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of()); - computeExpected(String.format("INSERT INTO %s VALUES (DATE '1996-01-02'), (DATE '1996-12-01')", tmpTableName), ImmutableList.of()); + computeExpected(String.format("CREATE TABLE %s (c0 DATE) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of(BIGINT)); + computeExpected(String.format("INSERT INTO %s VALUES (DATE '1996-01-02'), (DATE '1996-12-01')", tmpTableName), ImmutableList.of(BIGINT)); assertQueryResultCount(session, String.format("SELECT * from %s where c0 in (select c0 from %s) ", tmpTableName, tmpTableName), 2); } @@ -420,60 +420,10 @@ public void testReadTableWithTextfileFormat() String tmpTableName = generateRandomTableName(); try { - getExpectedQueryRunner().execute(getSession(), format( - "CREATE TABLE %s (" + - "id BIGINT," + - "name VARCHAR," + - "is_active BOOLEAN," + - "score DOUBLE," + - "created_at TIMESTAMP," + - "tags ARRAY," + - "metrics ARRAY," + - "properties MAP," + - "flags MAP," + - "nested_struct ROW(sub_id INTEGER, sub_name VARCHAR, sub_scores ARRAY, sub_map MAP)," + - "price DECIMAL(15,2)," + - "amount DECIMAL(21,6)," + - "event_date DATE," + - "ds VARCHAR" + - ") WITH (format = 'TEXTFILE', partitioned_by = ARRAY['ds'])", tmpTableName), ImmutableList.of()); - getExpectedQueryRunner().execute(getSession(), format( - "INSERT INTO %s (" + - "id," + - "name," + - "is_active," + - "score," + - "created_at," + - "tags," + - "metrics," + - "properties," + - "flags," + - "nested_struct," + - "price," + - "amount," + - "event_date," + - "ds" + - ") VALUES (" + - "1001," + - "'Jane Doe'," + - "TRUE," + - "88.5," + - "TIMESTAMP '2025-07-23 10:00:00'," + - "ARRAY['alpha', 'beta', 'gamma']," + - "ARRAY[3.14, 2.71, 1.41]," + - "MAP(ARRAY['color', 'size'], ARRAY['blue', 'large'])," + - "MAP(ARRAY[TINYINT '1', TINYINT '2'], ARRAY[TRUE, FALSE])," + - "ROW(" + - "42," + - "'sub_jane'," + - "ARRAY[REAL '1.1', REAL '2.2', REAL '3.3']," + - "MAP(ARRAY[SMALLINT '10', SMALLINT '20'], ARRAY['foo', 'bar'])" + - ")," + - "DECIMAL '12.34'," + - "CAST('-123456789012345.123456' as DECIMAL(21,6))," + - "DATE '2024-02-29'," + - "'2025-07-01'" + - ")", tmpTableName), ImmutableList.of()); + getExpectedQueryRunner().execute(getSession(), + createTextFileTableSql(tmpTableName, ImmutableList.of()), + ImmutableList.of(BIGINT)); + getExpectedQueryRunner().execute(getSession(), insertTextFileTableSql(tmpTableName), ImmutableList.of(BIGINT)); // created_at is skipped because of the inconsistency in TIMESTAMP columns between Presto and Velox. // https://github.com/facebookincubator/velox/issues/8127 assertQuery(format("SELECT id, name, is_active, score, tags, metrics, properties, flags, nested_struct, price, amount, event_date, ds FROM %s", tmpTableName)); @@ -483,6 +433,94 @@ public void testReadTableWithTextfileFormat() } } + @Test(groups = {"textfile"}) + public void testReadTableWithCustomSerdeTextfile() + { + String tmpTableName = generateRandomTableName(); + List serdeParams = ImmutableList.of( + "textfile_field_delim='|'", + "textfile_escape_delim='\u0001'", + "textfile_collection_delim=';'", + "textfile_mapkey_delim=':'"); + try { + getExpectedQueryRunner().execute(getSession(), + createTextFileTableSql(tmpTableName, serdeParams), + ImmutableList.of(BIGINT)); + getExpectedQueryRunner().execute(getSession(), insertTextFileTableSql(tmpTableName), ImmutableList.of(BIGINT)); + // created_at is skipped because of the inconsistency in TIMESTAMP columns between Presto and Velox. + // https://github.com/facebookincubator/velox/issues/8127 + assertQuery(format("SELECT id, name, is_active, score, tags, metrics, properties, flags, nested_struct, price, amount, event_date, ds FROM %s", tmpTableName)); + } + finally { + dropTableIfExists(tmpTableName); + } + } + + private String createTextFileTableSql(String tableName, List serdeParams) + { + String serde = serdeParams.isEmpty() ? "" : ", " + String.join(", ", serdeParams); + return format( + "CREATE TABLE %s (" + + "id BIGINT," + + "name VARCHAR," + + "is_active BOOLEAN," + + "score DOUBLE," + + "created_at TIMESTAMP," + + "tags ARRAY," + + "metrics ARRAY," + + "properties MAP," + + "flags MAP," + + "nested_struct ROW(sub_id INTEGER, sub_name VARCHAR, sub_scores ARRAY, sub_map MAP)," + + "price DECIMAL(15,2)," + + "amount DECIMAL(21,6)," + + "event_date DATE," + + "ds VARCHAR" + + ") WITH (format = 'TEXTFILE'%s, partitioned_by = ARRAY['ds'])", + tableName, + serde); + } + + private String insertTextFileTableSql(String tableName) + { + return format( + "INSERT INTO %s (" + + "id," + + "name," + + "is_active," + + "score," + + "created_at," + + "tags," + + "metrics," + + "properties," + + "flags," + + "nested_struct," + + "price," + + "amount," + + "event_date," + + "ds" + + ") VALUES (" + + "1001," + + "'Jane Doe'," + + "TRUE," + + "88.5," + + "TIMESTAMP '2025-07-23 10:00:00'," + + "ARRAY['alpha', 'beta', 'gamma']," + + "ARRAY[3.14, 2.71, 1.41]," + + "MAP(ARRAY['color', 'size'], ARRAY['blue', 'large'])," + + "MAP(ARRAY[TINYINT '1', TINYINT '2'], ARRAY[TRUE, FALSE])," + + "ROW(" + + "42," + + "'sub_jane'," + + "ARRAY[REAL '1.1', REAL '2.2', REAL '3.3']," + + "MAP(ARRAY[SMALLINT '10', SMALLINT '20'], ARRAY['foo', 'bar'])" + + ")," + + "DECIMAL '12.34'," + + "CAST('-123456789012345.123456' as DECIMAL(21,6))," + + "DATE '2024-02-29'," + + "'2025-07-01'" + + ")", tableName); + } + @Test public void testOrderBy() { @@ -971,7 +1009,7 @@ public void testStringFunctions() // Reverse assertQuery("SELECT comment, reverse(comment) FROM orders"); - // Normalize + // Normalize, key_sampling_percent. String tmpTableName = generateRandomTableName(); try { getQueryRunner().execute(String.format("CREATE TABLE %s (c0 VARCHAR)", tmpTableName)); @@ -985,6 +1023,17 @@ public void testStringFunctions() assertQuery("SELECT normalize(comment, NFD) FROM nation"); assertQuery(String.format("SELECT normalize(c0) from %s", tmpTableName)); assertQuery(String.format("SELECT normalize(c0, NFKD) from %s", tmpTableName)); + getQueryRunner().execute(String.format("INSERT INTO %s VALUES " + + "(NULL), " + + "('abc'), " + + "('abcdefghskwkjadhwd'), " + + "('001yxzuj'), " + + "('56wfythjhdhvgewuikwemn'), " + + "('special_#@,$|%%/^~?{}+-'), " + + "(' '), " + + "(''), " + + "('Hello World from Velox!')", tmpTableName)); + assertQuery(String.format("SELECT key_sampling_percent(c0) FROM %s", tmpTableName)); } finally { dropTableIfExists(tmpTableName); @@ -1484,8 +1533,8 @@ public void testDecimalRangeFilters() try { // Create a Parquet table with decimal types and test data. - getExpectedQueryRunner().execute(expectedSession, String.format("CREATE TABLE %s (c0 DECIMAL(15,2), c1 DECIMAL(38,2)) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of()); - getExpectedQueryRunner().execute(expectedSession, String.format("INSERT INTO %s VALUES (DECIMAL '0', DECIMAL '0'), (DECIMAL '1.2', DECIMAL '3.4'), (DECIMAL '1000000.12', DECIMAL '28239823232323.57'), (DECIMAL '-542392.89', DECIMAL '-6723982392109.29')", tmpTableName), ImmutableList.of()); + getExpectedQueryRunner().execute(expectedSession, String.format("CREATE TABLE %s (c0 DECIMAL(15,2), c1 DECIMAL(38,2)) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of(BIGINT)); + getExpectedQueryRunner().execute(expectedSession, String.format("INSERT INTO %s VALUES (DECIMAL '0', DECIMAL '0'), (DECIMAL '1.2', DECIMAL '3.4'), (DECIMAL '1000000.12', DECIMAL '28239823232323.57'), (DECIMAL '-542392.89', DECIMAL '-6723982392109.29')", tmpTableName), ImmutableList.of(BIGINT)); String[] queries = { String.format("SELECT * FROM %s WHERE c0 > DECIMAL '1.1' and c1 < DECIMAL '5.2'", tmpTableName), @@ -1538,11 +1587,11 @@ public void testDecimalApproximateAggregates() String tmpTableName = generateRandomTableName(); try { // Create a Parquet table with decimal types and test data. - getExpectedQueryRunner().execute(expectedSession, String.format("CREATE TABLE %s (c0 DECIMAL(15,2), c1 DECIMAL(38,2)) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of()); + getExpectedQueryRunner().execute(expectedSession, String.format("CREATE TABLE %s (c0 DECIMAL(15,2), c1 DECIMAL(38,2)) WITH (format = 'PARQUET')", tmpTableName), ImmutableList.of(BIGINT)); getExpectedQueryRunner().execute(expectedSession, String.format("INSERT INTO %s VALUES (DECIMAL '0', DECIMAL '0'), (DECIMAL '1.2', DECIMAL '3.4'), " + "(DECIMAL '1000000.12', DECIMAL '28239823232323.57'), " + "(DECIMAL '-542392.89', DECIMAL '-6723982392109.29'), (NULL, NULL), " - + "(NULL, DECIMAL'-6723982392109.29'),(DECIMAL'1.2', NULL)", tmpTableName), ImmutableList.of()); + + "(NULL, DECIMAL'-6723982392109.29'),(DECIMAL'1.2', NULL)", tmpTableName), ImmutableList.of(BIGINT)); String[] queries = { String.format("Select approx_distinct(c0) from %s", tmpTableName), String.format("Select approx_distinct(c1) from %s", tmpTableName), diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeJmxMetadataMetrics.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeJmxMetadataMetrics.java new file mode 100644 index 0000000000000..f01cc3dbaed5f --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeJmxMetadataMetrics.java @@ -0,0 +1,205 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativeworker; + +import com.facebook.airlift.discovery.client.Announcer; +import com.facebook.presto.connector.jmx.JmxPlugin; +import com.facebook.presto.metadata.InternalNodeManager; +import com.facebook.presto.metadata.MetadataManagerStats; +import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.inject.Key; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.weakref.jmx.MBeanExporter; + +import javax.management.MBeanServer; + +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createNationWithFormat; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createRegion; +import static com.facebook.presto.server.testing.TestingPrestoServer.updateConnectorIdAnnouncement; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public abstract class AbstractTestNativeJmxMetadataMetrics + extends AbstractTestQueryFramework +{ + private final String storageFormat = "PARQUET"; + protected abstract String getCatalogName(); + protected abstract String getSchemaName(); + protected abstract MBeanServer getMBeanServer(); + protected String getTableName(String table) + { + return String.format("%s.%s.%s", getCatalogName(), getSchemaName(), table); + } + + @Override + protected void createTables() + { + QueryRunner queryRunner = (QueryRunner) getExpectedQueryRunner(); + createRegion(queryRunner); + createNationWithFormat(queryRunner, storageFormat); + } + + @BeforeClass + public void setUp() + { + try { + com.facebook.presto.tests.DistributedQueryRunner distributedQueryRunner = (com.facebook.presto.tests.DistributedQueryRunner) getQueryRunner(); + // Install JMX plugin on coordinator only + distributedQueryRunner.getCoordinator().installPlugin(new JmxPlugin()); + ConnectorId jmxConnectorId = distributedQueryRunner.getCoordinator().createCatalog("jmx", "jmx"); + + // Manually update the coordinator's connectorIds announcement + // This is needed because createCatalog() skips the announcement update for coordinators + // when node-scheduler.include-coordinator is false (see TestingPrestoServer.createCatalog) + Announcer announcer = distributedQueryRunner.getCoordinator().getInstance(Key.get(Announcer.class)); + InternalNodeManager nodeManager = distributedQueryRunner.getCoordinator().getNodeManager(); + updateConnectorIdAnnouncement(announcer, jmxConnectorId, nodeManager); + // Register MetadataManagerStats to the MBeanServer used by JmxPlugin + // This is needed because MetadataManagerStats is registered to TestingMBeanServer (via ServerMainModule) + // but JmxPlugin queries a different MBeanServer (platform or custom) + MetadataManagerStats stats = distributedQueryRunner.getCoordinator().getInstance(Key.get(MetadataManagerStats.class)); + MBeanExporter exporter = new MBeanExporter(getMBeanServer()); + exporter.export("com.facebook.presto.metadata:name=MetadataManagerStats", stats); + } + catch (Exception e) { + throw new RuntimeException("Failed to set up JMX connector announcement and register MetadataManagerStats MBean", e); + } + } + + @Test + public void testMetadataManagerStatsExist() + { + String jmxQuery = "SELECT * FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + MaterializedResult result = computeActual(jmxQuery); + assertTrue(result.getRowCount() > 0); + } + + @Test + public void testMetadataMetricsAfterQueries() + { + // Query to get metadata metrics + String metricsQuery = "SELECT getTableMetadataCalls, listTablesCalls, getColumnHandlesCalls " + + "FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + + MaterializedResult initialResult = computeActual(metricsQuery); + assertEquals(initialResult.getRowCount(), 1); + long initialTableMetadataCalls = (long) initialResult.getMaterializedRows().get(0).getField(0); + long initialColumnHandlesCalls = (long) initialResult.getMaterializedRows().get(0).getField(2); + + assertQuerySucceeds(String.format("SHOW TABLES FROM %s.%s", getCatalogName(), getSchemaName())); + assertQuerySucceeds(String.format("SELECT * FROM %s", getTableName("nation"))); + assertQuerySucceeds(String.format("DESCRIBE %s", getTableName("nation"))); + assertQuerySucceeds(String.format("SHOW COLUMNS FROM %s", getTableName("region"))); + assertQuerySucceeds(String.format("SELECT n.name, r.name FROM %s n JOIN %s r ON n.regionkey = r.regionkey", getTableName("nation"), getTableName("region"))); + + MaterializedResult updatedResult = computeActual(metricsQuery); + assertEquals(updatedResult.getRowCount(), 1); + long updatedTableMetadataCalls = (long) updatedResult.getMaterializedRows().get(0).getField(0); + long updatedColumnHandlesCalls = (long) updatedResult.getMaterializedRows().get(0).getField(2); + + assertTrue(updatedTableMetadataCalls >= initialTableMetadataCalls); + assertTrue(updatedColumnHandlesCalls >= initialColumnHandlesCalls); + } + + @Test + public void testMetadataTimingMetrics() + { + assertQuerySucceeds(String.format("SHOW TABLES FROM %s.%s", getCatalogName(), getSchemaName())); + assertQuerySucceeds(String.format("DESCRIBE %s", getTableName("nation"))); + assertQuerySucceeds(String.format("SELECT * FROM %s LIMIT 10", getTableName("nation"))); + assertQuerySucceeds(String.format("SELECT count(*) FROM %s", getTableName("region"))); + + String timingQuery = "SELECT \"gettablemetadatatime.alltime.count\", \"listtablestime.alltime.count\",\"getcolumnhandlestime.alltime.count\" " + + "FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + + MaterializedResult result = computeActual(timingQuery); + assertEquals(result.getRowCount(), 1); + assertEquals(result.getMaterializedRows().get(0).getFieldCount(), 3); + + double getTableMetadataCount = (double) result.getMaterializedRows().get(0).getField(0); + double listTablesCount = (double) result.getMaterializedRows().get(0).getField(1); + double getColumnHandlesCount = (double) result.getMaterializedRows().get(0).getField(2); + assertTrue(getTableMetadataCount >= 0); + assertTrue(listTablesCount >= 0); + assertTrue(getColumnHandlesCount >= 0); + } + + @Test + public void testMultipleOperationsIncrementMetrics() + { + String countQuery = "SELECT " + + "getTableMetadataCalls + listTablesCalls + getColumnHandlesCalls as total_calls " + + "FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + + MaterializedResult initialResult = computeActual(countQuery); + long initialTotalCalls = (long) initialResult.getMaterializedRows().get(0).getField(0); + + for (int i = 0; i < 5; i++) { + assertQuerySucceeds(String.format("SHOW TABLES FROM %s.%s", getCatalogName(), getSchemaName())); + assertQuerySucceeds(String.format("SELECT * FROM %s WHERE nationkey < 10", getTableName("nation"))); + assertQuerySucceeds(String.format("SELECT count(*) FROM %s", getTableName("region"))); + } + + MaterializedResult updatedResult = computeActual(countQuery); + long updatedTotalCalls = (long) updatedResult.getMaterializedRows().get(0).getField(0); + + assertTrue(updatedTotalCalls > initialTotalCalls); + } + + @Test + public void testMetadataStatsWithComplexQueries() + { + String metricsQuery = "SELECT getTableMetadataCalls, getColumnHandlesCalls " + + "FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + + MaterializedResult initialResult = computeActual(metricsQuery); + long initialTableMetadataCalls = (long) initialResult.getMaterializedRows().get(0).getField(0); + long initialColumnHandlesCalls = (long) initialResult.getMaterializedRows().get(0).getField(1); + + assertQuerySucceeds(String.format("SELECT n.name, r.name FROM %s n JOIN %s r ON n.regionkey = r.regionkey WHERE n.nationkey < 10", + getTableName("nation"), getTableName("region"))); + assertQuerySucceeds(String.format("SELECT * FROM %s WHERE nationkey IN (SELECT nationkey FROM %s WHERE regionkey = 1)", + getTableName("nation"), getTableName("nation"))); + assertQuerySucceeds(String.format("WITH nation_counts AS (SELECT regionkey, count(*) as cnt FROM %s GROUP BY regionkey) " + + "SELECT r.name, nc.cnt FROM %s r JOIN nation_counts nc ON r.regionkey = nc.regionkey", + getTableName("nation"), getTableName("region"))); + + MaterializedResult updatedResult = computeActual(metricsQuery); + long updatedTableMetadataCalls = (long) updatedResult.getMaterializedRows().get(0).getField(0); + long updatedColumnHandlesCalls = (long) updatedResult.getMaterializedRows().get(0).getField(1); + + assertTrue(updatedTableMetadataCalls >= initialTableMetadataCalls); + assertTrue(updatedColumnHandlesCalls >= initialColumnHandlesCalls); + } + + @Test + public void testMetadataStatsNodeColumn() + { + String nodeQuery = "SELECT DISTINCT node FROM jmx.current.\"com.facebook.presto.metadata:name=MetadataManagerStats\""; + MaterializedResult result = computeActual(nodeQuery); + + assertEquals(result.getRowCount(), 1); + + MaterializedResult coordinatorNodes = computeActual("SELECT node_id FROM system.runtime.nodes WHERE coordinator = true"); + assertEquals(coordinatorNodes.getRowCount(), 1); + + String metadataStatsNode = (String) result.getMaterializedRows().get(0).getField(0); + String coordinatorNode = (String) coordinatorNodes.getMaterializedRows().get(0).getField(0); + assertEquals(metadataStatsNode, coordinatorNode); + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeWindowQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeWindowQueries.java index 781b9088c519c..02a1a80ab668b 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeWindowQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/AbstractTestNativeWindowQueries.java @@ -13,6 +13,10 @@ */ package com.facebook.presto.nativeworker; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; +import com.facebook.presto.sql.planner.assertions.PlanMatchPattern; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; @@ -24,6 +28,11 @@ import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createLineitem; import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrders; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyTree; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.limit; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; public abstract class AbstractTestNativeWindowQueries extends AbstractTestQueryFramework @@ -179,6 +188,61 @@ public void testRowNumberWithFilter_2() assertQuery("SELECT * FROM (SELECT row_number() over(partition by orderstatus order by orderkey) rn, * from orders) WHERE rn = 1"); } + private static final PlanMatchPattern topNForFilter = anyTree( + anyNot(FilterNode.class, + node(TopNRowNumberNode.class, + anyTree( + tableScan("orders"))))); + + private static final PlanMatchPattern topNForLimit = anyTree( + limit(10, + anyTree( + node(TopNRowNumberNode.class, + anyTree( + tableScan("orders")))))); + @Test + public void testTopNRowNumber() + { + String sql = "SELECT sum(rn) FROM (SELECT row_number() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders) WHERE rn <= 10"; + assertQuery(sql); + assertPlan(sql, topNForFilter); + + // Cannot test results for this query as they are not guaranteed to be the same due to lack of ORDER BY in LIMIT. + // But adding an ORDER BY would prevent the TopNRowNumber optimization from being applied. + sql = "SELECT sum(rn) FROM (SELECT row_number() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders limit 10)"; + assertPlan(sql, topNForLimit); + } + + @Test + public void testTopNRank() + { + String sql = "SELECT sum(rn) FROM (SELECT rank() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders) WHERE rn <= 10"; + assertQuery(sql); + + if (SystemSessionProperties.isOptimizeTopNRank(getSession())) { + assertPlan(sql, topNForFilter); + // Cannot test results for this query as they are not guaranteed to be the same due to lack of ORDER BY in LIMIT. + // But adding an ORDER BY would prevent the TopNRowNumber optimization from being applied. + sql = "SELECT sum(rn) FROM (SELECT rank() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders limit 10)"; + assertPlan(sql, topNForLimit); + } + } + + @Test + public void testTopNDenseRank() + { + String sql = "SELECT sum(rn) FROM (SELECT dense_rank() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders) WHERE rn <= 10"; + assertQuery(sql); + if (SystemSessionProperties.isOptimizeTopNRank(getSession())) { + assertPlan(sql, topNForFilter); + + // Cannot test results for this query as they are not guaranteed to be the same due to lack of ORDER BY in LIMIT. + // But adding an ORDER BY would prevent the TopNRowNumber optimization from being applied. + sql = "SELECT dense_rank() over(PARTITION BY orderdate ORDER BY totalprice) rn, * from orders limit 10"; + assertPlan(sql, topNForLimit); + } + } + @Test public void testFirstValueOrderKey() { diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 0f6c6fc25122e..f4281157b9123 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -336,11 +336,13 @@ public static class IcebergQueryRunnerBuilder private Map extraConnectorProperties = new HashMap<>(); private Optional remoteFunctionServerUds = Optional.empty(); private boolean addStorageFormatToPath; + private Optional schemaName = Optional.empty(); // External worker launcher is applicable only for the native iceberg query runner, since it depends on other // properties it should be created once all the other query runner configs are set. This variable indicates // whether the query runner returned by builder should use an external worker launcher, it will be true only // for the native query runner and should NOT be explicitly configured by users. private boolean useExternalWorkerLauncher; + private boolean addJmxPlugin; private IcebergQueryRunnerBuilder(QueryRunnerType queryRunnerType) { @@ -362,6 +364,12 @@ private IcebergQueryRunnerBuilder(QueryRunnerType queryRunnerType) } } + public IcebergQueryRunnerBuilder setAddJmxPlugin(boolean addJmxPlugin) + { + this.addJmxPlugin = addJmxPlugin; + return this; + } + public IcebergQueryRunnerBuilder setStorageFormat(String storageFormat) { this.storageFormat = storageFormat; @@ -381,27 +389,64 @@ public IcebergQueryRunnerBuilder setUseThrift(boolean useThrift) return this; } + public IcebergQueryRunnerBuilder setCatalogType(CatalogType catalogType) + { + this.catalogType = catalogType; + return this; + } + + public IcebergQueryRunnerBuilder setExtraProperty(String key, String value) + { + this.extraProperties.put(key, value); + return this; + } + + public IcebergQueryRunnerBuilder setExtraConnectorProperty(String key, String value) + { + this.extraConnectorProperties.put(key, value); + return this; + } + + public IcebergQueryRunnerBuilder setSchemaName(String schemaName) + { + this.schemaName = Optional.of(schemaName); + return this; + } + + public IcebergQueryRunnerBuilder setDataDirectory(Path dataDirectory) + { + this.dataDirectory = dataDirectory; + return this; + } + public QueryRunner build() throws Exception + { + return buildIcebergQueryRunner().getQueryRunner(); + } + + public IcebergQueryRunner buildIcebergQueryRunner() + throws Exception { Optional> externalWorkerLauncher = Optional.empty(); if (this.useExternalWorkerLauncher) { externalWorkerLauncher = getExternalWorkerLauncher("iceberg", "iceberg", serverBinary, cacheMaxSize, remoteFunctionServerUds, Optional.empty(), false, false, false, false, false, false); } - return IcebergQueryRunner.builder() + IcebergQueryRunner.Builder builder = IcebergQueryRunner.builder() .setExtraProperties(extraProperties) .setExtraConnectorProperties(extraConnectorProperties) .setFormat(FileFormat.valueOf(storageFormat)) .setCreateTpchTables(false) - .setAddJmxPlugin(false) + .setAddJmxPlugin(addJmxPlugin) .setNodeCount(OptionalInt.of(workerCount)) .setExternalWorkerLauncher(externalWorkerLauncher) .setAddStorageFormatToPath(addStorageFormatToPath) .setDataDirectory(Optional.of(dataDirectory)) .setTpcdsProperties(getNativeWorkerTpcdsProperties()) - .setCatalogType(catalogType) - .build().getQueryRunner(); + .setCatalogType(catalogType); + schemaName.ifPresent(builder::setSchemaName); + return builder.build(); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java index 314144c322fc5..7a60e7d612d51 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeAsyncDataCacheCleanupAPI.java @@ -67,7 +67,7 @@ protected void createTables() createCustomer(queryRunner); } - @Test(groups = {"async_data_cache"}, enabled = false) + @Test(groups = {"async_data_cache"}) public void testAsyncDataCacheCleanup() throws Exception { Session session = Session.builder(super.getSession()) @@ -151,7 +151,7 @@ private Metrics collectCacheMetrics(Set workerNodes, String endpoi for (InternalNode worker : workerNodes) { Map metrics = fetchScalarLongMetrics(worker.getInternalUri().toString(), endpoint, "GET"); memoryCacheHits += metrics.get("velox_memory_cache_num_hits"); - memoryCacheEntries += metrics.get("velox_memory_cache_num_entries"); + memoryCacheEntries += metrics.get("velox_memory_cache_num_tiny_entries") + metrics.get("velox_memory_cache_num_large_entries"); ssdCacheReadEntries += metrics.get("velox_ssd_cache_read_entries"); ssdCacheWriteEntries += metrics.get("velox_ssd_cache_written_entries"); ssdCacheCachedEntries += metrics.get("velox_ssd_cache_cached_entries"); @@ -193,7 +193,7 @@ private Set getWorkerNodes(DistributedQueryRunner queryRunner) .collect(Collectors.toSet()); } - @Test(groups = {"async_data_cache"}, enabled = false) + @Test(groups = {"async_data_cache"}) public void testAsyncDataCacheCleanupApiFormat() { QueryRunner queryRunner = getQueryRunner(); diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java index 49f6a21fc2080..91ba6483feb4f 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java @@ -28,6 +28,7 @@ import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.AggregationFunctionMetadata; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.RoutineCharacteristics; @@ -149,7 +150,7 @@ private void assertJsonPlan(@Language("SQL") String query, boolean withBuiltInSi transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(queryRunner.getDefaultSession(), transactionSession -> { - String actualPlan = explainer.getJsonPlan(transactionSession, getSqlParser().createStatement(query, createParsingOptions(transactionSession)), ExplainType.Type.LOGICAL, emptyList(), WarningCollector.NOOP, query); + String actualPlan = explainer.getJsonPlan(transactionSession, getSqlParser().createStatement(query, createParsingOptions(transactionSession)), ExplainType.Type.LOGICAL, emptyList(), WarningCollector.NOOP, query, new ViewDefinitionReferences()); Pattern p = Pattern.compile(jsonPlanRegex, Pattern.MULTILINE); if (shouldContainRegex) { if (!p.matcher(actualPlan).find()) { diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergMaterializedViews.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergMaterializedViews.java new file mode 100644 index 0000000000000..5e05c4422f77d --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergMaterializedViews.java @@ -0,0 +1,98 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativeworker; + +import com.facebook.airlift.http.server.testing.TestingHttpServer; +import com.facebook.presto.iceberg.TestIcebergMaterializedViewsBase; +import com.facebook.presto.testing.ExpectedQueryRunner; +import com.facebook.presto.testing.QueryRunner; +import org.assertj.core.util.Files; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.iceberg.CatalogType.REST; +import static com.facebook.presto.iceberg.rest.IcebergRestTestUtil.getRestServer; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.ICEBERG_DEFAULT_STORAGE_FORMAT; +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; + +@Test(singleThreaded = true) +public class TestPrestoNativeIcebergMaterializedViews + extends TestIcebergMaterializedViewsBase +{ + private TestingHttpServer restServer; + private String serverUri; + + @BeforeClass + @Override + public void init() + throws Exception + { + warehouseLocation = Files.newTemporaryFolder(); + + restServer = getRestServer(warehouseLocation.getAbsolutePath()); + restServer.start(); + + serverUri = restServer.getBaseUrl().toString(); + super.init(); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + throws Exception + { + if (restServer != null) { + restServer.stop(); + } + if (warehouseLocation != null) { + deleteRecursively(warehouseLocation.toPath(), ALLOW_INSECURE); + } + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setCatalogType(REST) + .setSchemaName("test_schema") + .setDataDirectory(warehouseLocation.toPath()) + .setExtraConnectorProperty("iceberg.rest.uri", serverUri) + .setExtraProperty("experimental.legacy-materialized-views", "false") + .build(); + } + + @Override + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setCatalogType(REST) + .setSchemaName("test_schema") + .setDataDirectory(warehouseLocation.toPath()) + .setExtraConnectorProperty("iceberg.rest.uri", serverUri) + .setExtraProperty("experimental.legacy-materialized-views", "false") + .build(); + } + + @Override + @Test + public void testMaterializedViewStitchingForTimestamp() + { + // Disabled: Velox does not support timestamp partition filters in stitching scans + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTagsAndBranches.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTagsAndBranches.java new file mode 100644 index 0000000000000..b5e866a84b260 --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeIcebergTagsAndBranches.java @@ -0,0 +1,204 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativeworker; + +import com.facebook.presto.testing.ExpectedQueryRunner; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import org.testng.annotations.Test; + +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.ICEBERG_DEFAULT_STORAGE_FORMAT; +import static org.testng.Assert.assertEquals; + +public class TestPrestoNativeIcebergTagsAndBranches + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(true) + .build(); + } + + @Override + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(true) + .build(); + } + + @Test + public void testQueryBranch() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_branch_dot_notation"); + assertUpdate("CREATE TABLE test_branch_dot_notation (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_branch_dot_notation VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + assertQuerySucceeds("ALTER TABLE test_branch_dot_notation CREATE BRANCH 'audit_branch'"); + assertUpdate("INSERT INTO test_branch_dot_notation VALUES (3, 'Charlie', 300), (4, 'David', 400)", 2); + // Test querying branch using FOR SYSTEM_VERSION AS OF syntax + assertQuery("SELECT count(*) FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'audit_branch'", "VALUES 2"); + assertQuery("SELECT count(*) FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'main'", "VALUES 4"); + // Test querying branch using dot notation syntax + assertQuery("SELECT count(*) FROM \"test_branch_dot_notation.branch_audit_branch\"", "VALUES 2"); + assertQuery("SELECT id, name, value FROM \"test_branch_dot_notation.branch_audit_branch\" ORDER BY id", "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + + // Verify both syntaxes return the same results by comparing actual results + MaterializedResult resultWithForSyntax = computeActual("SELECT id FROM test_branch_dot_notation FOR SYSTEM_VERSION AS OF 'audit_branch' ORDER BY id"); + MaterializedResult resultWithDotNotation = computeActual("SELECT id FROM \"test_branch_dot_notation.branch_audit_branch\" ORDER BY id"); + assertEquals(resultWithForSyntax, resultWithDotNotation); + // Test that main table has all records + assertQuery("SELECT count(*) FROM test_branch_dot_notation", "VALUES 4"); + } + + @Test + public void testQueryTag() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_query_tag"); + assertUpdate("CREATE TABLE test_query_tag (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO test_query_tag VALUES (1, 'Alice', 100), (2, 'Bob', 200)", 2); + assertQuerySucceeds("ALTER TABLE test_query_tag CREATE TAG 'audit_tag'"); + assertUpdate("INSERT INTO test_query_tag VALUES (3, 'Charlie', 300), (4, 'David', 400)", 2); + // Test querying tag using FOR SYSTEM_VERSION AS OF syntax + assertQuery("SELECT count(*) FROM test_query_tag FOR SYSTEM_VERSION AS OF 'audit_tag'", "VALUES 2"); + assertQuery("SELECT count(*) FROM test_query_tag FOR SYSTEM_VERSION AS OF 'main'", "VALUES 4"); + // Verify tag returns correct data + assertQuery("SELECT id, name, value FROM test_query_tag FOR SYSTEM_VERSION AS OF 'audit_tag' ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200)"); + // Test that main table has all records + assertQuery("SELECT count(*) FROM test_query_tag", "VALUES 4"); + } + + @Test + public void testCreateTag() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_create_tag"); + assertUpdate("CREATE TABLE test_create_tag (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_create_tag VALUES (1, 'Alice'), (2, 'Bob')", 2); + // Create tag on current snapshot + assertQuerySucceeds("ALTER TABLE test_create_tag CREATE TAG 'audit_tag'"); + assertUpdate("INSERT INTO test_create_tag VALUES (3, 'Charlie')", 1); + // Verify tag points to the snapshot before the last insert + assertQuery("SELECT count(*) FROM test_create_tag FOR SYSTEM_VERSION AS OF 'audit_tag'", "VALUES 2"); + assertQuery("SELECT count(*) FROM test_create_tag", "VALUES 3"); + // Verify we can query the refs table to see created tags + assertQuery("SELECT count(*) FROM \"test_create_tag$refs\" WHERE type = 'TAG'", "VALUES 1"); + } + + @Test + public void testCreateBranch() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_create_branch"); + assertUpdate("CREATE TABLE test_create_branch (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_create_branch VALUES (1, 'Alice'), (2, 'Bob')", 2); + // Create branch on current snapshot + assertQuerySucceeds("ALTER TABLE test_create_branch CREATE BRANCH 'dev_branch'"); + assertUpdate("INSERT INTO test_create_branch VALUES (3, 'Charlie')", 1); + // Verify branch points to the snapshot before the last insert + assertQuery("SELECT count(*) FROM test_create_branch FOR SYSTEM_VERSION AS OF 'dev_branch'", "VALUES 2"); + assertQuery("SELECT count(*) FROM test_create_branch", "VALUES 3"); + // Verify we can query the refs table to see created branches + assertQuery("SELECT count(*) FROM \"test_create_branch$refs\" WHERE type = 'BRANCH'", "VALUES 2"); // main + dev_branch + } + + @Test + public void testDropTag() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_drop_tag"); + assertUpdate("CREATE TABLE test_drop_tag (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_drop_tag VALUES (1, 'Alice'), (2, 'Bob')", 2); + assertQuerySucceeds("ALTER TABLE test_drop_tag CREATE TAG 'tag1'"); + assertUpdate("INSERT INTO test_drop_tag VALUES (3, 'Charlie')", 1); + assertQuerySucceeds("ALTER TABLE test_drop_tag CREATE TAG 'tag2'"); + // Verify both tags exist + assertQuery("SELECT count(*) FROM \"test_drop_tag$refs\" WHERE type = 'TAG'", "VALUES 2"); + // Drop tag1 + assertQuerySucceeds("ALTER TABLE test_drop_tag DROP TAG 'tag1'"); + assertQuery("SELECT count(*) FROM \"test_drop_tag$refs\" WHERE type = 'TAG'", "VALUES 1"); + // Verify tag1 is dropped and tag2 still exists + assertQueryFails("SELECT count(*) FROM test_drop_tag FOR SYSTEM_VERSION AS OF 'tag1'", ".*tag1.*"); + assertQuery("SELECT count(*) FROM test_drop_tag FOR SYSTEM_VERSION AS OF 'tag2'", "VALUES 3"); + // Drop non-existent tag should fail + assertQueryFails("ALTER TABLE test_drop_tag DROP TAG 'non_existent_tag'", ".*non_existent_tag.*"); + // Drop with IF EXISTS should succeed + assertQuerySucceeds("ALTER TABLE test_drop_tag DROP TAG IF EXISTS 'tag2'"); + assertQuerySucceeds("ALTER TABLE test_drop_tag DROP TAG IF EXISTS 'non_existent_tag'"); + } + + @Test + public void testDropBranch() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_drop_branch"); + assertUpdate("CREATE TABLE test_drop_branch (id BIGINT, name VARCHAR)"); + assertUpdate("INSERT INTO test_drop_branch VALUES (1, 'Alice'), (2, 'Bob')", 2); + assertQuerySucceeds("ALTER TABLE test_drop_branch CREATE BRANCH 'branch1'"); + assertUpdate("INSERT INTO test_drop_branch VALUES (3, 'Charlie')", 1); + assertQuerySucceeds("ALTER TABLE test_drop_branch CREATE BRANCH 'branch2'"); + // Verify both branches exist (main + branch1 + branch2) + assertQuery("SELECT count(*) FROM \"test_drop_branch$refs\" WHERE type = 'BRANCH'", "VALUES 3"); + // Drop branch1 + assertQuerySucceeds("ALTER TABLE test_drop_branch DROP BRANCH 'branch1'"); + assertQuery("SELECT count(*) FROM \"test_drop_branch$refs\" WHERE type = 'BRANCH'", "VALUES 2"); + // Verify branch1 is dropped and branch2 still exists + assertQueryFails("SELECT count(*) FROM test_drop_branch FOR SYSTEM_VERSION AS OF 'branch1'", ".*branch1.*"); + assertQuery("SELECT count(*) FROM test_drop_branch FOR SYSTEM_VERSION AS OF 'branch2'", "VALUES 3"); + // Drop non-existent branch should fail + assertQueryFails("ALTER TABLE test_drop_branch DROP BRANCH 'non_existent_branch'", ".*non_existent_branch.*"); + // Drop with IF EXISTS should succeed + assertQuerySucceeds("ALTER TABLE test_drop_branch DROP BRANCH IF EXISTS 'branch2'"); + assertQuerySucceeds("ALTER TABLE test_drop_branch DROP BRANCH IF EXISTS 'non_existent_branch'"); + } + + // Note: INSERT and TRUNCATE operations on branches are currently supported in Prestissimo. + // UPDATE, DELETE, and MERGE operations on branches using dot notation are not yet implemented + @Test + public void testBranchMutation() + { + assertQuerySucceeds("DROP TABLE IF EXISTS test_branch_mutation"); + assertUpdate("CREATE TABLE test_branch_mutation (id BIGINT, product VARCHAR, price DOUBLE)"); + assertUpdate("INSERT INTO test_branch_mutation VALUES (1, 'Product A', 100.00), (2, 'Product B', 200.00)", 2); + // Create a branch + assertQuerySucceeds("ALTER TABLE test_branch_mutation CREATE BRANCH 'audit_branch'"); + // Insert into branch using dot notation + assertUpdate("INSERT INTO \"test_branch_mutation.branch_audit_branch\" VALUES (3, 'Product C', 300.00)", 1); + assertQuery("SELECT count(*) FROM \"test_branch_mutation.branch_audit_branch\"", "VALUES 3"); + assertQuery("SELECT count(*) FROM test_branch_mutation", "VALUES 2"); // Main branch unchanged + // Verify data in branch + assertQuery("SELECT id, product, price FROM \"test_branch_mutation.branch_audit_branch\" ORDER BY id", + "VALUES (1, 'Product A', 100.00), (2, 'Product B', 200.00), (3, 'Product C', 300.00)"); + // TRUNCATE branch - this is supported + assertQuerySucceeds("TRUNCATE TABLE \"test_branch_mutation.branch_audit_branch\""); + assertQuery("SELECT count(*) FROM \"test_branch_mutation.branch_audit_branch\"", "VALUES 0"); + // Verify main branch is still unchanged after TRUNCATE + assertQuery("SELECT id, product, price FROM test_branch_mutation ORDER BY id", + "VALUES (1, 'Product A', 100.00), (2, 'Product B', 200.00)"); + + // Re-insert data for testing unsupported operations + assertUpdate("INSERT INTO \"test_branch_mutation.branch_audit_branch\" VALUES (1, 'Product A', 100.00), (2, 'Product B', 200.00)", 2); + assertQueryFails("UPDATE \"test_branch_mutation.branch_audit_branch\" SET price = 120.00 WHERE id = 1", "(?s).*"); + assertQueryFails("DELETE FROM \"test_branch_mutation.branch_audit_branch\" WHERE id = 2", "(?s).*"); + assertUpdate("CREATE TABLE test_branch_mutation_source (id BIGINT, product VARCHAR, price DOUBLE)"); + assertUpdate("INSERT INTO test_branch_mutation_source VALUES (1, 'Product A Updated', 150.00)", 1); + assertQueryFails("MERGE INTO \"test_branch_mutation.branch_audit_branch\" t " + + "USING test_branch_mutation_source s ON t.id = s.id " + + "WHEN MATCHED THEN UPDATE SET price = s.price " + + "WHEN NOT MATCHED THEN INSERT (id, product, price) VALUES (s.id, s.product, s.price)", "(?s).*"); + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java index dfa8a91db721b..25dd8616d3ce9 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeTpcdsQueriesOrcUsingThrift.java @@ -15,6 +15,7 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @Test(groups = {"orc"}) @@ -29,6 +30,7 @@ protected QueryRunner createQueryRunner() .setStorageFormat("ORC") .setAddStorageFormatToPath(true) .setUseThrift(true) + .setExtraCoordinatorProperties(ImmutableMap.of("optimizer.optimize-top-n-rank", "true")) .build(); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java index 908fe3ea0c605..91018ebd46ca4 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeWindowQueries.java @@ -15,6 +15,7 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; public class TestPrestoNativeWindowQueries extends AbstractTestNativeWindowQueries @@ -25,6 +26,7 @@ protected QueryRunner createQueryRunner() throws Exception return PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() .setAddStorageFormatToPath(true) .setUseThrift(true) + .setExtraCoordinatorProperties(ImmutableMap.of("optimizer.optimize-top-n-rank", "true")) .build(); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/functions/TestThetaSketchFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/functions/TestThetaSketchFunctions.java new file mode 100644 index 0000000000000..f01f29846dbcf --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/functions/TestThetaSketchFunctions.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativeworker.functions; + +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.ExpectedQueryRunner; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.facebook.presto.tests.DistributedQueryRunner; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class TestThetaSketchFunctions + extends AbstractTestQueryFramework +{ + private String storageFormat; + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = "PARQUET"; + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); + return queryRunner; + } + + @Override + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .build(); + } + + @Override + protected void createTables() + { + QueryRunner queryRunner = (QueryRunner) getExpectedQueryRunner(); + queryRunner.execute("DROP TABLE IF EXISTS test_sketch_theta_functions"); + queryRunner.execute("CREATE TABLE test_sketch_theta_functions (" + + "nullColumn integer, t tinyint, s smallint, i integer," + + "l bigint, r real, d double, v varchar, " + "dt date" + ", ts timestamp," + + "sd decimal(10,4)" + + ", ld decimal(30,8))"); + queryRunner.execute("INSERT INTO test_sketch_theta_functions VALUES(" + + "null,cast(25 as tinyint),cast(250 as smallint),40000,2147483650," + + "214748.3650,2147483650123283628.72323,'sampletesttext'," + + "date'2025-11-12'" + + ",timestamp'2025-11-12 03:47:58',cast(214748.3650 as DECIMAL(10,4))," + + "cast(2147483650123283628123.72323123 as DECIMAL(30,8)))"); + } + + @Test + public void testSketchThetaSummary() + { + assertQuery("SELECT sketch_theta_summary(sketch_theta(nullColumn)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_summary(sketch_theta(i)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(s)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(t)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(l)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_summary(sketch_theta(r)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(d)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_summary(sketch_theta(v)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_summary(sketch_theta(dt)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(ts)) FROM test_sketch_theta_functions"); + +// assertQuery("SELECT sketch_theta_summary(sketch_theta(sd)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_summary(sketch_theta(ld)) FROM test_sketch_theta_functions"); + } + + @Test + public void testSketchThetaEstimate() + { + assertQuery("SELECT sketch_theta_estimate(sketch_theta(nullColumn)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_estimate(sketch_theta(i)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(s)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(t)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(l)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_estimate(sketch_theta(r)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(d)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_estimate(sketch_theta(v)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_estimate(sketch_theta(dt)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(ts)) FROM test_sketch_theta_functions"); + + assertQuery("SELECT sketch_theta_estimate(sketch_theta(sd)) FROM test_sketch_theta_functions"); + assertQuery("SELECT sketch_theta_estimate(sketch_theta(ld)) FROM test_sketch_theta_functions"); + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/iceberg/TestNativeIcebergJmxMetadataMetrics.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/iceberg/TestNativeIcebergJmxMetadataMetrics.java new file mode 100644 index 0000000000000..5792f2d04b218 --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/iceberg/TestNativeIcebergJmxMetadataMetrics.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativeworker.iceberg; + +import com.facebook.presto.iceberg.IcebergQueryRunner; +import com.facebook.presto.nativeworker.AbstractTestNativeJmxMetadataMetrics; +import com.facebook.presto.testing.ExpectedQueryRunner; +import com.facebook.presto.testing.QueryRunner; + +import javax.management.MBeanServer; + +import java.lang.management.ManagementFactory; + +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.ICEBERG_DEFAULT_STORAGE_FORMAT; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.javaIcebergQueryRunnerBuilder; +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.nativeIcebergQueryRunnerBuilder; + +/** + * Test class to verify JMX metadata metrics work correctly with Presto native (C++) workers using Iceberg connector. + */ +public class TestNativeIcebergJmxMetadataMetrics + extends AbstractTestNativeJmxMetadataMetrics +{ + private IcebergQueryRunner icebergQueryRunner; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + icebergQueryRunner = nativeIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(false) + .setAddJmxPlugin(false) + .buildIcebergQueryRunner(); + return icebergQueryRunner.getQueryRunner(); + } + + @Override + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception + { + return javaIcebergQueryRunnerBuilder() + .setStorageFormat(ICEBERG_DEFAULT_STORAGE_FORMAT) + .setAddStorageFormatToPath(false) + .build(); + } + + @Override + protected String getCatalogName() + { + return "iceberg"; + } + + @Override + protected String getSchemaName() + { + return "tpch"; + } + + @Override + protected MBeanServer getMBeanServer() + { + return ManagementFactory.getPlatformMBeanServer(); + } +} diff --git a/presto-native-execution/velox b/presto-native-execution/velox index 59b492a9bce45..3b263a79f879f 160000 --- a/presto-native-execution/velox +++ b/presto-native-execution/velox @@ -1 +1 @@ -Subproject commit 59b492a9bce45f487f24b5cbae7dc845ea3d0827 +Subproject commit 3b263a79f879f363fe59187cab286d358b4ec5a6 diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index 6ddcc91fcb757..6a6298953d51c 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -60,11 +60,6 @@ log-manager - - com.squareup.okhttp3 - okhttp - - com.facebook.presto presto-function-namespace-managers-common @@ -84,6 +79,12 @@ provided + + com.facebook.presto + presto-analyzer + test + + com.facebook.airlift units @@ -260,19 +261,6 @@ compile - - com.squareup.okhttp3 - mockwebserver - test - - - - org.hamcrest - hamcrest-core - - - - com.facebook.presto presto-built-in-worker-function-tools @@ -306,6 +294,7 @@ **/TestNativeSidecar*.java **/TestNativeExpressionInterpreter.java + **/TestNativeExpressionOptimizer.java @@ -343,12 +332,6 @@ org.apache.maven.plugins maven-dependency-plugin - - - org.jetbrains:annotations - com.facebook.airlift.drift:drift-codec:jar diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/expressions/NativeExpressionOptimizer.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/expressions/NativeExpressionOptimizer.java index 59b03497726d1..d5e5702033a16 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/expressions/NativeExpressionOptimizer.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/expressions/NativeExpressionOptimizer.java @@ -16,6 +16,8 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.spi.ConnectorSession; import com.facebook.presto.spi.SourceLocation; +import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.relation.CallExpression; @@ -176,8 +178,13 @@ public Void visitVariableReference(VariableReferenceExpression node, Object cont @Override public Void visitCall(CallExpression node, Object context) { + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(node.getFunctionHandle()); + + // Only constant fold scalar functions, not aggregate or window functions. + boolean isScalarFunction = functionMetadata.getFunctionKind() == FunctionKind.SCALAR; + // If the optimization level is not EVALUATED, then we cannot optimize non-deterministic functions - boolean isDeterministic = functionMetadataManager.getFunctionMetadata(node.getFunctionHandle()).isDeterministic(); + boolean isDeterministic = functionMetadata.isDeterministic(); boolean canBeEvaluated = (optimizationLevel.ordinal() < EVALUATED.ordinal() && isDeterministic) || optimizationLevel.ordinal() == EVALUATED.ordinal(); @@ -189,7 +196,7 @@ public Void visitCall(CallExpression node, Object context) boolean allConstantFoldable = node.getArguments().stream() .allMatch(this::canBeOptimized); - if (canBeEvaluated && allConstantFoldable) { + if (isScalarFunction && canBeEvaluated && allConstantFoldable) { visitNode(node, true); return null; } @@ -338,12 +345,15 @@ public RowExpression visitExpression(RowExpression originalExpression, Void cont @Override public RowExpression visitLambda(LambdaDefinitionExpression lambda, Void context) { - if (canBeReplaced(lambda.getBody())) { + if (canBeReplaced(lambda)) { + RowExpression replacement = resolver.apply(lambda); + // Sidecar optimizes only the body of lambda expression. + RowExpression optimizedBody = ((LambdaDefinitionExpression) replacement).getBody().accept(this, context); return new LambdaDefinitionExpression( lambda.getSourceLocation(), lambda.getArgumentTypes(), lambda.getArguments(), - toRowExpression(lambda.getSourceLocation(), resolver.apply(lambda.getBody()), lambda.getBody().getType())); + toRowExpression(lambda.getSourceLocation(), optimizedBody, optimizedBody.getType())); } return lambda; } diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java index ae17d3f0c2995..ee3aa06856c4e 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java @@ -94,6 +94,15 @@ public String toString() public static class Resolver implements FunctionHandleResolver { + private static final NativeFunctionHandle.Resolver INSTANCE = new Resolver(); + + private Resolver() {} + + public static NativeFunctionHandle.Resolver getInstance() + { + return INSTANCE; + } + @Override public Class getFunctionHandleClass() { diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java index f1b7142a188ee..3c7281022bf9c 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java @@ -63,7 +63,6 @@ import static com.facebook.presto.common.type.TypeSignatureUtils.resolveIntermediateType; import static com.facebook.presto.spi.StandardErrorCode.DUPLICATE_FUNCTION_ERROR; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -136,25 +135,14 @@ private synchronized void populateNamespaceManager(UdfFunctionSignatureMap udfFu public final AggregationFunctionImplementation getAggregateFunctionImplementation(FunctionHandle functionHandle, TypeManager typeManager) { checkCatalog(functionHandle); - checkArgument(functionHandle instanceof SqlFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); - - SqlFunctionHandle sqlFunctionHandle = (SqlFunctionHandle) functionHandle; - if (aggregationImplementationByHandle.containsKey(sqlFunctionHandle)) { - return aggregationImplementationByHandle.get(sqlFunctionHandle); - } - if (functionHandle instanceof NativeFunctionHandle) { - return processNativeFunctionHandle((NativeFunctionHandle) sqlFunctionHandle, typeManager); - } - else { - return processSqlFunctionHandle(sqlFunctionHandle, typeManager); - } + checkArgument(functionHandle instanceof NativeFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); + return processNativeFunctionHandle((NativeFunctionHandle) functionHandle, typeManager); } private AggregationFunctionImplementation processNativeFunctionHandle(NativeFunctionHandle nativeFunctionHandle, TypeManager typeManager) { Signature signature = nativeFunctionHandle.getSignature(); - SqlFunction function = getSqlFunctionFromSignature(signature); - SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + SqlInvokedFunction sqlFunction = getSqlInvokedFunctionFromSignature(signature); checkArgument( sqlFunction.getAggregationMetadata().isPresent(), @@ -176,19 +164,6 @@ private AggregationFunctionImplementation processNativeFunctionHandle(NativeFunc return aggregationImplementationByHandle.get(nativeFunctionHandle); } - private AggregationFunctionImplementation processSqlFunctionHandle(SqlFunctionHandle sqlFunctionHandle, TypeManager typeManager) - { - SqlFunctionId functionId = sqlFunctionHandle.getFunctionId(); - if (!memoizedFunctionsSupplier.get().containsKey(functionId)) { - throw new PrestoException(GENERIC_USER_ERROR, format("Function '%s' is missing from cache", functionId.getId())); - } - - aggregationImplementationByHandle.put( - sqlFunctionHandle, - sqlInvokedFunctionToAggregationImplementation(memoizedFunctionsSupplier.get().get(functionId), typeManager)); - return aggregationImplementationByHandle.get(sqlFunctionHandle); - } - @Override protected Collection fetchFunctionsDirect(QualifiedObjectName functionName) { @@ -206,13 +181,8 @@ protected UserDefinedType fetchUserDefinedTypeDirect(QualifiedObjectName typeNam @Override protected FunctionMetadata fetchFunctionMetadataDirect(SqlFunctionHandle functionHandle) { - if (functionHandle instanceof NativeFunctionHandle) { - return getMetadataFromNativeFunctionHandle(functionHandle); - } - - return fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()).stream() - .filter(function -> function.getRequiredFunctionHandle().equals(functionHandle)) - .map(this::sqlInvokedFunctionToMetadata).collect(onlyElement()); + checkArgument(functionHandle instanceof NativeFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); + return getMetadataFromNativeFunctionHandle(functionHandle); } @Override @@ -262,13 +232,7 @@ public void addUserDefinedType(UserDefinedType userDefinedType) @Override public final FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) { - FunctionHandle functionHandle = super.getFunctionHandle(transactionHandle, signature); - - // only handle generic variadic signatures here , for normal signature we use the AbstractSqlInvokedFunctionNamespaceManager function handle. - if (functionHandle == null) { - return new NativeFunctionHandle(signature); - } - return functionHandle; + return new NativeFunctionHandle(signature); } @VisibleForTesting @@ -288,10 +252,11 @@ private synchronized void createFunction(SqlInvokedFunction function) functions.put(functionId, function.withVersion("1")); } - private SqlFunction getSqlFunctionFromSignature(Signature signature) + private SqlInvokedFunction getSqlInvokedFunctionFromSignature(Signature signature) { try { - return specializedFunctionKeyCache.getUnchecked(signature).getFunction(); + SqlFunction sqlFunction = specializedFunctionKeyCache.getUnchecked(signature).getFunction(); + return (SqlInvokedFunction) sqlFunction; } catch (UncheckedExecutionException e) { throw convertToPrestoException(e, format("Error getting FunctionMetadata for signature: %s", signature)); @@ -302,9 +267,7 @@ private FunctionMetadata getMetadataFromNativeFunctionHandle(SqlFunctionHandle f { NativeFunctionHandle nativeFunctionHandle = (NativeFunctionHandle) functionHandle; Signature signature = nativeFunctionHandle.getSignature(); - SqlFunction function = getSqlFunctionFromSignature(signature); - - SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + SqlInvokedFunction sqlFunction = getSqlInvokedFunctionFromSignature(signature); return new FunctionMetadata( signature.getName(), signature.getArgumentTypes(), @@ -312,13 +275,13 @@ private FunctionMetadata getMetadataFromNativeFunctionHandle(SqlFunctionHandle f .map(Parameter::getName) .collect(toImmutableList()), signature.getReturnType(), - function.getSignature().getKind(), + sqlFunction.getSignature().getKind(), sqlFunction.getRoutineCharacteristics().getLanguage(), getFunctionImplementationType(sqlFunction), - function.isDeterministic(), - function.isCalledOnNullInput(), + sqlFunction.isDeterministic(), + sqlFunction.isCalledOnNullInput(), sqlFunction.getVersion(), - function.getComplexTypeFunctionDescriptor()); + sqlFunction.getComplexTypeFunctionDescriptor()); } private static PrestoException convertToPrestoException(UncheckedExecutionException exception, String failureMessage) diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java index 65af31d56c867..c16ec9569c3d6 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java @@ -21,7 +21,6 @@ import com.facebook.presto.spi.function.FunctionNamespaceManager; import com.facebook.presto.spi.function.FunctionNamespaceManagerContext; import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; -import com.facebook.presto.spi.function.SqlFunctionHandle; import com.google.inject.Injector; import java.util.Map; @@ -37,7 +36,7 @@ public class NativeFunctionNamespaceManagerFactory { public static final String NAME = "native"; - private static final SqlFunctionHandle.Resolver HANDLE_RESOLVER = SqlFunctionHandle.Resolver.getInstance(); + private static final NativeFunctionHandle.Resolver HANDLE_RESOLVER = NativeFunctionHandle.Resolver.getInstance(); @Override public String getName() diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanChecker.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanChecker.java index 60124c4b3c72e..3af7e57c395bd 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanChecker.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanChecker.java @@ -13,11 +13,17 @@ */ package com.facebook.presto.sidecar.nativechecker; +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.http.client.StringResponseHandler.StringResponse; import com.facebook.airlift.json.JsonCodec; import com.facebook.airlift.log.Logger; +import com.facebook.presto.sidecar.ForSidecarInfo; import com.facebook.presto.sidecar.NativeSidecarFailureInfo; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; @@ -33,23 +39,25 @@ import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; +import com.google.inject.Inject; -import java.io.IOException; +import java.net.URI; import java.util.List; import java.util.Map; +import static com.facebook.airlift.http.client.Request.Builder.preparePost; +import static com.facebook.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static com.facebook.airlift.http.client.StringResponseHandler.createStringResponseHandler; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.sidecar.nativechecker.NativePlanCheckerErrorCode.NATIVEPLANCHECKER_CONNECTION_ERROR; import static com.facebook.presto.sidecar.nativechecker.NativePlanCheckerErrorCode.NATIVEPLANCHECKER_UNKNOWN_CONVERSION_FAILURE; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static com.google.common.net.MediaType.JSON_UTF_8; import static io.airlift.slice.Slices.utf8Slice; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; @@ -60,19 +68,19 @@ public final class NativePlanChecker implements PlanChecker { private static final Logger LOG = Logger.get(NativePlanChecker.class); - private static final MediaType JSON_CONTENT_TYPE = MediaType.parse("application/json; charset=utf-8"); private static final JsonCodec PLAN_CONVERSION_RESPONSE_JSON_CODEC = JsonCodec.jsonCodec(PlanConversionResponse.class); public static final String PLAN_CONVERSION_ENDPOINT = "/v1/velox/plan"; private final NodeManager nodeManager; private final JsonCodec planFragmentJsonCodec; - private final OkHttpClient httpClient; + private final HttpClient httpClient; - public NativePlanChecker(NodeManager nodeManager, JsonCodec planFragmentJsonCodec) + @Inject + public NativePlanChecker(NodeManager nodeManager, JsonCodec planFragmentJsonCodec, @ForSidecarInfo HttpClient httpClient) { this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.planFragmentJsonCodec = requireNonNull(planFragmentJsonCodec, "planFragmentJsonCodec is null"); - this.httpClient = new OkHttpClient.Builder().build(); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); } @Override @@ -123,45 +131,54 @@ private void runValidation(SimplePlanFragment planFragment) { LOG.debug("Starting native plan validation [fragment: %s, root: %s]", planFragment.getId(), planFragment.getRoot().getId()); String requestBodyJson = planFragmentJsonCodec.toJson(planFragment); - final Request request = buildRequest(requestBodyJson); - try (Response response = httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) { + try { + StringResponse response = httpClient.execute(getSidecarRequest(requestBodyJson), createStringResponseHandler()); + if (response.getStatusCode() != 200) { NativeSidecarFailureInfo failure = processResponseFailure(response); String message = String.format("Error from native plan checker: %s", firstNonNull(failure.getMessage(), "Internal error")); throw new PrestoException(failure::getErrorCode, message, failure.toException()); } } - catch (final IOException e) { - throw new PrestoException(NATIVEPLANCHECKER_CONNECTION_ERROR, "I/O error getting native plan checker response", e); + catch (RuntimeException e) { + if (e instanceof PrestoException) { + throw e; + } + throw new PrestoException(NATIVEPLANCHECKER_CONNECTION_ERROR, "Error getting native plan checker response", e); } finally { LOG.debug("Native plan validation complete [fragment: %s, root: %s]", planFragment.getId(), planFragment.getRoot().getId()); } } - private Request buildRequest(String requestBodyJson) + private Request getSidecarRequest(String requestBodyJson) { - // Use native sidecar plan conversion endpoint to validate - String planConversionUrl = nodeManager.getSidecarNode().getHttpUri().toString() + PLAN_CONVERSION_ENDPOINT; - - Request.Builder builder = new Request.Builder() - .url(planConversionUrl) - .addHeader("CONTENT_TYPE", "APPLICATION_JSON") - .post(RequestBody.create(JSON_CONTENT_TYPE, requestBodyJson)); + return preparePost() + .setUri(getSidecarLocation()) + .setHeader(CONTENT_TYPE, JSON_UTF_8.toString()) + .setBodyGenerator(createStaticBodyGenerator(requestBodyJson, UTF_8)) + .build(); + } - return builder.build(); + private URI getSidecarLocation() + { + Node sidecarNode = nodeManager.getSidecarNode(); + return HttpUriBuilder + .uriBuilderFrom(sidecarNode.getHttpUri()) + .appendPath(PLAN_CONVERSION_ENDPOINT) + .build(); } - private NativeSidecarFailureInfo processResponseFailure(Response response) throws IOException + private NativeSidecarFailureInfo processResponseFailure(StringResponse response) { - if (response.body() == null) { - throw new PrestoException(NATIVEPLANCHECKER_UNKNOWN_CONVERSION_FAILURE, "Error response without failure from native plan checker with code: " + response.code()); + String responseBody = response.getBody(); + if (responseBody == null || responseBody.isEmpty()) { + throw new PrestoException(NATIVEPLANCHECKER_UNKNOWN_CONVERSION_FAILURE, "Error response without failure from native plan checker with code: " + response.getStatusCode()); } - PlanConversionResponse planConversionResponse = PLAN_CONVERSION_RESPONSE_JSON_CODEC.fromJson(response.body().bytes()); + PlanConversionResponse planConversionResponse = PLAN_CONVERSION_RESPONSE_JSON_CODEC.fromJson(responseBody); if (planConversionResponse.getFailures().isEmpty()) { - throw new PrestoException(NATIVEPLANCHECKER_UNKNOWN_CONVERSION_FAILURE, "Error response without failure from native plan checker with code: " + response.code()); + throw new PrestoException(NATIVEPLANCHECKER_UNKNOWN_CONVERSION_FAILURE, "Error response without failure from native plan checker with code: " + response.getStatusCode()); } return planConversionResponse.getFailures().get(0); diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerModule.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerModule.java new file mode 100644 index 0000000000000..f1eca345ff3b8 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerModule.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.nativechecker; + +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.plan.PlanCheckerProvider; +import com.facebook.presto.spi.plan.SimplePlanFragment; +import com.facebook.presto.spi.plan.SimplePlanFragmentSerde; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; + +public class NativePlanCheckerModule + implements Module +{ + private final NodeManager nodeManager; + private final SimplePlanFragmentSerde simplePlanFragmentSerde; + + public NativePlanCheckerModule(NodeManager nodeManager, SimplePlanFragmentSerde simplePlanFragmentSerde) + { + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.simplePlanFragmentSerde = requireNonNull(simplePlanFragmentSerde, "simplePlanFragmentSerde is null"); + } + + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(NativePlanCheckerConfig.class, NativePlanCheckerConfig.CONFIG_PREFIX); + binder.install(new JsonModule()); + binder.bind(NodeManager.class).toInstance(nodeManager); + binder.bind(SimplePlanFragmentSerde.class).toInstance(simplePlanFragmentSerde); + jsonBinder(binder).addSerializerBinding(SimplePlanFragment.class).to(SimplePlanFragmentSerializer.class).in(Scopes.SINGLETON); + jsonCodecBinder(binder).bindJsonCodec(SimplePlanFragment.class); + binder.bind(NativePlanChecker.class).in(Scopes.SINGLETON); + binder.bind(PlanCheckerProvider.class).to(NativePlanCheckerProvider.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProvider.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProvider.java index aeba3c77f4939..5d89a064f16a3 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProvider.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProvider.java @@ -14,11 +14,8 @@ package com.facebook.presto.sidecar.nativechecker; -import com.facebook.airlift.json.JsonCodec; -import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.plan.PlanChecker; import com.facebook.presto.spi.plan.PlanCheckerProvider; -import com.facebook.presto.spi.plan.SimplePlanFragment; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; @@ -29,23 +26,20 @@ public class NativePlanCheckerProvider implements PlanCheckerProvider { - private final NodeManager nodeManager; - private final JsonCodec planFragmentJsonCodec; private final NativePlanCheckerConfig config; + private final NativePlanChecker planChecker; @Inject - public NativePlanCheckerProvider(NodeManager nodeManager, JsonCodec planFragmentJsonCodec, NativePlanCheckerConfig config) + public NativePlanCheckerProvider(NativePlanCheckerConfig config, NativePlanChecker planChecker) { - this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); - this.planFragmentJsonCodec = requireNonNull(planFragmentJsonCodec, "planFragmentJsonCodec is null"); this.config = requireNonNull(config, "config is null"); + this.planChecker = requireNonNull(planChecker, "planChecker is null"); } @Override public List getFragmentPlanCheckers() { return config.isPlanValidationEnabled() ? - ImmutableList.of(new NativePlanChecker(nodeManager, planFragmentJsonCodec)) : - ImmutableList.of(); + ImmutableList.of(planChecker) : ImmutableList.of(); } } diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProviderFactory.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProviderFactory.java index ce41294d40a1d..0cf2e30cb83c2 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProviderFactory.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/NativePlanCheckerProviderFactory.java @@ -14,22 +14,15 @@ package com.facebook.presto.sidecar.nativechecker; import com.facebook.airlift.bootstrap.Bootstrap; -import com.facebook.airlift.json.JsonModule; -import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.sidecar.NativeSidecarCommunicationModule; import com.facebook.presto.spi.classloader.ThreadContextClassLoader; import com.facebook.presto.spi.plan.PlanCheckerProvider; import com.facebook.presto.spi.plan.PlanCheckerProviderContext; import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; -import com.facebook.presto.spi.plan.SimplePlanFragment; -import com.facebook.presto.spi.plan.SimplePlanFragmentSerde; import com.google.inject.Injector; -import com.google.inject.Scopes; import java.util.Map; -import static com.facebook.airlift.configuration.ConfigBinder.configBinder; -import static com.facebook.airlift.json.JsonBinder.jsonBinder; -import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static java.util.Objects.requireNonNull; public class NativePlanCheckerProviderFactory @@ -49,19 +42,12 @@ public String getName() } @Override - public PlanCheckerProvider create(Map properties, PlanCheckerProviderContext planCheckerProviderContext) + public PlanCheckerProvider create(Map properties, PlanCheckerProviderContext context) { try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { Bootstrap app = new Bootstrap( - binder -> { - configBinder(binder).bindConfig(NativePlanCheckerConfig.class, NativePlanCheckerConfig.CONFIG_PREFIX); - binder.install(new JsonModule()); - binder.bind(NodeManager.class).toInstance(planCheckerProviderContext.getNodeManager()); - binder.bind(SimplePlanFragmentSerde.class).toInstance(planCheckerProviderContext.getSimplePlanFragmentSerde()); - jsonBinder(binder).addSerializerBinding(SimplePlanFragment.class).to(SimplePlanFragmentSerializer.class).in(Scopes.SINGLETON); - jsonCodecBinder(binder).bindJsonCodec(SimplePlanFragment.class); - binder.bind(PlanCheckerProvider.class).to(NativePlanCheckerProvider.class).in(Scopes.SINGLETON); - }); + new NativePlanCheckerModule(context.getNodeManager(), context.getSimplePlanFragmentSerde()), + new NativeSidecarCommunicationModule()); Injector injector = app .noStrictConfig() diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java index 54d39931acff3..7f6098b223ea4 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java @@ -53,6 +53,7 @@ import static com.facebook.presto.common.type.StandardTypes.IPADDRESS; import static com.facebook.presto.common.type.StandardTypes.IPPREFIX; import static com.facebook.presto.common.type.StandardTypes.JSON; +import static com.facebook.presto.common.type.StandardTypes.K_HYPER_LOG_LOG; import static com.facebook.presto.common.type.StandardTypes.MAP; import static com.facebook.presto.common.type.StandardTypes.P4_HYPER_LOG_LOG; import static com.facebook.presto.common.type.StandardTypes.QDIGEST; @@ -92,6 +93,7 @@ public class NativeTypeManager DOUBLE, SMALLINT, HYPER_LOG_LOG, + K_HYPER_LOG_LOG, P4_HYPER_LOG_LOG, JSON, TIME_WITH_TIME_ZONE, diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java index 9452fec3aa32e..d9c94fe6a2abf 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java @@ -19,6 +19,7 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.scalar.sql.NativeSqlInvokedFunctionsPlugin; import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; +import com.facebook.presto.sidecar.expressions.NativeExpressionOptimizerFactory; import com.facebook.presto.sidecar.functionNamespace.FunctionDefinitionProvider; import com.facebook.presto.sidecar.functionNamespace.NativeFunctionDefinitionProvider; import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManager; @@ -49,6 +50,7 @@ import java.util.stream.Collectors; import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE; +import static com.facebook.presto.SystemSessionProperties.EXPRESSION_OPTIMIZER_NAME; import static com.facebook.presto.SystemSessionProperties.INLINE_SQL_FUNCTIONS; import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED; import static com.facebook.presto.SystemSessionProperties.REMOVE_MAP_CAST; @@ -75,7 +77,7 @@ public class TestNativeSidecarPlugin private static final String REGEX_FUNCTION_NAMESPACE = "native.default.*"; private static final String REGEX_SESSION_NAMESPACE = "Native Execution only.*"; private static final long SIDECAR_HTTP_CLIENT_MAX_CONTENT_SIZE_MB = 128; - private static final int INLINED_SQL_FUNCTIONS_COUNT = 7; + private static final int INLINED_SQL_FUNCTIONS_COUNT = 6; @Override protected void createTables() @@ -127,6 +129,7 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner) "sidecar.http-client.max-content-length", SIDECAR_HTTP_CLIENT_MAX_CONTENT_SIZE_MB + "MB")); queryRunner.loadTypeManager(NativeTypeManagerFactory.NAME); queryRunner.loadPlanCheckerProviderManager("native", ImmutableMap.of()); + queryRunner.getExpressionManager().loadExpressionOptimizerFactory(NativeExpressionOptimizerFactory.NAME, "native", ImmutableMap.of()); queryRunner.installPlugin(new NativeSqlInvokedFunctionsPlugin()); } @@ -563,6 +566,7 @@ public void testOverriddenInlinedSqlInvokedFunctions() assertQuery("SELECT any_values_match(MAP(ARRAY[orderkey], ARRAY[totalprice]), k -> abs(k) > 20) from orders"); assertQuery("SELECT no_values_match(MAP(ARRAY[orderkey], ARRAY[comment]), k -> length(k) > 2) from orders"); assertQuery("SELECT no_keys_match(MAP(ARRAY[comment], ARRAY[custkey]), k -> ends_with(k, 'a')) from orders"); + assertQuery("select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey"); } @Test @@ -621,10 +625,6 @@ public void testNonOverriddenInlinedSqlInvokedFunctionsWhenConfigDisabled() "SELECT map_top_n_keys(MAP(ARRAY[regionkey], ARRAY[nationkey]), 5, (x, y) -> if (x < y, cast(1 as bigint), if (x > y, cast(-1 as bigint), cast(0 as bigint)))) from nation", ".*Scalar function native\\.default\\.map_top_n_keys not registered with arguments.*", true); - - assertQueryFails(session, - "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey", - ".*Scalar function name not registered: native.default.key_sampling_percent.*"); } @Test @@ -646,6 +646,91 @@ public void testP4HyperLogLogWithApproxSet() } } + // TODO: Remove this test once all remaining failures + // are addressed using the native expression optimizer, and it is enabled everywhere. + + @Test + public void testNativeExpressionOptimizer() + { + Session session = Session.builder(getSession()) + .setSystemProperty(EXPRESSION_OPTIMIZER_NAME, "native") + .build(); + + // When using the native expression optimizer, the resolved optimized expression may contain a FunctionHandle. It is important that the correct type of function handle is constructed. + // Previously, the optimizer returned a BuiltInFunctionHandle, which caused errors such as "function not registered/found" because the function name was prefixed with `native.default` + // and resolution was attempted using the BuiltInFunctionNamespaceManager. + // With VeloxToPrestoExpr now returning a NativeFunctionHandle for native (C++) functions, we ensure that the NativeFunctionNamespaceManager is used to resolve native (C++) functions correctly. + + // By adding these test cases, we verify that the reconstructed FunctionHandle is now a NativeFunctionHandle and is resolved correctly. + assertQuerySucceeds(session, "SELECT array_sort(ARRAY[-3, 2, -100, 5], x -> IF(x = 5, NULL, abs(x)))"); + assertQuerySucceeds(session, "SELECT array_sort_desc(ARRAY[-25, 20000, -17, 3672], x -> IF(x = 5, NULL, abs(x)))"); + + // aggregates + assertUpdate(session, "ANALYZE region", 5); + assertQuerySucceeds(session, "select count(*) from orders"); + assertQuerySucceeds(session, "select count(regionkey) from region"); + assertQuerySucceeds(session, "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey"); + // no-op + assertQuerySucceeds(session, "SELECT IF(1 = 1, count(*), count(*)) FROM orders"); + assertQuerySucceeds(session, "SELECT CASE WHEN true THEN count(*) ELSE count(*) END FROM ORDERS"); + + // windows + assertQuerySucceeds(session, "SELECT * FROM (SELECT row_number() over(partition by orderstatus order by orderkey, orderstatus) rn, * from orders) WHERE rn = 1"); + assertQuerySucceeds(session, "WITH t AS (SELECT linenumber, row_number() over (partition by linenumber order by linenumber) as rn FROM lineitem) SELECT * FROM t WHERE rn = 1"); + assertQuerySucceeds(session, "SELECT row_number() OVER (PARTITION BY orderdate ORDER BY orderdate) FROM orders"); + + // IN expressions + assertQuerySucceeds(session, "SELECT table_name FROM information_schema.columns WHERE table_name IN ('nation', 'region')"); + assertQuerySucceeds(session, "SELECT name FROM nation WHERE nationkey NOT IN (1, 2, 3, 4, 5, 10, 11, 12, 13)"); + assertQuerySucceeds(session, "SELECT orderkey FROM lineitem WHERE shipmode IN ('TRUCK', 'FOB', 'RAIL')"); + assertQuerySucceeds(session, "SELECT table_name, COALESCE(abs(ordinal_position), 0) as abs_pos FROM information_schema.columns WHERE table_catalog = 'hive' AND table_name IN ('nation', 'region') ORDER BY table_name, ordinal_position"); + assertQuerySucceeds(session, "SELECT table_name, ordinal_position FROM information_schema.columns WHERE abs(ordinal_position) IN (1, 2, 3) AND table_catalog = 'hive' AND table_name != 'roles' ORDER BY table_name, ordinal_position"); + assertQuerySucceeds(session, "select lower(table_name) from information_schema.tables where table_name = 'lineitem' or table_name = 'LINEITEM'"); + + // Test dereference expression. + assertQuerySucceeds(session, + "select cast(row(row(row(random(10), if(random(10) >= 0, 2)), random(10)), random(100)) AS row(x row(y row(a int, b int), c int), d int))[1][1][2]"); + assertQuerySucceeds(session, + "select cast(row(row(row(random(10), if(random(10) < 0, 2)), random(10)), random(100)) AS row(x row(y row(a int, b int), c int), d int))[1][2]"); + assertQuerySucceeds(session, + "select cast(row(row(null, random(10)), random(100)) AS row(x row(y row(a int, b int), c int), d int))[1][1][1]"); + assertQuerySucceeds(session, + "select cast(row(row(null, if(random(100) >= 0, 4)), random(10)) AS row(x row(y row(a int, b int), c int), d int))[2]"); + + // Test dereference expression with SQL invoked function, array_least_frequent. + assertQuerySucceeds(session, "SELECT array_least_frequent(array_agg(orderkey)) from orders"); + assertQuerySucceeds(session, "SELECT array_least_frequent(array_agg(nationkey)) from nation"); + } + + @Test + public void testMergeKHyperLogLog() + { + assertQuery( + "select cardinality(merge(khll)), uniqueness_distribution(merge(khll)) " + + "from (" + + " select k1, k2, khyperloglog_agg(v1, v2) khll " + + " from (values (1, 1, 2, 3), (1, 1, 4, 0), (1, 2, 90, 20), (1, 2, 87, 1), " + + " (2, 1, 11, 30), (2, 1, 11, 11), (2, 2, 9, 1), (2, 2, 87, 2)) t(k1, k2, v1, v2) " + + " group by k1, k2" + + ")"); + + // Test merge(KHyperLogLog) when there are no rows. + assertQuery( + "select cardinality(merge(khll)), uniqueness_distribution(merge(khll)) " + + "from (" + + " select khyperloglog_agg(v1, v2) khll " + + " from (values (1, 1, 2, 3)) t(k1, k2, v1, v2) " + + " where 1 = 0" + + ")"); + + // Verify merge(KHyperLogLog) handles null states correctly. + assertQuery( + "select cardinality(merge(khll)), uniqueness_distribution(merge(khll)) " + + "from (" + + " select CAST(null AS KHYPERLOGLOG) khll" + + ")"); + } + private String generateRandomTableName() { String tableName = "tmp_presto_" + UUID.randomUUID().toString().replace("-", ""); diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestPlanCheckerProvider.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestPlanCheckerProvider.java index 2b959bce62c8e..5e5db209556fc 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestPlanCheckerProvider.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestPlanCheckerProvider.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.sidecar; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.testing.TestingHttpClient; import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.common.ErrorCode; import com.facebook.presto.sidecar.nativechecker.NativePlanChecker; @@ -38,17 +40,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; +import com.google.common.net.MediaType; import org.testng.annotations.Test; -import java.io.IOException; import java.net.URI; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; +import static com.facebook.airlift.http.client.testing.TestingResponse.mockResponse; +import static com.facebook.presto.sidecar.nativechecker.NativePlanChecker.PLAN_CONVERSION_ENDPOINT; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.expectThrows; @@ -63,7 +65,11 @@ public void testGetPlanChecker() { NativePlanCheckerConfig config = new NativePlanCheckerConfig(); assertTrue(config.isPlanValidationEnabled()); - NativePlanCheckerProvider provider = new NativePlanCheckerProvider(new TestingNodeManager(URI.create("localhost")), PLAN_FRAGMENT_JSON_CODEC, config); + TestingHttpClient client = new TestingHttpClient( + request -> + mockResponse(HttpStatus.OK, MediaType.JSON_UTF_8, "")); + NativePlanChecker planChecker = new NativePlanChecker(new TestingNodeManager(URI.create("localhost")), PLAN_FRAGMENT_JSON_CODEC, client); + NativePlanCheckerProvider provider = new NativePlanCheckerProvider(config, planChecker); assertTrue(provider.getIntermediatePlanCheckers().isEmpty()); assertTrue(provider.getFinalPlanCheckers().isEmpty()); assertEquals(provider.getFragmentPlanCheckers().size(), 1); @@ -71,7 +77,6 @@ public void testGetPlanChecker() @Test public void testNativePlanMockValidate() - throws IOException { TestingPlanNode root = new TestingPlanNode(); ConnectorPartitioningHandle connectorPartitioningHandle = new TestingConnectorPartitioningHandle(); @@ -79,26 +84,34 @@ public void testNativePlanMockValidate() PartitioningScheme partitioningScheme = new PartitioningScheme(new Partitioning(handle, ImmutableList.of()), ImmutableList.of()); SimplePlanFragment fragment = new SimplePlanFragment(new PlanFragmentId(1), root, ImmutableSet.of(), handle, ImmutableList.of(), partitioningScheme, StageExecutionDescriptor.ungroupedExecution(), false); - try (MockWebServer server = new MockWebServer()) { - server.start(); - TestingNodeManager nodeManager = new TestingNodeManager(server.url(NativePlanChecker.PLAN_CONVERSION_ENDPOINT).uri()); - NativePlanChecker checker = new NativePlanChecker(nodeManager, PLAN_FRAGMENT_JSON_CODEC); - - PlanConversionResponse responseOk = new PlanConversionResponse(ImmutableList.of()); - String responseOkString = PLAN_CONVERSION_RESPONSE_JSON_CODEC.toJson(responseOk); - server.enqueue(new MockResponse().setBody(responseOkString)); - checker.validateFragment(fragment, null, null); - - String errorMessage = "native conversion error"; - ErrorCode errorCode = StandardErrorCode.NOT_SUPPORTED.toErrorCode(); - PlanConversionResponse responseError = new PlanConversionResponse(ImmutableList.of(new NativeSidecarFailureInfo("MockError", errorMessage, null, ImmutableList.of(), ImmutableList.of(), errorCode))); - String responseErrorString = PLAN_CONVERSION_RESPONSE_JSON_CODEC.toJson(responseError); - server.enqueue(new MockResponse().setResponseCode(500).setBody(responseErrorString)); - PrestoException error = expectThrows(PrestoException.class, - () -> checker.validateFragment(fragment, null, null)); - assertEquals(error.getErrorCode(), errorCode); - assertTrue(error.getMessage().contains(errorMessage)); - } + // set ok response + PlanConversionResponse responseOk = new PlanConversionResponse(ImmutableList.of()); + NativePlanChecker okPlanchecker = createChecker(responseOk, HttpStatus.OK); + okPlanchecker.validateFragment(fragment, null, null); + + // set error response + String errorMessage = "native conversion error"; + ErrorCode errorCode = StandardErrorCode.NOT_SUPPORTED.toErrorCode(); + PlanConversionResponse responseError = new PlanConversionResponse(ImmutableList.of(new NativeSidecarFailureInfo("MockError", errorMessage, null, ImmutableList.of(), ImmutableList.of(), errorCode))); + NativePlanChecker errorPlanChecker = createChecker(responseError, HttpStatus.BAD_REQUEST); + PrestoException error = expectThrows(PrestoException.class, + () -> errorPlanChecker.validateFragment(fragment, null, null)); + assertEquals(error.getErrorCode(), errorCode); + assertTrue(error.getMessage().contains(errorMessage)); + } + + private NativePlanChecker createChecker(PlanConversionResponse response, HttpStatus status) + { + TestingHttpClient client = new TestingHttpClient( + request -> mockResponse( + status, + MediaType.JSON_UTF_8, + PLAN_CONVERSION_RESPONSE_JSON_CODEC.toJson(response))); + + return new NativePlanChecker( + new TestingNodeManager(URI.create("http://localhost" + PLAN_CONVERSION_ENDPOINT)), + PLAN_FRAGMENT_JSON_CODEC, + client); } public static class TestingConnectorPartitioningHandle diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/expressions/TestNativeExpressionOptimizer.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/expressions/TestNativeExpressionOptimizer.java new file mode 100644 index 0000000000000..1df9d84e123c7 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/expressions/TestNativeExpressionOptimizer.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.expressions; + +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.operator.scalar.FunctionAssertions; +import com.facebook.presto.sidecar.NativeSidecarPluginQueryRunner; +import com.facebook.presto.spi.relation.ExpressionOptimizer; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.sql.TestingRowExpressionTranslator; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.tests.DistributedQueryRunner; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.util.function.Function; + +import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; +import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.metadata.MetadataManager.createTestMetadataManager; +import static com.facebook.presto.sidecar.expressions.NativeExpressionOptimizerFactory.NAME; +import static com.facebook.presto.sql.expressions.AbstractTestExpressionInterpreter.SYMBOL_TYPES; +import static com.facebook.presto.sql.expressions.AbstractTestExpressionInterpreter.assertRowExpressionEvaluationEquals; + +public class TestNativeExpressionOptimizer +{ + private final DistributedQueryRunner queryRunner; + private final MetadataManager metadata; + private final TestingRowExpressionTranslator translator; + private final NativeExpressionOptimizer expressionOptimizer; + + public TestNativeExpressionOptimizer() + throws Exception + { + this.queryRunner = NativeSidecarPluginQueryRunner.getQueryRunner(); + FunctionAndTypeManager functionAndTypeManager = queryRunner.getCoordinator().getFunctionAndTypeManager(); + this.metadata = createTestMetadataManager(functionAndTypeManager); + this.translator = new TestingRowExpressionTranslator(metadata); + this.expressionOptimizer = (NativeExpressionOptimizer) queryRunner.getCoordinator() + .getExpressionManager() + .getExpressionOptimizer(NAME); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + { + closeAllRuntimeException(queryRunner); + } + + @Test + public void testLambdaBodyConstantFolding() + { + // Simple lambda constant folding. + assertOptimizedEquals( + "transform(ARRAY[unbound_long, unbound_long2], x -> 1 + 1)", + "transform(ARRAY[unbound_long, unbound_long2], x -> 2)"); + assertOptimizedEquals( + "transform(ARRAY[unbound_long, unbound_long2], x -> cast('123' AS integer))", + "transform(ARRAY[unbound_long, unbound_long2], x -> 123)"); + assertOptimizedEquals( + "transform(ARRAY[unbound_long, unbound_long2], x -> cast(json_parse('[1, 2]') AS ARRAY)[1] + 1)", + "transform(ARRAY[unbound_long, unbound_long2], x -> 2)"); + + // Nested lambda constant folding. + assertOptimizedEquals( + "transform(ARRAY[unbound_long, unbound_long2], x -> transform(ARRAY[1, 2], y -> 1 + 1))", + "transform(ARRAY[unbound_long, unbound_long2], x -> transform(ARRAY[1, 2], y -> 2))"); + // Multiple lambda occurrences constant folding. + assertOptimizedEquals( + "filter(transform(ARRAY[unbound_long, unbound_long2], x -> 1 + 1), x -> true and false)", + "filter(transform(ARRAY[unbound_long, unbound_long2], x -> 2), x -> false)"); + } + + private void assertOptimizedEquals(@Language("SQL") String actual, @Language("SQL") String expected) + { + RowExpression optimizedActual = optimize(actual, ExpressionOptimizer.Level.OPTIMIZED); + RowExpression optimizedExpected = optimize(expected, ExpressionOptimizer.Level.OPTIMIZED); + assertRowExpressionEvaluationEquals(optimizedActual, optimizedExpected); + } + + private RowExpression optimize(@Language("SQL") String expression, ExpressionOptimizer.Level level) + { + RowExpression parsedExpression = sqlToRowExpression(expression); + Function variableResolver = variable -> null; + return expressionOptimizer.optimize(parsedExpression, level, TEST_SESSION.toConnectorSession(), variableResolver); + } + + private RowExpression sqlToRowExpression(String expression) + { + Expression parsedExpression = FunctionAssertions.createExpression(expression, metadata, SYMBOL_TYPES); + return translator.translate(parsedExpression, SYMBOL_TYPES); + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestTextReaderWithTpcdsQueriesUsingThrift.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestTextReaderWithTpcdsQueriesUsingThrift.java index 25686e175581c..b317387ffdd94 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestTextReaderWithTpcdsQueriesUsingThrift.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestTextReaderWithTpcdsQueriesUsingThrift.java @@ -17,6 +17,7 @@ import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; public class TestTextReaderWithTpcdsQueriesUsingThrift extends AbstractTestNativeTpcdsQueries @@ -31,6 +32,7 @@ protected QueryRunner createQueryRunner() .setStorageFormat(TEXTFILE) .setAddStorageFormatToPath(true) .setUseThrift(true) + .setExtraCoordinatorProperties(ImmutableMap.of("optimizer.optimize-top-n-rank", "true")) .build(); } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java index 6c99067f3f399..3952d8e82ef1a 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java @@ -80,6 +80,34 @@ public void assertNotSupported(String projection, @Language("RegExp") String mes } } + @Override + public void assertInvalidFunction(String projection, @Language("RegExp") String message) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + try { + computeActual(rewritten); + fail("expected exception"); + } + catch (RuntimeException ex) { + assertExceptionMessage(rewritten, ex, Pattern.quote(message), true, false); + } + } + + @Override + public void assertInvalidCast(String projection, @Language("RegExp") String message) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + try { + computeActual(rewritten); + fail("expected exception"); + } + catch (RuntimeException ex) { + assertExceptionMessage(rewritten, ex, message, true, false); + } + } + /** * Rewrite SQL of the form 'select cast(arg as type)' to 'select cast(a as type) from (values (arg)) t(a)', and * SQL of the form 'select function(arg1, arg2, ...)' to diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java new file mode 100644 index 0000000000000..ea181eee17544 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestIpPrefixFunctions.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests.operator.scalar; + +import com.facebook.presto.tests.operator.scalar.AbstractTestIpPrefix; + +public class TestIpPrefixFunctions + extends AbstractTestNativeFunctions + implements AbstractTestIpPrefix +{ +} diff --git a/presto-open-telemetry/pom.xml b/presto-open-telemetry/pom.xml index e06d9c21bb993..fddbdc9e7996b 100644 --- a/presto-open-telemetry/pom.xml +++ b/presto-open-telemetry/pom.xml @@ -65,7 +65,7 @@ - io.opentelemetry + io.opentelemetry.semconv opentelemetry-semconv diff --git a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java index e50d52f2b6918..f3ac91d8c59f5 100644 --- a/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java +++ b/presto-open-telemetry/src/main/java/com/facebook/presto/opentelemetry/OpenTelemetryBuilder.java @@ -24,7 +24,7 @@ import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import io.opentelemetry.semconv.ServiceAttributes; public final class OpenTelemetryBuilder { @@ -55,7 +55,7 @@ else if (contextPropagator.equals(OpenTelemetryContextPropagator.B3_SINGLE_HEADE public static OpenTelemetry build(String contextPropagator) { Resource resource = Resource.getDefault() - .merge(Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "presto"))); + .merge(Resource.create(Attributes.of(ServiceAttributes.SERVICE_NAME, "presto"))); SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() .addSpanProcessor(BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder().setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")).build()).build()) diff --git a/presto-openlineage-event-listener/pom.xml b/presto-openlineage-event-listener/pom.xml new file mode 100644 index 0000000000000..4f85a9f1c6bf5 --- /dev/null +++ b/presto-openlineage-event-listener/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.297-SNAPSHOT + + + presto-openlineage-event-listener + presto-openlineage-event-listener + Presto - OpenLineage Event Listener + presto-plugin + + + ${project.parent.basedir} + 17 + true + + + + + + io.openlineage + openlineage-java + 1.44.1 + + + commons-logging + commons-logging + + + + + + com.facebook.airlift + log + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.google.guava + guava + + + + + com.facebook.presto + presto-spi + provided + + + + com.facebook.presto + presto-common + provided + + + + com.facebook.airlift.drift + drift-api + provided + + + + io.airlift + slice + provided + + + + com.facebook.airlift + units + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + org.openjdk.jol + jol-core + provided + + + + + com.facebook.presto + presto-testng-services + test + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + + org.assertj + assertj-core + test + + + diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/FormatInterpolator.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/FormatInterpolator.java new file mode 100644 index 0000000000000..6a8d013423359 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/FormatInterpolator.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +/** + * Simple replacement for Trino's FormatInterpolator from trino-plugin-toolkit. + * Replaces $PLACEHOLDER tokens in a format string with values from the provided context. + */ +public class FormatInterpolator +{ + private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$([A-Z_]+)"); + // Valid format: only letters, digits, underscores, hyphens, commas, spaces, equal signs, and $PLACEHOLDER tokens + private static final Pattern VALID_FORMAT_PATTERN = Pattern.compile("^([a-zA-Z0-9_\\-,= ]|\\$(" + + "QUERY_ID|USER|SOURCE|CLIENT_IP))*$"); + + private final String format; + private final OpenLineageJobInterpolatedValues[] values; + + public FormatInterpolator(String format, OpenLineageJobInterpolatedValues[] values) + { + this.format = requireNonNull(format, "format is null"); + this.values = requireNonNull(values, "values is null"); + } + + public String interpolate(OpenLineageJobContext context) + { + Matcher matcher = PLACEHOLDER_PATTERN.matcher(format); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String placeholder = matcher.group(1); + String replacement = getValueForPlaceholder(placeholder, context); + matcher.appendReplacement(result, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(result); + return result.toString(); + } + + private String getValueForPlaceholder(String placeholder, OpenLineageJobContext context) + { + for (OpenLineageJobInterpolatedValues value : values) { + if (value.name().equals(placeholder)) { + return value.value(context); + } + } + return "$" + placeholder; + } + + public static boolean hasValidPlaceholders(String format, OpenLineageJobInterpolatedValues[] values) + { + return VALID_FORMAT_PATTERN.matcher(format).matches(); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListener.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListener.java new file mode 100644 index 0000000000000..0120dadd1dc44 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListener.java @@ -0,0 +1,403 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.resourceGroups.QueryType; +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.OutputColumnMetadata; +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; +import com.facebook.presto.spi.eventlistener.QueryFailureInfo; +import com.facebook.presto.spi.eventlistener.QueryIOMetadata; +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryOutputMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.openlineage.client.OpenLineage; +import io.openlineage.client.OpenLineage.DatasetFacetsBuilder; +import io.openlineage.client.OpenLineage.InputDataset; +import io.openlineage.client.OpenLineage.InputDatasetBuilder; +import io.openlineage.client.OpenLineage.JobBuilder; +import io.openlineage.client.OpenLineage.OutputDataset; +import io.openlineage.client.OpenLineage.RunEvent; +import io.openlineage.client.OpenLineage.RunFacet; +import io.openlineage.client.OpenLineage.RunFacetsBuilder; +import io.openlineage.client.OpenLineageClient; + +import java.net.URI; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.openlineage.client.utils.UUIDUtils.generateStaticUUID; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.ZoneOffset.UTC; +import static java.util.Objects.requireNonNull; + +public class OpenLineageEventListener + implements EventListener +{ + private static final Logger logger = Logger.get(OpenLineageEventListener.class); + private static final ObjectMapper QUERY_STATISTICS_MAPPER = new ObjectMapper() + .findAndRegisterModules() + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + + private final OpenLineage openLineage; + private final OpenLineageClient client; + private final URI prestoURI; + private final String jobNamespace; + private final String datasetNamespace; + private final Set includeQueryTypes; + private final FormatInterpolator interpolator; + + public OpenLineageEventListener(OpenLineage openLineage, OpenLineageClient client, OpenLineageEventListenerConfig listenerConfig) + { + this.openLineage = requireNonNull(openLineage, "openLineage is null"); + this.client = requireNonNull(client, "client is null"); + requireNonNull(listenerConfig, "listenerConfig is null"); + this.prestoURI = defaultNamespace(listenerConfig.getPrestoURI()); + this.jobNamespace = listenerConfig.getNamespace().orElse(prestoURI.toString()); + this.datasetNamespace = prestoURI.toString(); + this.includeQueryTypes = ImmutableSet.copyOf(listenerConfig.getIncludeQueryTypes()); + this.interpolator = new FormatInterpolator(listenerConfig.getJobNameFormat(), OpenLineageJobInterpolatedValues.values()); + } + + @Override + public void queryCreated(QueryCreatedEvent queryCreatedEvent) + { + // Presto does not have queryType on QueryCreatedEvent (only on QueryCompletedEvent), + // so we always emit the START event — we can't filter by type until completion. + RunEvent event = getStartEvent(queryCreatedEvent); + client.emit(event); + } + + @Override + public void queryCompleted(QueryCompletedEvent queryCompletedEvent) + { + if (queryTypeSupported(queryCompletedEvent.getQueryType())) { + RunEvent event = getCompletedEvent(queryCompletedEvent); + client.emit(event); + return; + } + logger.debug("Query type %s not supported. Supported query types %s", + queryCompletedEvent.getQueryType().toString(), + this.includeQueryTypes); + } + + private boolean queryTypeSupported(Optional queryType) + { + return queryType + .map(this.includeQueryTypes::contains) + .orElse(false); + } + + /* + * Construct UUIDv7 from query creation time and queryId hash. + * UUIDv7 are both globally unique and ordered. + */ + private UUID getRunId(Instant queryCreateTime, QueryMetadata queryMetadata) + { + return generateStaticUUID(queryCreateTime, queryMetadata.getQueryId().getBytes(UTF_8)); + } + + private RunFacet getPrestoQueryContextFacet(QueryContext queryContext) + { + RunFacet queryContextFacet = openLineage.newRunFacet(); + + ImmutableMap.Builder properties = ImmutableMap.builder(); + + properties.put("server_address", queryContext.getServerAddress()); + properties.put("environment", queryContext.getEnvironment()); + + // Presto has no getQueryType() on QueryContext; queryType is on the event itself. + // We omit it from the context facet since it's not available here. + + properties.put("user", queryContext.getUser()); + // Presto has no getOriginalUser() — omitted + + queryContext.getPrincipal().ifPresent(principal -> + properties.put("principal", principal)); + + queryContext.getSource().ifPresent(source -> + properties.put("source", source)); + + queryContext.getClientInfo().ifPresent(clientInfo -> + properties.put("client_info", clientInfo)); + + queryContext.getRemoteClientAddress().ifPresent(remoteClientAddress -> + properties.put("remote_client_address", remoteClientAddress)); + + queryContext.getUserAgent().ifPresent(userAgent -> + properties.put("user_agent", userAgent)); + + // Presto has no getTraceToken() — omitted + + queryContextFacet + .getAdditionalProperties() + .putAll(properties.buildOrThrow()); + + return queryContextFacet; + } + + private RunFacet getPrestoMetadataFacet(QueryMetadata queryMetadata) + { + RunFacet prestoMetadataFacet = openLineage.newRunFacet(); + + ImmutableMap.Builder properties = ImmutableMap.builder(); + + properties.put("query_id", queryMetadata.getQueryId()); + + queryMetadata.getPlan().ifPresent( + queryPlan -> properties.put("query_plan", queryPlan)); + + queryMetadata.getTransactionId().ifPresent( + transactionId -> properties.put("transaction_id", transactionId)); + + prestoMetadataFacet + .getAdditionalProperties() + .putAll(properties.buildOrThrow()); + + return prestoMetadataFacet; + } + + @SuppressWarnings("unchecked") + private RunFacet getPrestoQueryStatisticsFacet(QueryStatistics queryStatistics) + { + RunFacet prestoQueryStatisticsFacet = openLineage.newRunFacet(); + + ImmutableMap.Builder properties = ImmutableMap.builder(); + + QUERY_STATISTICS_MAPPER.convertValue(queryStatistics, HashMap.class).forEach( + (key, value) -> { + if (key != null && value != null) { + properties.put(key.toString(), value.toString()); + } + }); + + prestoQueryStatisticsFacet + .getAdditionalProperties() + .putAll(properties.buildOrThrow()); + + return prestoQueryStatisticsFacet; + } + + public RunEvent getStartEvent(QueryCreatedEvent queryCreatedEvent) + { + UUID runID = getRunId(queryCreatedEvent.getCreateTime(), queryCreatedEvent.getMetadata()); + RunFacetsBuilder runFacetsBuilder = getBaseRunFacetsBuilder(queryCreatedEvent.getContext()); + + runFacetsBuilder.put(OpenLineagePrestoFacet.PRESTO_METADATA.asText(), + getPrestoMetadataFacet(queryCreatedEvent.getMetadata())); + runFacetsBuilder.put(OpenLineagePrestoFacet.PRESTO_QUERY_CONTEXT.asText(), + getPrestoQueryContextFacet(queryCreatedEvent.getContext())); + + return openLineage.newRunEventBuilder() + .eventType(RunEvent.EventType.START) + .eventTime(queryCreatedEvent.getCreateTime().atZone(UTC)) + .run(openLineage.newRunBuilder().runId(runID).facets(runFacetsBuilder.build()).build()) + .job(getBaseJobBuilder(queryCreatedEvent.getContext(), queryCreatedEvent.getMetadata()).build()) + .build(); + } + + public RunEvent getCompletedEvent(QueryCompletedEvent queryCompletedEvent) + { + UUID runID = getRunId(queryCompletedEvent.getCreateTime(), queryCompletedEvent.getMetadata()); + RunFacetsBuilder runFacetsBuilder = getBaseRunFacetsBuilder(queryCompletedEvent.getContext()); + + runFacetsBuilder.put(OpenLineagePrestoFacet.PRESTO_METADATA.asText(), + getPrestoMetadataFacet(queryCompletedEvent.getMetadata())); + runFacetsBuilder.put(OpenLineagePrestoFacet.PRESTO_QUERY_CONTEXT.asText(), + getPrestoQueryContextFacet(queryCompletedEvent.getContext())); + runFacetsBuilder.put(OpenLineagePrestoFacet.PRESTO_QUERY_STATISTICS.asText(), + getPrestoQueryStatisticsFacet(queryCompletedEvent.getStatistics())); + runFacetsBuilder.nominalTime( + openLineage.newNominalTimeRunFacet( + queryCompletedEvent.getCreateTime().atZone(ZoneOffset.UTC), + queryCompletedEvent.getEndTime().atZone(ZoneOffset.UTC))); + + boolean failed = queryCompletedEvent.getMetadata().getQueryState().equals("FAILED"); + if (failed) { + queryCompletedEvent + .getFailureInfo() + .flatMap(QueryFailureInfo::getFailureMessage) + .ifPresent(failureMessage -> runFacetsBuilder + .errorMessage(openLineage + .newErrorMessageRunFacetBuilder() + .message(failureMessage) + .build())); + } + + return openLineage.newRunEventBuilder() + .eventType( + failed + ? RunEvent.EventType.FAIL + : RunEvent.EventType.COMPLETE) + .eventTime(queryCompletedEvent.getEndTime().atZone(UTC)) + .run(openLineage.newRunBuilder().runId(runID).facets(runFacetsBuilder.build()).build()) + .job(getBaseJobBuilder(queryCompletedEvent.getContext(), queryCompletedEvent.getMetadata()).build()) + .inputs(buildInputs(queryCompletedEvent.getIoMetadata())) + .outputs(buildOutputs(queryCompletedEvent.getIoMetadata())) + .build(); + } + + private RunFacetsBuilder getBaseRunFacetsBuilder(QueryContext queryContext) + { + return openLineage.newRunFacetsBuilder() + .processing_engine(openLineage.newProcessingEngineRunFacetBuilder() + .name("presto") + .version(queryContext.getServerVersion()) + .build()); + } + + private JobBuilder getBaseJobBuilder(QueryContext queryContext, QueryMetadata queryMetadata) + { + return openLineage.newJobBuilder() + .namespace(this.jobNamespace) + .name(interpolator.interpolate(new OpenLineageJobContext(queryContext, queryMetadata))) + .facets(openLineage.newJobFacetsBuilder() + .jobType(openLineage.newJobTypeJobFacet("BATCH", "PRESTO", "QUERY")) + .sql(openLineage.newSQLJobFacet(queryMetadata.getQuery(), "presto")) + .build()); + } + + /** + * Build inputs from QueryIOMetadata. + * Unlike Trino which uses queryMetadata.getTables() (compile-time analysis), + * Presto has no equivalent. We use ioMetadata.getInputs() (runtime) which is + * only available in queryCompleted events. + */ + private List buildInputs(QueryIOMetadata ioMetadata) + { + return ioMetadata + .getInputs() + .stream() + .map(input -> { + String datasetName = getDatasetName(input.getCatalogName(), input.getSchema(), input.getTable()); + InputDatasetBuilder inputDatasetBuilder = openLineage + .newInputDatasetBuilder() + .namespace(this.datasetNamespace) + .name(datasetName); + + DatasetFacetsBuilder datasetFacetsBuilder = openLineage.newDatasetFacetsBuilder() + .dataSource(openLineage.newDatasourceDatasetFacet( + toQualifiedSchemaName(input.getCatalogName(), input.getSchema()), + prestoURI.resolve(toQualifiedSchemaName(input.getCatalogName(), input.getSchema())))) + .schema(openLineage.newSchemaDatasetFacetBuilder() + .fields( + input.getColumnObjects() + .stream() + .map(column -> openLineage.newSchemaDatasetFacetFieldsBuilder() + .name(column.getName()) + .type(column.getType()) + .build()) + .collect(toImmutableList())) + .build()); + + return inputDatasetBuilder + .facets(datasetFacetsBuilder.build()) + .build(); + }) + .collect(toImmutableList()); + } + + private List buildOutputs(QueryIOMetadata ioMetadata) + { + Optional outputs = ioMetadata.getOutput(); + if (outputs.isPresent()) { + QueryOutputMetadata outputMetadata = outputs.get(); + List outputColumns = outputMetadata.getColumns().orElse(List.of()); + + OpenLineage.ColumnLineageDatasetFacetFieldsBuilder columnLineageDatasetFacetFieldsBuilder = openLineage.newColumnLineageDatasetFacetFieldsBuilder(); + outputColumns.forEach(column -> + columnLineageDatasetFacetFieldsBuilder.put(column.getColumnName(), + openLineage.newColumnLineageDatasetFacetFieldsAdditionalBuilder() + .inputFields(column + .getSourceColumns() + .stream() + .map(inputColumn -> openLineage.newInputFieldBuilder() + .field(inputColumn.getColumnName()) + .namespace(this.datasetNamespace) + .name(getDatasetName( + inputColumn.getTableName().getCatalogName(), + inputColumn.getTableName().getSchemaName(), + inputColumn.getTableName().getObjectName())) + .build()) + .collect(toImmutableList())) + .build())); + + ImmutableList.Builder inputFields = ImmutableList.builder(); + ioMetadata.getInputs().forEach(input -> { + for (com.facebook.presto.spi.eventlistener.Column column : input.getColumnObjects()) { + inputFields.add(openLineage.newInputFieldBuilder() + .field(column.getName()) + .namespace(this.datasetNamespace) + .name(getDatasetName(input.getCatalogName(), input.getSchema(), input.getTable())) + .build()); + } + }); + + return ImmutableList.of( + openLineage.newOutputDatasetBuilder() + .namespace(this.datasetNamespace) + .name(getDatasetName(outputMetadata.getCatalogName(), outputMetadata.getSchema(), outputMetadata.getTable())) + .facets(openLineage.newDatasetFacetsBuilder() + .columnLineage(openLineage.newColumnLineageDatasetFacet(columnLineageDatasetFacetFieldsBuilder.build(), inputFields.build())) + .schema(openLineage.newSchemaDatasetFacetBuilder() + .fields( + outputColumns.stream() + .map(column -> openLineage.newSchemaDatasetFacetFieldsBuilder() + .name(column.getColumnName()) + .type(column.getColumnType()) + .build()) + .collect(toImmutableList())) + .build()) + .dataSource(openLineage.newDatasourceDatasetFacet( + toQualifiedSchemaName(outputMetadata.getCatalogName(), outputMetadata.getSchema()), + prestoURI.resolve(toQualifiedSchemaName(outputMetadata.getCatalogName(), outputMetadata.getSchema())))) + .build()) + .build()); + } + return ImmutableList.of(); + } + + private String getDatasetName(String catalogName, String schemaName, String tableName) + { + return format("%s.%s.%s", catalogName, schemaName, tableName); + } + + static URI defaultNamespace(URI uri) + { + if (!uri.getScheme().isEmpty()) { + return URI.create(uri.toString().replaceFirst(uri.getScheme(), "presto")); + } + return URI.create("presto://" + uri); + } + + private static String toQualifiedSchemaName(String catalogName, String schemaName) + { + return catalogName + "." + schemaName; + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerConfig.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerConfig.java new file mode 100644 index 0000000000000..0b0a57be74d55 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerConfig.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.common.resourceGroups.QueryType; +import com.google.common.collect.ImmutableSet; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; + +public class OpenLineageEventListenerConfig +{ + private URI prestoURI; + private Set disabledFacets = ImmutableSet.of(); + private Optional namespace = Optional.empty(); + private String jobNameFormat = "$QUERY_ID"; + + private Set includeQueryTypes = ImmutableSet.builder() + .add(QueryType.DELETE) + .add(QueryType.INSERT) + .add(QueryType.MERGE) + .add(QueryType.UPDATE) + .add(QueryType.DATA_DEFINITION) + .build(); + + public OpenLineageEventListenerConfig() + { + } + + public OpenLineageEventListenerConfig(Map config) + { + requireNonNull(config, "config is null"); + + String uriStr = config.get("openlineage-event-listener.presto.uri"); + if (uriStr != null) { + try { + this.prestoURI = new URI(uriStr); + } + catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid Presto URI: " + uriStr, e); + } + } + + String queryTypesStr = config.get("openlineage-event-listener.presto.include-query-types"); + if (queryTypesStr != null && !queryTypesStr.isEmpty()) { + this.includeQueryTypes = Arrays.stream(queryTypesStr.split(",")) + .map(String::trim) + .map(QueryType::valueOf) + .collect(Collectors.toSet()); + } + + String disabledFacetsStr = config.get("openlineage-event-listener.disabled-facets"); + if (disabledFacetsStr != null && !disabledFacetsStr.isEmpty()) { + this.disabledFacets = Arrays.stream(disabledFacetsStr.split(",")) + .map(String::trim) + .map(s -> OpenLineagePrestoFacet.valueOf(s.toUpperCase())) + .collect(Collectors.toSet()); + } + + String namespaceStr = config.get("openlineage-event-listener.namespace"); + this.namespace = Optional.ofNullable(namespaceStr); + + String jobNameFormatStr = config.get("openlineage-event-listener.job.name-format"); + if (jobNameFormatStr != null) { + this.jobNameFormat = jobNameFormatStr; + } + } + + public URI getPrestoURI() + { + return prestoURI; + } + + public OpenLineageEventListenerConfig setPrestoURI(URI prestoURI) + { + this.prestoURI = prestoURI; + return this; + } + + public Set getIncludeQueryTypes() + { + return includeQueryTypes; + } + + public OpenLineageEventListenerConfig setIncludeQueryTypes(Set includeQueryTypes) + { + this.includeQueryTypes = ImmutableSet.copyOf(includeQueryTypes); + return this; + } + + public Set getDisabledFacets() + { + return disabledFacets; + } + + public OpenLineageEventListenerConfig setDisabledFacets(Set disabledFacets) + { + this.disabledFacets = ImmutableSet.copyOf(disabledFacets); + return this; + } + + public Optional getNamespace() + { + return namespace; + } + + public OpenLineageEventListenerConfig setNamespace(String namespace) + { + this.namespace = Optional.ofNullable(namespace); + return this; + } + + public String getJobNameFormat() + { + return jobNameFormat; + } + + public OpenLineageEventListenerConfig setJobNameFormat(String jobNameFormat) + { + this.jobNameFormat = jobNameFormat; + return this; + } + + public boolean isJobNameFormatValid() + { + return FormatInterpolator.hasValidPlaceholders(jobNameFormat, OpenLineageJobInterpolatedValues.values()); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerFactory.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerFactory.java new file mode 100644 index 0000000000000..7236d8f504705 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import io.openlineage.client.OpenLineage; +import io.openlineage.client.OpenLineageClient; +import io.openlineage.client.transports.ConsoleTransport; +import io.openlineage.client.transports.HttpConfig; +import io.openlineage.client.transports.HttpSslContextConfig; +import io.openlineage.client.transports.HttpTransport; +import io.openlineage.client.transports.TokenProvider; +import io.openlineage.client.transports.Transport; + +import java.net.URI; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public class OpenLineageEventListenerFactory + implements EventListenerFactory +{ + private static final URI PRODUCER_URI = URI.create("https://github.com/prestodb/presto/plugin/presto-openlineage-event-listener"); + + @Override + public String getName() + { + return "openlineage-event-listener"; + } + + @Override + public EventListener create(Map config) + { + requireNonNull(config, "config is null"); + + OpenLineageEventListenerConfig listenerConfig = new OpenLineageEventListenerConfig(config); + OpenLineageTransportConfig transportConfig = new OpenLineageTransportConfig(config); + Transport transport = buildTransport(transportConfig, config); + + String[] disabledFacets = listenerConfig.getDisabledFacets().stream() + .map(OpenLineagePrestoFacet::asText) + .toArray(String[]::new); + + OpenLineageClient client = OpenLineageClient.builder() + .transport(transport) + .disableFacets(disabledFacets) + .build(); + + OpenLineage openLineage = new OpenLineage(PRODUCER_URI); + return new OpenLineageEventListener(openLineage, client, listenerConfig); + } + + private static Transport buildTransport(OpenLineageTransportConfig transportConfig, Map config) + { + switch (transportConfig.getTransport()) { + case CONSOLE: + return new ConsoleTransport(); + case HTTP: + return buildHttpTransport(new OpenLineageHttpTransportConfig(config)); + default: + throw new IllegalArgumentException("Unsupported transport type: " + transportConfig.getTransport()); + } + } + + private static HttpTransport buildHttpTransport(OpenLineageHttpTransportConfig config) + { + TokenProvider tokenProvider = config.getApiKey() + .map(key -> (TokenProvider) () -> "Bearer " + key) + .orElse(null); + + OpenLineageHttpTransportConfig.Compression configCompression = config.getCompression(); + HttpConfig.Compression httpCompression = configCompression == OpenLineageHttpTransportConfig.Compression.GZIP + ? HttpConfig.Compression.GZIP + : null; + + return new HttpTransport( + new HttpConfig( + config.getUrl(), + config.getEndpoint(), + (int) config.getTimeoutMillis(), + tokenProvider, + config.getUrlParams(), + config.getHeaders(), + httpCompression, + new HttpSslContextConfig())); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerPlugin.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerPlugin.java new file mode 100644 index 0000000000000..91eac3df2f504 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageEventListenerPlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import com.google.common.collect.ImmutableList; + +public class OpenLineageEventListenerPlugin + implements Plugin +{ + @Override + public Iterable getEventListenerFactories() + { + return ImmutableList.of(new OpenLineageEventListenerFactory()); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageHttpTransportConfig.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageHttpTransportConfig.java new file mode 100644 index 0000000000000..27de344d7dbea --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageHttpTransportConfig.java @@ -0,0 +1,193 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class OpenLineageHttpTransportConfig +{ + public enum Compression + { + NONE, GZIP + } + + private URI url; + private String endpoint; + private Optional apiKey = Optional.empty(); + private long timeoutMillis = TimeUnit.SECONDS.toMillis(5); + private Map headers = new HashMap<>(); + private Map urlParams = new HashMap<>(); + private Compression compression = Compression.NONE; + + public OpenLineageHttpTransportConfig() + { + } + + public OpenLineageHttpTransportConfig(Map config) + { + requireNonNull(config, "config is null"); + String urlStr = config.get("openlineage-event-listener.transport.url"); + if (urlStr != null) { + try { + this.url = new URI(urlStr); + } + catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid transport URL: " + urlStr, e); + } + } + this.endpoint = config.get("openlineage-event-listener.transport.endpoint"); + String apiKeyStr = config.get("openlineage-event-listener.transport.api-key"); + this.apiKey = Optional.ofNullable(apiKeyStr); + + String timeoutStr = config.get("openlineage-event-listener.transport.timeout"); + if (timeoutStr != null) { + this.timeoutMillis = parseDurationToMillis(timeoutStr); + } + + String headersStr = config.get("openlineage-event-listener.transport.headers"); + if (headersStr != null && !headersStr.isEmpty()) { + this.headers = parseKeyValuePairs(headersStr, "headers"); + } + + String urlParamsStr = config.get("openlineage-event-listener.transport.url-params"); + if (urlParamsStr != null && !urlParamsStr.isEmpty()) { + this.urlParams = parseKeyValuePairs(urlParamsStr, "url-params"); + } + + String compressionStr = config.get("openlineage-event-listener.transport.compression"); + if (compressionStr != null) { + this.compression = Compression.valueOf(compressionStr.toUpperCase()); + } + } + + public URI getUrl() + { + return url; + } + + public OpenLineageHttpTransportConfig setUrl(URI url) + { + this.url = url; + return this; + } + + public String getEndpoint() + { + return endpoint; + } + + public OpenLineageHttpTransportConfig setEndpoint(String endpoint) + { + this.endpoint = endpoint; + return this; + } + + public Optional getApiKey() + { + return apiKey; + } + + public OpenLineageHttpTransportConfig setApiKey(String apiKey) + { + this.apiKey = Optional.ofNullable(apiKey); + return this; + } + + public long getTimeoutMillis() + { + return timeoutMillis; + } + + public OpenLineageHttpTransportConfig setTimeoutMillis(long timeoutMillis) + { + this.timeoutMillis = timeoutMillis; + return this; + } + + public Map getHeaders() + { + return headers; + } + + public OpenLineageHttpTransportConfig setHeaders(Map headers) + { + this.headers = headers; + return this; + } + + public Map getUrlParams() + { + return urlParams; + } + + public OpenLineageHttpTransportConfig setUrlParams(Map urlParams) + { + this.urlParams = urlParams; + return this; + } + + public Compression getCompression() + { + return compression; + } + + public OpenLineageHttpTransportConfig setCompression(Compression compression) + { + this.compression = compression; + return this; + } + + private static Map parseKeyValuePairs(String input, String propertyName) + { + Map result = new HashMap<>(); + String[] pairs = input.split(","); + for (String pair : pairs) { + String[] parts = pair.split(":", 2); + if (parts.length != 2) { + throw new IllegalArgumentException(format( + "Cannot parse %s from property; value provided was %s, " + + "expected format is \"key1:value1,key2:value2,...\"", + propertyName, input)); + } + result.put(parts[0].trim(), parts[1].trim()); + } + return result; + } + + private static long parseDurationToMillis(String duration) + { + duration = duration.trim().toLowerCase(); + if (duration.endsWith("ms")) { + return Long.parseLong(duration.substring(0, duration.length() - 2)); + } + else if (duration.endsWith("s")) { + return TimeUnit.SECONDS.toMillis(Long.parseLong(duration.substring(0, duration.length() - 1))); + } + else if (duration.endsWith("m")) { + return TimeUnit.MINUTES.toMillis(Long.parseLong(duration.substring(0, duration.length() - 1))); + } + else if (duration.endsWith("h")) { + return TimeUnit.HOURS.toMillis(Long.parseLong(duration.substring(0, duration.length() - 1))); + } + return Long.parseLong(duration); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobContext.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobContext.java new file mode 100644 index 0000000000000..797e47f77577c --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobContext.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryMetadata; + +import static java.util.Objects.requireNonNull; + +public class OpenLineageJobContext +{ + private final QueryContext queryContext; + private final QueryMetadata queryMetadata; + + public OpenLineageJobContext(QueryContext queryContext, QueryMetadata queryMetadata) + { + this.queryContext = requireNonNull(queryContext, "queryContext is null"); + this.queryMetadata = requireNonNull(queryMetadata, "queryMetadata is null"); + } + + public QueryContext getQueryContext() + { + return queryContext; + } + + public QueryMetadata getQueryMetadata() + { + return queryMetadata; + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobInterpolatedValues.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobInterpolatedValues.java new file mode 100644 index 0000000000000..c6e994dd22f72 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageJobInterpolatedValues.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; + +public enum OpenLineageJobInterpolatedValues +{ + QUERY_ID(jobContext -> jobContext.getQueryMetadata().getQueryId()), + SOURCE(jobContext -> jobContext.getQueryContext().getSource().orElse("")), + CLIENT_IP(jobContext -> jobContext.getQueryContext().getRemoteClientAddress().orElse("")), + USER(jobContext -> jobContext.getQueryContext().getUser()); + + private final Function valueProvider; + + OpenLineageJobInterpolatedValues(Function valueProvider) + { + this.valueProvider = requireNonNull(valueProvider, "valueProvider is null"); + } + + public String value(OpenLineageJobContext context) + { + return valueProvider.apply(context); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineagePrestoFacet.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineagePrestoFacet.java new file mode 100644 index 0000000000000..ec5cd0eaf041c --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineagePrestoFacet.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import static java.util.Locale.ENGLISH; + +public enum OpenLineagePrestoFacet +{ + PRESTO_METADATA, + PRESTO_QUERY_STATISTICS, + PRESTO_QUERY_CONTEXT; + + public String asText() + { + return name().toLowerCase(ENGLISH); + } +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransport.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransport.java new file mode 100644 index 0000000000000..fc950740a6a3d --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransport.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +public enum OpenLineageTransport +{ + CONSOLE, + HTTP, + /**/ +} diff --git a/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransportConfig.java b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransportConfig.java new file mode 100644 index 0000000000000..d8e6dd4f07149 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/java/com/facebook/presto/plugin/openlineage/OpenLineageTransportConfig.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public class OpenLineageTransportConfig +{ + private OpenLineageTransport transport = OpenLineageTransport.CONSOLE; + + public OpenLineageTransportConfig() + { + } + + public OpenLineageTransportConfig(Map config) + { + requireNonNull(config, "config is null"); + String transportType = config.get("openlineage-event-listener.transport.type"); + if (transportType != null) { + this.transport = OpenLineageTransport.valueOf(transportType.toUpperCase()); + } + } + + public OpenLineageTransport getTransport() + { + return transport; + } + + public OpenLineageTransportConfig setTransport(OpenLineageTransport transport) + { + this.transport = transport; + return this; + } +} diff --git a/presto-openlineage-event-listener/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin b/presto-openlineage-event-listener/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin new file mode 100644 index 0000000000000..b8cf1f6abf7e3 --- /dev/null +++ b/presto-openlineage-event-listener/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin @@ -0,0 +1 @@ +com.facebook.presto.plugin.openlineage.OpenLineageEventListenerPlugin diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/OpenLineageMemoryTransport.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/OpenLineageMemoryTransport.java new file mode 100644 index 0000000000000..d1da13a95d512 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/OpenLineageMemoryTransport.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.google.common.collect.ImmutableList; +import io.openlineage.client.OpenLineage; +import io.openlineage.client.transports.Transport; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class OpenLineageMemoryTransport + extends Transport +{ + private final List processedEvents = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void emit(OpenLineage.RunEvent runEvent) + { + processedEvents.add(runEvent); + } + + @Override + public void emit(OpenLineage.DatasetEvent datasetEvent) + { + processedEvents.add(datasetEvent); + } + + @Override + public void emit(OpenLineage.JobEvent jobEvent) + { + processedEvents.add(jobEvent); + } + + public void clearProcessedEvents() + { + processedEvents.clear(); + } + + public List getProcessedEvents() + { + return ImmutableList.copyOf(processedEvents); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/PrestoEventData.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/PrestoEventData.java new file mode 100644 index 0000000000000..fcbfee384a847 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/PrestoEventData.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.common.resourceGroups.QueryType; +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; +import com.facebook.presto.spi.eventlistener.QueryIOMetadata; +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; +import com.facebook.presto.spi.resourceGroups.ResourceGroupId; +import com.facebook.presto.spi.session.ResourceEstimates; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; + +public final class PrestoEventData +{ + public static final QueryIOMetadata queryIOMetadata; + public static final QueryContext queryContext; + public static final QueryMetadata queryMetadata; + public static final QueryStatistics queryStatistics; + public static final QueryCompletedEvent queryCompleteEvent; + public static final QueryCreatedEvent queryCreatedEvent; + + private PrestoEventData() + { + throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); + } + + static { + queryIOMetadata = new QueryIOMetadata(Collections.emptyList(), Optional.empty()); + + queryContext = new QueryContext( + "user", + Optional.of("principal"), + Optional.of("127.0.0.1"), + Optional.of("Some-User-Agent"), + Optional.of("Some client info"), + new HashSet<>(), // clientTags + Optional.of("some-presto-client"), + Optional.of("catalog"), + Optional.of("schema"), + Optional.of(new ResourceGroupId("name")), + new HashMap<>(), // sessionProperties + new ResourceEstimates(Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()), + "serverAddress", + "serverVersion", + "environment", + "worker"); // workerType + + queryMetadata = new QueryMetadata( + "queryId", + Optional.of("transactionId"), + "create table b.c as select * from y.z", + "queryHash", + Optional.of("preparedQuery"), + "COMPLETED", + URI.create("http://localhost"), + Optional.of("queryPlan"), + Optional.empty(), // jsonPlan + Optional.empty(), // graphvizPlan + Optional.empty(), // payload + List.of(), // runtimeOptimizedStages + Optional.empty(), // tracingId + Optional.of("updateType")); + + queryStatistics = new QueryStatistics( + Duration.ofSeconds(1), // cpuTime + Duration.ofSeconds(1), // retriedCpuTime + Duration.ofSeconds(1), // wallTime + Duration.ofSeconds(1), // totalScheduledTime + Duration.ofSeconds(0), // waitingForPrerequisitesTime + Duration.ofSeconds(0), // queuedTime + Duration.ofSeconds(0), // waitingForResourcesTime + Duration.ofSeconds(0), // semanticAnalyzingTime + Duration.ofSeconds(0), // columnAccessPermissionCheckingTime + Duration.ofSeconds(0), // dispatchingTime + Duration.ofSeconds(0), // planningTime + Optional.empty(), // analysisTime + Duration.ofSeconds(1), // executionTime + 0, // peakRunningTasks + 0L, // peakUserMemoryBytes + 0L, // peakTotalNonRevocableMemoryBytes + 0L, // peakTaskUserMemory + 0L, // peakTaskTotalMemory + 0L, // peakNodeTotalMemory + 0L, // shuffledBytes + 0L, // shuffledRows + 0L, // totalBytes + 0L, // totalRows + 0L, // outputBytes + 0L, // outputRows + 0L, // writtenOutputBytes + 0L, // writtenOutputRows + 0L, // writtenIntermediateBytes + 0L, // spilledBytes + 0.0, // cumulativeMemory + 0.0, // cumulativeTotalMemory + 0, // completedSplits + true, // complete + new RuntimeStats()); + + queryCompleteEvent = new QueryCompletedEvent( + queryMetadata, + queryStatistics, + queryContext, + queryIOMetadata, + Optional.empty(), // failureInfo + Collections.emptyList(), // warnings + Optional.of(QueryType.INSERT), // queryType + Collections.emptyList(), // failedTasks + Instant.parse("2025-04-28T11:23:55.384424Z"), // createTime + Instant.parse("2025-04-28T11:24:16.256207Z"), // executionStartTime + Instant.parse("2025-04-28T11:24:26.993340Z"), // endTime + Collections.emptyList(), // stageStatistics + Collections.emptyList(), // operatorStatistics + Collections.emptyList(), // planStatisticsRead + Collections.emptyList(), // planStatisticsWritten + Collections.emptyMap(), // planNodeHash + Collections.emptyMap(), // canonicalPlan + Optional.empty(), // statsEquivalentPlan + Optional.empty(), // expandedQuery + Collections.emptyList(), // optimizerInformation + Collections.emptyList(), // cteInformationList + Collections.emptySet(), // scalarFunctions + Collections.emptySet(), // aggregateFunctions + Collections.emptySet(), // windowFunctions + Optional.empty(), // prestoSparkExecutionContext + Collections.emptyMap(), // hboPlanHash + Optional.empty(), // planNodeIdMap + Optional.empty()); // qualifiedName + + queryCreatedEvent = new QueryCreatedEvent( + Instant.parse("2025-04-28T11:23:55.384424Z"), + queryContext, + queryMetadata); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListener.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListener.java new file mode 100644 index 0000000000000..c94b4dd8a0cb1 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListener.java @@ -0,0 +1,109 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.google.common.collect.ImmutableMap; +import io.openlineage.client.OpenLineage.RunEvent; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; + +public class TestOpenLineageEventListener +{ + @Test + public void testGetCompleteEvent() + { + OpenLineageEventListener listener = (OpenLineageEventListener) createEventListener(Map.of( + "openlineage-event-listener.transport.type", "CONSOLE", + "openlineage-event-listener.presto.uri", "http://testhost")); + + RunEvent result = listener.getCompletedEvent(PrestoEventData.queryCompleteEvent); + + assertThat(result.getEventType()).isEqualTo(RunEvent.EventType.COMPLETE); + assertThat(result.getEventTime().toInstant()).isEqualTo(PrestoEventData.queryCompleteEvent.getEndTime()); + assertThat(result.getRun().getRunId().toString()).startsWith("01967c23-ae78-7"); + assertThat(result.getJob().getNamespace()).isEqualTo("presto://testhost"); + assertThat(result.getJob().getName()).isEqualTo("queryId"); + + Map prestoQueryMetadata = result + .getRun() + .getFacets() + .getAdditionalProperties() + .get("presto_metadata") + .getAdditionalProperties(); + + assertThat(prestoQueryMetadata) + .containsOnly( + entry("query_id", "queryId"), + entry("transaction_id", "transactionId"), + entry("query_plan", "queryPlan")); + + Map prestoQueryContext = + result + .getRun() + .getFacets() + .getAdditionalProperties() + .get("presto_query_context") + .getAdditionalProperties(); + + assertThat(prestoQueryContext) + .containsOnly( + entry("server_address", "serverAddress"), + entry("environment", "environment"), + entry("user", "user"), + entry("principal", "principal"), + entry("source", "some-presto-client"), + entry("client_info", "Some client info"), + entry("remote_client_address", "127.0.0.1"), + entry("user_agent", "Some-User-Agent")); + } + + @Test + public void testGetStartEvent() + { + OpenLineageEventListener listener = (OpenLineageEventListener) createEventListener(Map.of( + "openlineage-event-listener.transport.type", OpenLineageTransport.CONSOLE.toString(), + "openlineage-event-listener.presto.uri", "http://testhost:8080")); + + RunEvent result = listener.getStartEvent(PrestoEventData.queryCreatedEvent); + + assertThat(result.getEventType()).isEqualTo(RunEvent.EventType.START); + assertThat(result.getEventTime().toInstant()).isEqualTo(PrestoEventData.queryCreatedEvent.getCreateTime()); + assertThat(result.getRun().getRunId().toString()).startsWith("01967c23-ae78-7"); + assertThat(result.getJob().getNamespace()).isEqualTo("presto://testhost:8080"); + assertThat(result.getJob().getName()).isEqualTo("queryId"); + } + + @Test + public void testJobNameFormatting() + { + OpenLineageEventListener listener = (OpenLineageEventListener) createEventListener(Map.of( + "openlineage-event-listener.transport.type", "CONSOLE", + "openlineage-event-listener.presto.uri", "http://testhost:8080", + "openlineage-event-listener.job.name-format", "$QUERY_ID-$USER-$SOURCE-$CLIENT_IP-abc123")); + + RunEvent result = listener.getCompletedEvent(PrestoEventData.queryCompleteEvent); + + assertThat(result.getJob().getNamespace()).isEqualTo("presto://testhost:8080"); + assertThat(result.getJob().getName()).isEqualTo("queryId-user-some-presto-client-127.0.0.1-abc123"); + } + + private static com.facebook.presto.spi.eventlistener.EventListener createEventListener(Map config) + { + return new OpenLineageEventListenerFactory().create(ImmutableMap.copyOf(config)); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerConfig.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerConfig.java new file mode 100644 index 0000000000000..c2751858b46e2 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.Map; + +import static com.facebook.presto.common.resourceGroups.QueryType.DELETE; +import static com.facebook.presto.common.resourceGroups.QueryType.INSERT; +import static com.facebook.presto.common.resourceGroups.QueryType.SELECT; +import static com.facebook.presto.plugin.openlineage.OpenLineagePrestoFacet.PRESTO_METADATA; +import static com.facebook.presto.plugin.openlineage.OpenLineagePrestoFacet.PRESTO_QUERY_STATISTICS; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestOpenLineageEventListenerConfig +{ + @Test + public void testDefaults() + { + OpenLineageEventListenerConfig config = new OpenLineageEventListenerConfig(); + assertThat(config.getPrestoURI()).isNull(); + assertThat(config.getNamespace()).isEmpty(); + assertThat(config.getJobNameFormat()).isEqualTo("$QUERY_ID"); + assertThat(config.getDisabledFacets()).isEmpty(); + assertThat(config.getIncludeQueryTypes()).containsExactlyInAnyOrder( + DELETE, INSERT, + com.facebook.presto.common.resourceGroups.QueryType.MERGE, + com.facebook.presto.common.resourceGroups.QueryType.UPDATE, + com.facebook.presto.common.resourceGroups.QueryType.DATA_DEFINITION); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("openlineage-event-listener.presto.uri", "http://testpresto") + .put("openlineage-event-listener.presto.include-query-types", "SELECT,DELETE") + .put("openlineage-event-listener.disabled-facets", "PRESTO_METADATA,PRESTO_QUERY_STATISTICS") + .put("openlineage-event-listener.namespace", "testnamespace") + .put("openlineage-event-listener.job.name-format", "$QUERY_ID-$USER-$SOURCE-$CLIENT_IP-abc123") + .build(); + + OpenLineageEventListenerConfig config = new OpenLineageEventListenerConfig(properties); + + assertThat(config.getPrestoURI()).isEqualTo(URI.create("http://testpresto")); + assertThat(config.getIncludeQueryTypes()).containsExactlyInAnyOrder(SELECT, DELETE); + assertThat(config.getDisabledFacets()).containsExactlyInAnyOrder(PRESTO_METADATA, PRESTO_QUERY_STATISTICS); + assertThat(config.getNamespace()).hasValue("testnamespace"); + assertThat(config.getJobNameFormat()).isEqualTo("$QUERY_ID-$USER-$SOURCE-$CLIENT_IP-abc123"); + } + + @Test + public void testIsJobNameFormatValid() + { + assertThat(configWithFormat("abc123").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$QUERY_ID").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$USER").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$SOURCE").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$CLIENT_IP").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$QUERY_ID-$USER-$SOURCE-$CLIENT_IP-abc123").isJobNameFormatValid()).isTrue(); + assertThat(configWithFormat("$QUERY_ID $USER $SOURCE $CLIENT_IP abc123").isJobNameFormatValid()).isTrue(); + + assertThat(configWithFormat("$query_id").isJobNameFormatValid()).isFalse(); + assertThat(configWithFormat("$UNKNOWN").isJobNameFormatValid()).isFalse(); + assertThat(configWithFormat("${QUERY_ID}").isJobNameFormatValid()).isFalse(); + assertThat(configWithFormat("$$QUERY_ID").isJobNameFormatValid()).isFalse(); + assertThat(configWithFormat("\\$QUERY_ID").isJobNameFormatValid()).isFalse(); + } + + private static OpenLineageEventListenerConfig configWithFormat(String format) + { + return new OpenLineageEventListenerConfig() + .setPrestoURI(URI.create("http://testpresto")) + .setJobNameFormat(format); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerPlugin.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerPlugin.java new file mode 100644 index 0000000000000..54de0767a77b7 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageEventListenerPlugin.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; + +public class TestOpenLineageEventListenerPlugin +{ + @Test + public void testCreateConsoleEventListener() + { + OpenLineageEventListenerPlugin plugin = new OpenLineageEventListenerPlugin(); + + EventListenerFactory factory = getOnlyElement(plugin.getEventListenerFactories()); + factory.create( + ImmutableMap.builder() + .put("openlineage-event-listener.presto.uri", "http://localhost:8080") + .put("openlineage-event-listener.transport.type", "console") + .build()); + } + + @Test + public void testCreateHttpEventListener() + { + OpenLineageEventListenerPlugin plugin = new OpenLineageEventListenerPlugin(); + + EventListenerFactory factory = getOnlyElement(plugin.getEventListenerFactories()); + factory.create( + ImmutableMap.builder() + .put("openlineage-event-listener.presto.uri", "http://localhost:8080") + .put("openlineage-event-listener.transport.type", "http") + .put("openlineage-event-listener.transport.url", "http://testurl") + .build()); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageHttpTransportConfig.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageHttpTransportConfig.java new file mode 100644 index 0000000000000..e572a30bcb3fb --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageHttpTransportConfig.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.net.URI; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestOpenLineageHttpTransportConfig +{ + @Test + public void testDefaults() + { + OpenLineageHttpTransportConfig config = new OpenLineageHttpTransportConfig(); + assertThat(config.getUrl()).isNull(); + assertThat(config.getEndpoint()).isNull(); + assertThat(config.getTimeoutMillis()).isEqualTo(5000L); + assertThat(config.getApiKey()).isEmpty(); + assertThat(config.getHeaders()).isEmpty(); + assertThat(config.getUrlParams()).isEmpty(); + assertThat(config.getCompression()).isEqualTo(OpenLineageHttpTransportConfig.Compression.NONE); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("openlineage-event-listener.transport.url", "http://testurl") + .put("openlineage-event-listener.transport.endpoint", "/test/endpoint") + .put("openlineage-event-listener.transport.api-key", "dummy") + .put("openlineage-event-listener.transport.timeout", "30s") + .put("openlineage-event-listener.transport.headers", "header1:value1,header2:value2") + .put("openlineage-event-listener.transport.url-params", "urlParam1:urlVal1,urlParam2:urlVal2") + .put("openlineage-event-listener.transport.compression", "gzip") + .build(); + + OpenLineageHttpTransportConfig config = new OpenLineageHttpTransportConfig(properties); + + assertThat(config.getUrl()).isEqualTo(URI.create("http://testurl")); + assertThat(config.getEndpoint()).isEqualTo("/test/endpoint"); + assertThat(config.getApiKey()).hasValue("dummy"); + assertThat(config.getTimeoutMillis()).isEqualTo(30000L); + assertThat(config.getHeaders()).containsEntry("header1", "value1").containsEntry("header2", "value2"); + assertThat(config.getUrlParams()).containsEntry("urlParam1", "urlVal1").containsEntry("urlParam2", "urlVal2"); + assertThat(config.getCompression()).isEqualTo(OpenLineageHttpTransportConfig.Compression.GZIP); + } +} diff --git a/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageTransportConfig.java b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageTransportConfig.java new file mode 100644 index 0000000000000..7ac7782a5fd74 --- /dev/null +++ b/presto-openlineage-event-listener/src/test/java/com/facebook/presto/plugin/openlineage/TestOpenLineageTransportConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.openlineage; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestOpenLineageTransportConfig +{ + @Test + public void testDefaults() + { + OpenLineageTransportConfig config = new OpenLineageTransportConfig(); + assertThat(config.getTransport()).isEqualTo(OpenLineageTransport.CONSOLE); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("openlineage-event-listener.transport.type", "HTTP") + .build(); + + OpenLineageTransportConfig config = new OpenLineageTransportConfig(properties); + assertThat(config.getTransport()).isEqualTo(OpenLineageTransport.HTTP); + } +} diff --git a/presto-parquet/pom.xml b/presto-parquet/pom.xml index 69e002e7f883c..61bd0ae80499b 100644 --- a/presto-parquet/pom.xml +++ b/presto-parquet/pom.xml @@ -135,12 +135,6 @@ test - - org.apache.commons - commons-lang3 - test - - com.facebook.presto presto-tests diff --git a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 index 92eabfe99cb04..54fc2dd4cc99e 100644 --- a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 +++ b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 @@ -68,6 +68,15 @@ statement ALTER (COLUMN)? column=identifier DROP NOT NULL #alterColumnDropNotNull | ALTER TABLE (IF EXISTS)? tableName=qualifiedName SET PROPERTIES properties #setTableProperties + | ALTER TABLE (IF EXISTS)? tableName=qualifiedName + CREATE (OR REPLACE)? BRANCH (IF NOT EXISTS)? name=string + tableVersionExpression? + (RETAIN retainDays=INTEGER_VALUE DAYS)? + (WITH SNAPSHOT RETENTION (minSnapshots=INTEGER_VALUE SNAPSHOTS)? (maxSnapshotAge=INTEGER_VALUE DAYS)?)? #createBranch + | ALTER TABLE (IF EXISTS)? tableName=qualifiedName + CREATE (OR REPLACE)? TAG (IF NOT EXISTS)? name=string + tableVersionExpression? + (RETAIN retainDays=INTEGER_VALUE DAYS)? #createTag | ALTER TABLE (IF EXISTS)? tableName=qualifiedName DROP BRANCH (IF EXISTS)? name=string #dropBranch | ALTER TABLE (IF EXISTS)? tableName=qualifiedName @@ -690,7 +699,7 @@ nonReserved : ADD | ADMIN | ALL | ANALYZE | ANY | ARRAY | ASC | AT | BEFORE | BERNOULLI | BRANCH | CALL | CALLED | CASCADE | CATALOGS | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | COPARTITION | CURRENT | CURRENT_ROLE - | DATA | DATE | DAY | DEFINER | DESC | DESCRIPTOR | DETERMINISTIC | DISABLED | DISTRIBUTED + | DATA | DATE | DAY | DEFINER | DESC | DESCRIPTOR | DETERMINISTIC | DISABLED | DISTRIBUTED | DAYS | EMPTY | ENABLED | ENFORCED | EXCLUDING | EXPLAIN | EXTERNAL | FETCH | FILTER | FIRST | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS | GRANT | GRANTED | GRANTS | GRAPHVIZ | GROUPS @@ -703,8 +712,8 @@ nonReserved | NAME | NFC | NFD | NFKC | NFKD | NO | NONE | NULLIF | NULLS | OF | OFFSET | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | PARTITION | PARTITIONS | POSITION | PRECEDING | PRIMARY | PRIVILEGES | PROPERTIES | PRUNE - | RANGE | READ | REFRESH | RELY | RENAME | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS - | SCHEMA | SCHEMAS | SECOND | SECURITY | SERIALIZABLE | SESSION | SET | SETS | SQL + | RANGE | READ | REFRESH | RELY | RENAME | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETAIN | RETENTION | RETURN | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS + | SCHEMA | SCHEMAS | SECOND | SECURITY | SERIALIZABLE | SESSION | SET | SETS | SNAPSHOT | SNAPSHOTS | SQL | SHOW | SOME | START | STATS | SUBSTRING | SYSTEM | SYSTEM_TIME | SYSTEM_VERSION | TABLES | TABLESAMPLE | TAG | TEMPORARY | TEXT | TIME | TIMESTAMP | TO | TRANSACTION | TRUNCATE | TRY_CAST | TYPE | UNBOUNDED | UNCOMMITTED | UNIQUE | UPDATE | USE | USER @@ -754,6 +763,7 @@ CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; CURRENT_USER: 'CURRENT_USER'; DATA: 'DATA'; DATE: 'DATE'; +DAYS: 'DAYS'; DAY: 'DAY'; DEALLOCATE: 'DEALLOCATE'; DEFINER: 'DEFINER'; @@ -877,6 +887,8 @@ REPLACE: 'REPLACE'; RESET: 'RESET'; RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; +RETAIN: 'RETAIN'; +RETENTION: 'RETENTION'; RETURN: 'RETURN'; RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; @@ -897,6 +909,8 @@ SESSION: 'SESSION'; SET: 'SET'; SETS: 'SETS'; SHOW: 'SHOW'; +SNAPSHOT: 'SNAPSHOT'; +SNAPSHOTS: 'SNAPSHOTS'; SOME: 'SOME'; SQL: 'SQL'; START: 'START'; diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java index 44a0f0fdd3e4e..d7d7c3a1e0cef 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java @@ -27,12 +27,14 @@ import com.facebook.presto.sql.tree.ColumnDefinition; import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.ConstraintSpecification; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; +import com.facebook.presto.sql.tree.CreateTag; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Deallocate; import com.facebook.presto.sql.tree.Delete; @@ -123,6 +125,7 @@ import com.facebook.presto.sql.tree.TableFunctionInvocation; import com.facebook.presto.sql.tree.TableFunctionTableArgument; import com.facebook.presto.sql.tree.TableSubquery; +import com.facebook.presto.sql.tree.TableVersionExpression; import com.facebook.presto.sql.tree.TransactionAccessMode; import com.facebook.presto.sql.tree.TransactionMode; import com.facebook.presto.sql.tree.TruncateTable; @@ -1830,6 +1833,88 @@ protected Void visitShowRoleGrants(ShowRoleGrants node, Integer context) return null; } + @Override + protected Void visitCreateBranch(CreateBranch node, Integer indent) + { + builder.append("ALTER TABLE "); + if (node.isTableExists()) { + builder.append("IF EXISTS "); + } + builder.append(formatName(node.getTableName())) + .append(" CREATE "); + if (node.isReplace()) { + builder.append("OR REPLACE "); + } + builder.append("BRANCH "); + if (node.isIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatStringLiteral(node.getBranchName())); + if (node.getTableVersion().isPresent()) { + TableVersionExpression tableVersion = node.getTableVersion().get(); + builder.append(" FOR ") + .append(tableVersion.getTableVersionType().name()) + .append(tableVersion.getTableVersionOperator() == TableVersionExpression.TableVersionOperator.EQUAL ? " AS OF " : " BEFORE ") + .append(formatExpression(tableVersion.getStateExpression(), parameters)); + } + + if (node.getRetainDays().isPresent()) { + builder.append(" RETAIN ") + .append(node.getRetainDays().get()) + .append(" DAYS"); + } + + if (node.getMinSnapshotsToKeep().isPresent() || node.getMaxSnapshotAgeDays().isPresent()) { + builder.append(" WITH SNAPSHOT RETENTION"); + if (node.getMinSnapshotsToKeep().isPresent()) { + builder.append(" ") + .append(node.getMinSnapshotsToKeep().get()) + .append(" SNAPSHOTS"); + } + if (node.getMaxSnapshotAgeDays().isPresent()) { + builder.append(" ") + .append(node.getMaxSnapshotAgeDays().get()) + .append(" DAYS"); + } + } + + return null; + } + + @Override + protected Void visitCreateTag(CreateTag node, Integer indent) + { + builder.append("ALTER TABLE "); + if (node.isTableExists()) { + builder.append("IF EXISTS "); + } + builder.append(formatName(node.getTableName())) + .append(" CREATE "); + if (node.isReplace()) { + builder.append("OR REPLACE "); + } + builder.append("TAG "); + if (node.isIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatStringLiteral(node.getTagName())); + if (node.getTableVersion().isPresent()) { + TableVersionExpression tableVersion = node.getTableVersion().get(); + builder.append(" FOR ") + .append(tableVersion.getTableVersionType().name()) + .append(tableVersion.getTableVersionOperator() == TableVersionExpression.TableVersionOperator.EQUAL ? " AS OF " : " BEFORE ") + .append(formatExpression(tableVersion.getStateExpression(), parameters)); + } + + if (node.getRetainDays().isPresent()) { + builder.append(" RETAIN ") + .append(node.getRetainDays().get()) + .append(" DAYS"); + } + + return null; + } + @Override protected Void visitDropBranch(DropBranch node, Integer indent) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java index 4a4e91eb576c6..b1ad2a9c290d2 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java @@ -39,12 +39,14 @@ import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.ConstraintSpecification; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; +import com.facebook.presto.sql.tree.CreateTag; import com.facebook.presto.sql.tree.CreateType; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Cube; @@ -605,6 +607,74 @@ public Node visitDropBranch(SqlBaseParser.DropBranchContext context) context.EXISTS().stream().anyMatch(node -> node.getSymbol().getTokenIndex() > context.BRANCH().getSymbol().getTokenIndex())); } + @Override + public Node visitCreateBranch(SqlBaseParser.CreateBranchContext context) + { + boolean tableExists = context.EXISTS().stream() + .anyMatch(node -> node.getSymbol().getTokenIndex() > context.TABLE().getSymbol().getTokenIndex() && + node.getSymbol().getTokenIndex() < context.CREATE().getSymbol().getTokenIndex()); + boolean replace = context.REPLACE() != null; + boolean ifNotExists = context.EXISTS().stream() + .anyMatch(node -> node.getSymbol().getTokenIndex() > context.BRANCH().getSymbol().getTokenIndex()); + + Optional tableVersion = context.tableVersionExpression() != null + ? Optional.of((TableVersionExpression) visit(context.tableVersionExpression())) + : Optional.empty(); + + Optional retainDays = context.retainDays != null + ? Optional.of(Long.parseLong(context.retainDays.getText())) + : Optional.empty(); + + Optional minSnapshotsToKeep = context.minSnapshots != null + ? Optional.of(Integer.parseInt(context.minSnapshots.getText())) + : Optional.empty(); + + Optional maxSnapshotAgeDays = context.maxSnapshotAge != null + ? Optional.of(Long.parseLong(context.maxSnapshotAge.getText())) + : Optional.empty(); + + return new CreateBranch( + getLocation(context), + getQualifiedName(context.tableName), + tableExists, + replace, + ifNotExists, + ((StringLiteral) visit(context.name)).getValue(), + tableVersion, + retainDays, + minSnapshotsToKeep, + maxSnapshotAgeDays); + } + + @Override + public Node visitCreateTag(SqlBaseParser.CreateTagContext context) + { + boolean tableExists = context.EXISTS().stream() + .anyMatch(node -> node.getSymbol().getTokenIndex() > context.TABLE().getSymbol().getTokenIndex() && + node.getSymbol().getTokenIndex() < context.CREATE().getSymbol().getTokenIndex()); + boolean replace = context.REPLACE() != null; + boolean ifNotExists = context.EXISTS().stream() + .anyMatch(node -> node.getSymbol().getTokenIndex() > context.TAG().getSymbol().getTokenIndex()); + + Optional tableVersion = context.tableVersionExpression() != null + ? Optional.of((TableVersionExpression) visit(context.tableVersionExpression())) + : Optional.empty(); + + Optional retainDays = context.retainDays != null + ? Optional.of(Long.parseLong(context.retainDays.getText())) + : Optional.empty(); + + return new CreateTag( + getLocation(context), + getQualifiedName(context.tableName), + tableExists, + replace, + ifNotExists, + ((StringLiteral) visit(context.name)).getValue(), + tableVersion, + retainDays); + } + @Override public Node visitDropTag(SqlBaseParser.DropTagContext context) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java index 6d250a6e22867..40ae3d7398bc1 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java @@ -627,6 +627,16 @@ protected R visitDropBranch(DropBranch node, C context) return visitStatement(node, context); } + protected R visitCreateBranch(CreateBranch node, C context) + { + return visitStatement(node, context); + } + + protected R visitCreateTag(CreateTag node, C context) + { + return visitStatement(node, context); + } + protected R visitDropTag(DropTag node, C context) { return visitStatement(node, context); diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateBranch.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateBranch.java new file mode 100644 index 0000000000000..d148c3b43fcc5 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateBranch.java @@ -0,0 +1,192 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreateBranch + extends Statement +{ + private final QualifiedName tableName; + private final boolean tableExists; + private final boolean replace; + private final boolean ifNotExists; + private final String branchName; + private final Optional tableVersion; + private final Optional retainDays; + private final Optional minSnapshotsToKeep; + private final Optional maxSnapshotAgeDays; + + public CreateBranch( + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String branchName, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + this(Optional.empty(), tableName, tableExists, replace, ifNotExists, branchName, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + + public CreateBranch( + NodeLocation location, + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String branchName, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + this(Optional.of(location), tableName, tableExists, replace, ifNotExists, branchName, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + + private CreateBranch( + Optional location, + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String branchName, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + super(location); + this.tableName = requireNonNull(tableName, "table is null"); + this.tableExists = tableExists; + this.replace = replace; + this.ifNotExists = ifNotExists; + this.branchName = requireNonNull(branchName, "branchName is null"); + this.tableVersion = requireNonNull(tableVersion, "tableVersion is null"); + this.retainDays = requireNonNull(retainDays, "retainDays is null"); + this.minSnapshotsToKeep = requireNonNull(minSnapshotsToKeep, "minSnapshotsToKeep is null"); + this.maxSnapshotAgeDays = requireNonNull(maxSnapshotAgeDays, "maxSnapshotAgeDays is null"); + } + + public QualifiedName getTableName() + { + return tableName; + } + + public boolean isTableExists() + { + return tableExists; + } + + public boolean isReplace() + { + return replace; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + public String getBranchName() + { + return branchName; + } + + public Optional getTableVersion() + { + return tableVersion; + } + + public Optional getRetainDays() + { + return retainDays; + } + + public Optional getMinSnapshotsToKeep() + { + return minSnapshotsToKeep; + } + + public Optional getMaxSnapshotAgeDays() + { + return maxSnapshotAgeDays; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitCreateBranch(this, context); + } + + @Override + public List getChildren() + { + ImmutableList.Builder children = ImmutableList.builder(); + tableVersion.ifPresent(children::add); + return children.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CreateBranch that = (CreateBranch) o; + return tableExists == that.tableExists && + replace == that.replace && + ifNotExists == that.ifNotExists && + Objects.equals(tableName, that.tableName) && + Objects.equals(branchName, that.branchName) && + Objects.equals(tableVersion, that.tableVersion) && + Objects.equals(retainDays, that.retainDays) && + Objects.equals(minSnapshotsToKeep, that.minSnapshotsToKeep) && + Objects.equals(maxSnapshotAgeDays, that.maxSnapshotAgeDays); + } + + @Override + public int hashCode() + { + return Objects.hash(tableName, tableExists, replace, ifNotExists, branchName, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("table", tableName) + .add("tableExists", tableExists) + .add("replace", replace) + .add("ifNotExists", ifNotExists) + .add("branchName", branchName) + .add("tableVersion", tableVersion) + .add("retainDays", retainDays) + .add("minSnapshotsToKeep", minSnapshotsToKeep) + .add("maxSnapshotAgeDays", maxSnapshotAgeDays) + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateTag.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateTag.java new file mode 100644 index 0000000000000..39754974839f0 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/CreateTag.java @@ -0,0 +1,168 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreateTag + extends Statement +{ + private final QualifiedName tableName; + private final boolean tableExists; + private final boolean replace; + private final boolean ifNotExists; + private final String tagName; + private final Optional tableVersion; + private final Optional retainDays; + + public CreateTag( + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String tagName, + Optional tableVersion, + Optional retainDays) + { + this(Optional.empty(), tableName, tableExists, replace, ifNotExists, tagName, tableVersion, retainDays); + } + + public CreateTag( + NodeLocation location, + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String tagName, + Optional tableVersion, + Optional retainDays) + { + this(Optional.of(location), tableName, tableExists, replace, ifNotExists, tagName, tableVersion, retainDays); + } + + private CreateTag( + Optional location, + QualifiedName tableName, + boolean tableExists, + boolean replace, + boolean ifNotExists, + String tagName, + Optional tableVersion, + Optional retainDays) + { + super(location); + this.tableName = requireNonNull(tableName, "table is null"); + this.tableExists = tableExists; + this.replace = replace; + this.ifNotExists = ifNotExists; + this.tagName = requireNonNull(tagName, "tagName is null"); + this.tableVersion = requireNonNull(tableVersion, "tableVersion is null"); + this.retainDays = requireNonNull(retainDays, "retainDays is null"); + } + + public QualifiedName getTableName() + { + return tableName; + } + + public boolean isTableExists() + { + return tableExists; + } + + public boolean isReplace() + { + return replace; + } + + public boolean isIfNotExists() + { + return ifNotExists; + } + + public String getTagName() + { + return tagName; + } + + public Optional getTableVersion() + { + return tableVersion; + } + + public Optional getRetainDays() + { + return retainDays; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitCreateTag(this, context); + } + + @Override + public List getChildren() + { + ImmutableList.Builder children = ImmutableList.builder(); + tableVersion.ifPresent(children::add); + return children.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CreateTag that = (CreateTag) o; + return tableExists == that.tableExists && + replace == that.replace && + ifNotExists == that.ifNotExists && + Objects.equals(tableName, that.tableName) && + Objects.equals(tagName, that.tagName) && + Objects.equals(tableVersion, that.tableVersion) && + Objects.equals(retainDays, that.retainDays); + } + + @Override + public int hashCode() + { + return Objects.hash(tableName, tableExists, replace, ifNotExists, tagName, tableVersion, retainDays); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("table", tableName) + .add("tableExists", tableExists) + .add("replace", replace) + .add("ifNotExists", ifNotExists) + .add("tagName", tagName) + .add("tableVersion", tableVersion) + .add("retainDays", retainDays) + .toString(); + } +} diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java index 8586b26022072..01a60fce95955 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java @@ -36,12 +36,14 @@ import com.facebook.presto.sql.tree.Commit; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.ConstraintSpecification; +import com.facebook.presto.sql.tree.CreateBranch; import com.facebook.presto.sql.tree.CreateFunction; import com.facebook.presto.sql.tree.CreateMaterializedView; import com.facebook.presto.sql.tree.CreateRole; import com.facebook.presto.sql.tree.CreateSchema; import com.facebook.presto.sql.tree.CreateTable; import com.facebook.presto.sql.tree.CreateTableAsSelect; +import com.facebook.presto.sql.tree.CreateTag; import com.facebook.presto.sql.tree.CreateView; import com.facebook.presto.sql.tree.Cube; import com.facebook.presto.sql.tree.CurrentTime; @@ -2877,6 +2879,78 @@ public void testDropBranch() assertStatement("ALTER TABLE IF EXISTS foo.t DROP BRANCH IF EXISTS 'cons'", new DropBranch(QualifiedName.of("foo", "t"), "cons", true, true)); } + @Test + public void testCreateBranch() + { + assertStatement("ALTER TABLE foo.t CREATE BRANCH 'test_branch'", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, false, "test_branch", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE IF EXISTS foo.t CREATE BRANCH 'test_branch'", + new CreateBranch(QualifiedName.of("foo", "t"), true, false, false, "test_branch", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH IF NOT EXISTS 'test_branch'", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, true, "test_branch", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE BRANCH 'test_branch'", + new CreateBranch(QualifiedName.of("foo", "t"), false, true, false, "test_branch", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE IF EXISTS foo.t CREATE OR REPLACE BRANCH 'test_branch'", + new CreateBranch(QualifiedName.of("foo", "t"), true, true, false, "test_branch", Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH IF NOT EXISTS 'test_branch' FOR SYSTEM_VERSION AS OF 123", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, true, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123", + new CreateBranch(QualifiedName.of("foo", "t"), false, true, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH 'test_branch' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, false, "test_branch", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH IF NOT EXISTS 'test_branch' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, true, "test_branch", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE BRANCH 'test_branch' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateBranch(QualifiedName.of("foo", "t"), false, true, false, "test_branch", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty(), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH IF NOT EXISTS 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, true, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, true, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 3 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.of(2), Optional.of(3L))); + assertStatement("ALTER TABLE foo.t CREATE BRANCH IF NOT EXISTS 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 3 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, false, true, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.of(2), Optional.of(3L))); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE BRANCH 'test_branch' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS WITH SNAPSHOT RETENTION 2 SNAPSHOTS 3 DAYS", + new CreateBranch(QualifiedName.of("foo", "t"), false, true, false, "test_branch", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L), Optional.of(2), Optional.of(3L))); + } + + @Test + public void testCreateTag() + { + assertStatement("ALTER TABLE foo.t CREATE TAG 'test_tag'", + new CreateTag(QualifiedName.of("foo", "t"), false, false, false, "test_tag", Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE IF EXISTS foo.t CREATE TAG 'test_tag'", + new CreateTag(QualifiedName.of("foo", "t"), true, false, false, "test_tag", Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG IF NOT EXISTS 'test_tag'", + new CreateTag(QualifiedName.of("foo", "t"), false, false, true, "test_tag", Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE TAG 'test_tag'", + new CreateTag(QualifiedName.of("foo", "t"), false, true, false, "test_tag", Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE IF EXISTS foo.t CREATE OR REPLACE TAG 'test_tag'", + new CreateTag(QualifiedName.of("foo", "t"), true, true, false, "test_tag", Optional.empty(), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG 'test_tag' FOR SYSTEM_VERSION AS OF 123", + new CreateTag(QualifiedName.of("foo", "t"), false, false, false, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG IF NOT EXISTS 'test_tag' FOR SYSTEM_VERSION AS OF 123", + new CreateTag(QualifiedName.of("foo", "t"), false, false, true, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE TAG 'test_tag' FOR SYSTEM_VERSION AS OF 123", + new CreateTag(QualifiedName.of("foo", "t"), false, true, false, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG 'test_tag' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateTag(QualifiedName.of("foo", "t"), false, false, false, "test_tag", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG IF NOT EXISTS 'test_tag' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateTag(QualifiedName.of("foo", "t"), false, false, true, "test_tag", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE TAG 'test_tag' FOR SYSTEM_TIME AS OF TIMESTAMP '2024-01-01 00:00:00'", + new CreateTag(QualifiedName.of("foo", "t"), false, true, false, "test_tag", Optional.of(new TableVersionExpression(TIMESTAMP, TableVersionExpression.TableVersionOperator.EQUAL, new TimestampLiteral("2024-01-01 00:00:00"))), Optional.empty())); + assertStatement("ALTER TABLE foo.t CREATE TAG 'test_tag' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateTag(QualifiedName.of("foo", "t"), false, false, false, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L))); + assertStatement("ALTER TABLE foo.t CREATE TAG IF NOT EXISTS 'test_tag' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateTag(QualifiedName.of("foo", "t"), false, false, true, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L))); + assertStatement("ALTER TABLE foo.t CREATE OR REPLACE TAG 'test_tag' FOR SYSTEM_VERSION AS OF 123 RETAIN 7 DAYS", + new CreateTag(QualifiedName.of("foo", "t"), false, true, false, "test_tag", Optional.of(new TableVersionExpression(VERSION, TableVersionExpression.TableVersionOperator.EQUAL, new LongLiteral("123"))), Optional.of(7L))); + } + @Test public void testDropTag() { diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java index e684a3d9395c4..8b1dcc5845d03 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java @@ -209,6 +209,33 @@ public void testStatementBuilder() printStatement("alter table a.b.c drop column x"); printStatement("alter table a.b.c drop branch 'x'"); + printStatement("alter table a.b.c create branch 'test_branch'"); + printStatement("alter table a.b.c create branch if not exists 'test_branch'"); + printStatement("alter table a.b.c create or replace branch 'test_branch'"); + printStatement("alter table a.b.c create branch 'test_branch' for system_version as of 123"); + printStatement("alter table a.b.c create branch if not exists 'test_branch' for system_version as of 123"); + printStatement("alter table a.b.c create or replace branch 'test_branch' for system_version as of 123"); + printStatement("alter table a.b.c create branch 'test_branch' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create branch if not exists 'test_branch' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create or replace branch 'test_branch' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create branch 'test_branch' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create branch if not exists 'test_branch' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create or replace branch 'test_branch' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create branch 'test_branch' for system_version as of 123 retain 7 days with snapshot retention 2 snapshots 3 days"); + printStatement("alter table a.b.c create branch if not exists 'test_branch' for system_version as of 123 retain 7 days with snapshot retention 2 snapshots 3 days"); + printStatement("alter table a.b.c create tag 'test_tag'"); + printStatement("alter table a.b.c create tag if not exists 'test_tag'"); + printStatement("alter table a.b.c create or replace tag 'test_tag'"); + printStatement("alter table a.b.c create tag 'test_tag' for system_version as of 123"); + printStatement("alter table a.b.c create tag if not exists 'test_tag' for system_version as of 123"); + printStatement("alter table a.b.c create or replace tag 'test_tag' for system_version as of 123"); + printStatement("alter table a.b.c create tag 'test_tag' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create tag if not exists 'test_tag' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create or replace tag 'test_tag' for system_time as of timestamp '2024-01-01 00:00:00'"); + printStatement("alter table a.b.c create tag 'test_tag' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create tag if not exists 'test_tag' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create or replace tag 'test_tag' for system_version as of 123 retain 7 days"); + printStatement("alter table a.b.c create or replace branch 'test_branch' for system_version as of 123 retain 7 days with snapshot retention 2 snapshots 3 days"); printStatement("alter table a.b.c drop tag 'testTag'"); printStatement("create schema test"); diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml index 2168852e9f82b..7dbef0deaa09c 100644 --- a/presto-pinot-toolkit/pom.xml +++ b/presto-pinot-toolkit/pom.xml @@ -108,9 +108,20 @@ org.glassfish.hk2.external aopalliance-repackaged + + org.lz4 + lz4-java + + + + at.yawk.lz4 + lz4-java + runtime + + org.apache.pinot pinot-core diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java index e012dc40ca6b7..6c5a8b76c6232 100644 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotConnectorFactory.java @@ -16,6 +16,7 @@ import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.airlift.json.JsonModule; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.expressions.LogicalRowExpressions; import com.facebook.presto.pinot.auth.PinotTlsAuthenticationModule; import com.facebook.presto.spi.ConnectorHandleResolver; diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java index d74eb888b5f69..89db03d745770 100644 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/PinotStreamingQueryClient.java @@ -15,8 +15,8 @@ import org.apache.pinot.common.config.GrpcConfig; import org.apache.pinot.common.proto.Server; -import org.apache.pinot.common.utils.grpc.GrpcQueryClient; -import org.apache.pinot.common.utils.grpc.GrpcRequestBuilder; +import org.apache.pinot.common.utils.grpc.ServerGrpcQueryClient; +import org.apache.pinot.common.utils.grpc.ServerGrpcRequestBuilder; import java.util.HashMap; import java.util.Iterator; @@ -30,7 +30,7 @@ */ public class PinotStreamingQueryClient { - private final Map grpcQueryClientMap = new HashMap<>(); + private final Map grpcQueryClientMap = new HashMap<>(); private final GrpcConfig config; public PinotStreamingQueryClient(GrpcConfig config) @@ -38,17 +38,17 @@ public PinotStreamingQueryClient(GrpcConfig config) this.config = config; } - public Iterator submit(String host, int port, GrpcRequestBuilder requestBuilder) + public Iterator submit(String host, int port, ServerGrpcRequestBuilder requestBuilder) { - GrpcQueryClient client = getOrCreateGrpcQueryClient(host, port); + ServerGrpcQueryClient client = getOrCreateGrpcQueryClient(host, port); return client.submit(requestBuilder.build()); } - private GrpcQueryClient getOrCreateGrpcQueryClient(String host, int port) + private ServerGrpcQueryClient getOrCreateGrpcQueryClient(String host, int port) { String key = String.format("%s_%d", host, port); if (!grpcQueryClientMap.containsKey(key)) { - grpcQueryClientMap.put(key, new GrpcQueryClient(host, port, config)); + grpcQueryClientMap.put(key, new ServerGrpcQueryClient(host, port, config)); } return grpcQueryClientMap.get(key); } diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java deleted file mode 100644 index 37e63d2ee8e3f..0000000000000 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/RebindSafeMBeanServer.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.pinot; - -import com.google.errorprone.annotations.ThreadSafe; - -import javax.management.Attribute; -import javax.management.AttributeList; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceAlreadyExistsException; -import javax.management.InstanceNotFoundException; -import javax.management.IntrospectionException; -import javax.management.InvalidAttributeValueException; -import javax.management.ListenerNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanInfo; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.NotCompliantMBeanException; -import javax.management.NotificationFilter; -import javax.management.NotificationListener; -import javax.management.ObjectInstance; -import javax.management.ObjectName; -import javax.management.OperationsException; -import javax.management.QueryExp; -import javax.management.ReflectionException; -import javax.management.loading.ClassLoaderRepository; - -import java.io.ObjectInputStream; -import java.util.Set; - -/** - * MBeanServer wrapper that a ignores calls to registerMBean when there is already - * a MBean registered with the specified object name. - *

- * This originally existed in hive, raptor and cassandra and I am promoting it to SPI - */ -@ThreadSafe -public class RebindSafeMBeanServer - implements MBeanServer -{ - private final MBeanServer mbeanServer; - - public RebindSafeMBeanServer(MBeanServer mbeanServer) - { - this.mbeanServer = mbeanServer; - } - - /** - * Delegates to the wrapped mbean server, but if a mbean is already registered - * with the specified name, the existing instance is returned. - */ - @Override - public ObjectInstance registerMBean(Object object, ObjectName name) - throws MBeanRegistrationException, NotCompliantMBeanException - { - while (true) { - try { - // try to register the mbean - return mbeanServer.registerMBean(object, name); - } - catch (InstanceAlreadyExistsException ignored) { - } - - try { - // a mbean is already installed, try to return the already registered instance - ObjectInstance objectInstance = mbeanServer.getObjectInstance(name); - return objectInstance; - } - catch (InstanceNotFoundException ignored) { - // the mbean was removed before we could get the reference - // start the whole process over again - } - } - } - - @Override - public void unregisterMBean(ObjectName name) - throws InstanceNotFoundException, MBeanRegistrationException - { - mbeanServer.unregisterMBean(name); - } - - @Override - public ObjectInstance getObjectInstance(ObjectName name) - throws InstanceNotFoundException - { - return mbeanServer.getObjectInstance(name); - } - - @Override - public Set queryMBeans(ObjectName name, QueryExp query) - { - return mbeanServer.queryMBeans(name, query); - } - - @Override - public Set queryNames(ObjectName name, QueryExp query) - { - return mbeanServer.queryNames(name, query); - } - - @Override - public boolean isRegistered(ObjectName name) - { - return mbeanServer.isRegistered(name); - } - - @Override - public Integer getMBeanCount() - { - return mbeanServer.getMBeanCount(); - } - - @Override - public Object getAttribute(ObjectName name, String attribute) - throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttribute(name, attribute); - } - - @Override - public AttributeList getAttributes(ObjectName name, String[] attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttributes(name, attributes); - } - - @Override - public void setAttribute(ObjectName name, Attribute attribute) - throws InstanceNotFoundException, AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException - { - mbeanServer.setAttribute(name, attribute); - } - - @Override - public AttributeList setAttributes(ObjectName name, AttributeList attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.setAttributes(name, attributes); - } - - @Override - public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) - throws InstanceNotFoundException, MBeanException, ReflectionException - { - return mbeanServer.invoke(name, operationName, params, signature); - } - - @Override - public String getDefaultDomain() - { - return mbeanServer.getDefaultDomain(); - } - - @Override - public String[] getDomains() - { - return mbeanServer.getDomains(); - } - - @Override - public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public MBeanInfo getMBeanInfo(ObjectName name) - throws InstanceNotFoundException, IntrospectionException, ReflectionException - { - return mbeanServer.getMBeanInfo(name); - } - - @Override - public boolean isInstanceOf(ObjectName name, String className) - throws InstanceNotFoundException - { - return mbeanServer.isInstanceOf(name, className); - } - - @Override - public Object instantiate(String className) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className); - } - - @Override - public Object instantiate(String className, ObjectName loaderName) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName); - } - - @Override - public Object instantiate(String className, Object[] params, String[] signature) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className, params, signature); - } - - @Override - public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName, params, signature); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(ObjectName name, byte[] data) - throws OperationsException - { - return mbeanServer.deserialize(name, data); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(String className, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, data); - } - - @Override - @Deprecated - @SuppressWarnings("deprecation") - public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, loaderName, data); - } - - @Override - public ClassLoader getClassLoaderFor(ObjectName mbeanName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoaderFor(mbeanName); - } - - @Override - public ClassLoader getClassLoader(ObjectName loaderName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoader(loaderName); - } - - @Override - public ClassLoaderRepository getClassLoaderRepository() - { - return mbeanServer.getClassLoaderRepository(); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name, params, signature); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName, params, signature); - } -} diff --git a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProxyGrpcRequestBuilder.java b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProxyGrpcRequestBuilder.java index e68dfd0170358..88afea80be2d7 100644 --- a/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProxyGrpcRequestBuilder.java +++ b/presto-pinot-toolkit/src/main/java/com/facebook/presto/pinot/query/PinotProxyGrpcRequestBuilder.java @@ -16,7 +16,7 @@ import com.facebook.presto.pinot.PinotErrorCode; import com.facebook.presto.pinot.PinotException; import org.apache.pinot.common.proto.Server; -import org.apache.pinot.common.utils.grpc.GrpcRequestBuilder; +import org.apache.pinot.common.utils.grpc.ServerGrpcRequestBuilder; import org.apache.pinot.spi.utils.CommonConstants; import java.util.HashMap; @@ -25,7 +25,7 @@ import java.util.Optional; public class PinotProxyGrpcRequestBuilder - extends GrpcRequestBuilder + extends ServerGrpcRequestBuilder { private static final String KEY_OF_PROXY_GRPC_FORWARD_HOST = "FORWARD_HOST"; private static final String KEY_OF_PROXY_GRPC_FORWARD_PORT = "FORWARD_PORT"; diff --git a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java index a6b98514a191d..56a9676d7e655 100644 --- a/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java +++ b/presto-pinot-toolkit/src/test/java/com/facebook/presto/pinot/TestPinotSegmentPageSource.java @@ -34,11 +34,12 @@ import org.apache.pinot.common.datatable.DataTable; import org.apache.pinot.common.proto.Server; import org.apache.pinot.common.utils.DataSchema; -import org.apache.pinot.common.utils.grpc.GrpcRequestBuilder; +import org.apache.pinot.common.utils.grpc.ServerGrpcRequestBuilder; import org.apache.pinot.core.common.datatable.DataTableBuilder; import org.apache.pinot.core.common.datatable.DataTableBuilderV4; import org.apache.pinot.spi.data.DimensionFieldSpec; import org.apache.pinot.spi.data.FieldSpec; +import org.apache.pinot.spi.utils.ByteArray; import org.apache.pinot.spi.utils.CommonConstants; import org.testng.annotations.Test; @@ -172,9 +173,6 @@ protected static DataTable createDataTableWithAllTypes() case STRING: dataTableBuilder.setColumn(colId, generateRandomStringWithLength(RANDOM.nextInt(20))); break; - case OBJECT: - dataTableBuilder.setColumn(colId, (Object) RANDOM.nextDouble()); - break; case BOOLEAN_ARRAY: int length = RANDOM.nextInt(20); int[] booleanArray = new int[length]; @@ -233,7 +231,7 @@ protected static DataTable createDataTableWithAllTypes() case BYTES: try { dataTableBuilder.setColumn(colId, - Hex.decodeHex("0DE0B6B3A7640000".toCharArray())); // Hex of BigDecimal.ONE + new ByteArray(Hex.decodeHex("0DE0B6B3A7640000".toCharArray()))); // Hex of BigDecimal.ONE } catch (DecoderException e) { throw new RuntimeException(e); @@ -425,7 +423,7 @@ public void testPinotProxyGrpcRequest() @Test public void testPinotGrpcRequest() { - final Server.ServerRequest grpcRequest = new GrpcRequestBuilder() + final Server.ServerRequest grpcRequest = new ServerGrpcRequestBuilder() .setSegments(ImmutableList.of("segment1")) .setEnableStreaming(true) .setRequestId(121) @@ -435,12 +433,13 @@ public void testPinotGrpcRequest() Assert.assertEquals(grpcRequest.getSql(), "SELECT * FROM myTable"); Assert.assertEquals(grpcRequest.getSegmentsCount(), 1); Assert.assertEquals(grpcRequest.getSegments(0), "segment1"); - Assert.assertEquals(grpcRequest.getMetadataCount(), 5); + Assert.assertEquals(grpcRequest.getMetadataCount(), 6); Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.REQUEST_ID), "121"); Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.BROKER_ID), "presto-coordinator-grpc"); Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.ENABLE_TRACE), "false"); Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.ENABLE_STREAMING), "true"); Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.PAYLOAD_TYPE), "sql"); + Assert.assertEquals(grpcRequest.getMetadataOrThrow(CommonConstants.Query.Request.MetadataKeys.CORRELATION_ID), "121"); } private static final class TestingPinotStreamingQueryClient @@ -455,7 +454,7 @@ private static final class TestingPinotStreamingQueryClient } @Override - public Iterator submit(String host, int port, GrpcRequestBuilder requestBuilder) + public Iterator submit(String host, int port, ServerGrpcRequestBuilder requestBuilder) { return new Iterator() { diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/AllowAllAccessControl.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/AllowAllAccessControl.java index c1581f12ba5ee..3150a1cf42d9c 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/AllowAllAccessControl.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/AllowAllAccessControl.java @@ -232,6 +232,16 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con { } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControl.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControl.java index bb6db20397a61..dfe595ddf248d 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControl.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/FileBasedAccessControl.java @@ -46,8 +46,10 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddColumn; import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -378,6 +380,22 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con } } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + if (!checkTablePermission(identity, tableName, OWNERSHIP)) { + denyCreateBranch(tableName.toString()); + } + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + if (!checkTablePermission(identity, tableName, OWNERSHIP)) { + denyCreateTag(tableName.toString()); + } + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingConnectorAccessControl.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingConnectorAccessControl.java index df55228520809..000d577c39622 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingConnectorAccessControl.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingConnectorAccessControl.java @@ -284,6 +284,18 @@ public void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Con delegate().checkCanDropBranch(transactionHandle, identity, context, tableName); } + @Override + public void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + delegate().checkCanCreateBranch(transactionHandle, identity, context, tableName); + } + + @Override + public void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + delegate().checkCanCreateTag(transactionHandle, identity, context, tableName); + } + @Override public void checkCanDropTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) { diff --git a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingSystemAccessControl.java b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingSystemAccessControl.java index 5df3cc4947b76..aa64cb7f8e016 100644 --- a/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingSystemAccessControl.java +++ b/presto-plugin-toolkit/src/main/java/com/facebook/presto/plugin/base/security/ForwardingSystemAccessControl.java @@ -272,6 +272,18 @@ public void checkCanRevokeTablePrivilege(Identity identity, AccessControlContext delegate().checkCanRevokeTablePrivilege(identity, context, privilege, table, revokee, grantOptionFor); } + @Override + public void checkCanCreateBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + delegate().checkCanCreateBranch(identity, context, table); + } + + @Override + public void checkCanCreateTag(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + delegate().checkCanCreateTag(identity, context, table); + } + @Override public void checkCanDropBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) { diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 45940a4f9ae65..128891cbdfd42 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -175,7 +175,7 @@ org.testcontainers - postgresql + testcontainers-postgresql test diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java index 4e33184bd8b61..66e08547b005b 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/PostgreSqlQueryRunner.java @@ -19,7 +19,7 @@ import com.facebook.presto.tpch.TpchPlugin; import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import java.sql.Connection; import java.sql.DriverManager; @@ -93,7 +93,7 @@ private static void createSchema(String url, String schema, String username, Str } } - public static Properties createJdbcProperties(PostgreSQLContainer container) + public static Properties createJdbcProperties(PostgreSQLContainer container) { Properties properties = new Properties(); properties.setProperty("user", container.getUsername()); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java index 31eebf582d999..745abe74a3f22 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseInsensitiveMapping.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -37,11 +37,11 @@ public class TestPostgreSqlCaseInsensitiveMapping extends AbstractTestQueryFramework { - private final PostgreSQLContainer postgresContainer; + private final PostgreSQLContainer postgresContainer; public TestPostgreSqlCaseInsensitiveMapping() { - this.postgresContainer = new PostgreSQLContainer<>("postgres:14") + this.postgresContainer = new PostgreSQLContainer("postgres:14") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseSensitiveMapping.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseSensitiveMapping.java index bdceefb6fdba2..a93514aaa18d5 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseSensitiveMapping.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlCaseSensitiveMapping.java @@ -17,7 +17,7 @@ import com.facebook.presto.tests.AbstractTestQueryFramework; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -31,11 +31,11 @@ public class TestPostgreSqlCaseSensitiveMapping extends AbstractTestQueryFramework { - private final PostgreSQLContainer postgresContainer; + private final PostgreSQLContainer postgresContainer; public TestPostgreSqlCaseSensitiveMapping() { - this.postgresContainer = new PostgreSQLContainer<>("postgres:14") + this.postgresContainer = new PostgreSQLContainer("postgres:14") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java index 215944f1a94be..127acf3d0fa80 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlDistributedQueries.java @@ -17,7 +17,7 @@ import com.facebook.presto.tests.AbstractTestDistributedQueries; import com.google.common.collect.ImmutableMap; import io.airlift.tpch.TpchTable; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -27,11 +27,11 @@ public class TestPostgreSqlDistributedQueries extends AbstractTestDistributedQueries { - private final PostgreSQLContainer postgresContainer; + private PostgreSQLContainer postgresContainer; public TestPostgreSqlDistributedQueries() { - this.postgresContainer = new PostgreSQLContainer<>("postgres:14") + this.postgresContainer = new PostgreSQLContainer("postgres:14") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java index 5d33801ab9492..39e4a24ed58da 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlIntegrationSmokeTest.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -42,11 +42,11 @@ public class TestPostgreSqlIntegrationSmokeTest extends AbstractTestIntegrationSmokeTest { - private final PostgreSQLContainer postgresContainer; + private PostgreSQLContainer postgresContainer; public TestPostgreSqlIntegrationSmokeTest() { - this.postgresContainer = new PostgreSQLContainer<>("postgres:14") + this.postgresContainer = new PostgreSQLContainer("postgres:14") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java index f5c901c745699..21a9999e8d242 100644 --- a/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java +++ b/presto-postgresql/src/test/java/com/facebook/presto/plugin/postgresql/TestPostgreSqlTypeMapping.java @@ -26,7 +26,7 @@ import com.facebook.presto.tests.sql.PrestoSqlExecutor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testng.annotations.AfterClass; import org.testng.annotations.Test; @@ -70,12 +70,12 @@ public class TestPostgreSqlTypeMapping extends AbstractTestQueryFramework { - private final PostgreSQLContainer postgresContainer; + private final PostgreSQLContainer postgresContainer; public TestPostgreSqlTypeMapping() throws Exception { - this.postgresContainer = new PostgreSQLContainer<>("postgres:14") + this.postgresContainer = new PostgreSQLContainer("postgres:14") .withDatabaseName("tpch") .withUsername("testuser") .withPassword("testpass"); diff --git a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh index b4a82d7d4ce1e..e47e174d0a03b 100755 --- a/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh +++ b/presto-product-tests/conf/docker/files/presto-launcher-wrapper.sh @@ -25,8 +25,10 @@ if [ -d /docker/volumes/overridejdk ]; then export PATH=$JAVA_HOME/bin:$PATH if [[ "$CONFIG" == "singlenode-ldap" ]]; then - # For LDAP tests use the cacert file from the container JDK which has certs installed - JAVA_PROPERTIES="-Djavax.net.ssl.trustStore=$CONTAINER_JAVA_HOME/jre/lib/security/cacerts" + # For LDAP tests use the cacert file from the container JDK which has certs installed. + # Also relax TLS restrictions for compatibility with the CentOS 7 OpenLDAP server, + # which requires TLS_RSA cipher suites disabled in JDK 17.0.18+ (JDK-8344257). + JAVA_PROPERTIES="-Djavax.net.ssl.trustStore=$CONTAINER_JAVA_HOME/jre/lib/security/cacerts -Djava.security.properties=${PRESTO_CONFIG_DIRECTORY}/ldap-jdk-security.properties" fi fi diff --git a/presto-product-tests/conf/presto/etc/ldap-jdk-security.properties b/presto-product-tests/conf/presto/etc/ldap-jdk-security.properties new file mode 100644 index 0000000000000..3518e0bed01ea --- /dev/null +++ b/presto-product-tests/conf/presto/etc/ldap-jdk-security.properties @@ -0,0 +1,7 @@ +# Re-enable TLS_RSA cipher suites for LDAP product tests. +# JDK 17.0.18+ disabled TLS_RSA cipher suites (JDK-8344257) and SHA-1 +# handshake signatures (JDK-8353879), but the CentOS 7 OpenLDAP server +# in the test Docker image uses OpenSSL 1.0.2k which may require them. +# This restores the JDK 17.0.15 behavior for jdk.tls.disabledAlgorithms. +jdk.tls.disabledAlgorithms=SSLv3, TLSv1, TLSv1.1, DTLSv1.0, RC4, DES, \ + MD5withRSA, DH keySize < 1024, EC keySize < 224, 3DES_EDE_CBC, anon, NULL \ No newline at end of file diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/iceberg/TestIcebergHiveMetadataListing.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/iceberg/TestIcebergHiveMetadataListing.java index 60c89225ad1b4..b5418cd6dcbc0 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/iceberg/TestIcebergHiveMetadataListing.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/iceberg/TestIcebergHiveMetadataListing.java @@ -46,6 +46,8 @@ public void setUp() .map(list -> row(list.toArray())) .collect(Collectors.toList()); onPresto().executeQuery("CREATE TABLE iceberg.default.iceberg_table1 (_string VARCHAR, _integer INTEGER)"); + onPresto().executeQuery("INSERT into iceberg.default.iceberg_table1 values('1001', 1), ('1002', 2)"); + onPresto().executeQuery("CREATE TABLE iceberg.default.iceberg_table2 as select * from iceberg.default.iceberg_table1"); onPresto().executeQuery("CREATE TABLE hive.default.hive_table (_double DOUBLE)"); onPresto().executeQuery("CREATE VIEW hive.default.hive_view AS SELECT * FROM hive.default.hive_table"); onPresto().executeQuery("CREATE VIEW iceberg.default.iceberg_view AS SELECT * FROM iceberg.default.iceberg_table1"); @@ -58,6 +60,7 @@ public void cleanUp() onPresto().executeQuery("DROP VIEW IF EXISTS hive.default.hive_view"); onPresto().executeQuery("DROP VIEW IF EXISTS iceberg.default.iceberg_view"); onPresto().executeQuery("DROP TABLE IF EXISTS iceberg.default.iceberg_table1"); + onPresto().executeQuery("DROP TABLE IF EXISTS iceberg.default.iceberg_table2"); } @Test(groups = {ICEBERG, STORAGE_FORMATS}) @@ -66,6 +69,7 @@ public void testTableListing() assertThat(onPresto().executeQuery("SHOW TABLES FROM iceberg.default")) .containsOnly(ImmutableList.builder() .addAll(preexistingTables) + .add(row("iceberg_table2")) .add(row("iceberg_table1")) .add(row("iceberg_view")) .add(row("hive_view")) @@ -81,6 +85,8 @@ public void testColumnListing() "WHERE table_catalog = 'iceberg' AND table_schema = 'default'")) .containsOnly(ImmutableList.builder() .addAll(preexistingColumns) + .add(row("iceberg_table2", "_string")) + .add(row("iceberg_table2", "_integer")) .add(row("iceberg_table1", "_string")) .add(row("iceberg_table1", "_integer")) .add(row("iceberg_view", "_string")) diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 5d087e3decabc..481b87a26b982 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -55,7 +55,6 @@ org.xerial.snappy snappy-java - 1.1.10.4 runtime diff --git a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/reloading/TestReloadingResourceGroupConfigurationManager.java b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/reloading/TestReloadingResourceGroupConfigurationManager.java index 065fbfd740daf..760cd1a0b25ae 100644 --- a/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/reloading/TestReloadingResourceGroupConfigurationManager.java +++ b/presto-resource-group-managers/src/test/java/com/facebook/presto/resourceGroups/reloading/TestReloadingResourceGroupConfigurationManager.java @@ -17,6 +17,7 @@ import com.facebook.airlift.units.Duration; import com.facebook.presto.execution.ClusterOverloadConfig; import com.facebook.presto.execution.resourceGroups.InternalResourceGroup; +import com.facebook.presto.execution.resourceGroups.QueryPacingContext; import com.facebook.presto.execution.scheduler.clusterOverload.ClusterOverloadPolicy; import com.facebook.presto.execution.scheduler.clusterOverload.ClusterResourceChecker; import com.facebook.presto.metadata.InMemoryNodeManager; @@ -78,7 +79,7 @@ public void testConfiguration() DbManagerSpecProvider dbManagerSpecProvider = new DbManagerSpecProvider(daoProvider.get(), ENVIRONMENT, new ReloadingResourceGroupConfig()); ReloadingResourceGroupConfigurationManager manager = new ReloadingResourceGroupConfigurationManager((poolId, listener) -> {}, new ReloadingResourceGroupConfig(), dbManagerSpecProvider); AtomicBoolean exported = new AtomicBoolean(); - InternalResourceGroup global = new InternalResourceGroup.RootInternalResourceGroup("global", (group, export) -> exported.set(export), directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker()); + InternalResourceGroup global = new InternalResourceGroup.RootInternalResourceGroup("global", (group, export) -> exported.set(export), directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); manager.configure(global, new SelectionContext<>(global.getId(), new VariableMap(ImmutableMap.of("USER", "user")))); assertEqualsResourceGroup(global, "1MB", 1000, 100, 100, WEIGHTED, DEFAULT_WEIGHT, true, new Duration(1, HOURS), new Duration(1, DAYS), new ResourceGroupQueryLimits(Optional.of(new Duration(1, HOURS)), Optional.of(new DataSize(1, MEGABYTE)), Optional.of(new Duration(1, HOURS)))); exported.set(false); @@ -101,7 +102,7 @@ public void testMissing() dao.insertSelector(2, 1, null, null, null, null, null, null); DbManagerSpecProvider dbManagerSpecProvider = new DbManagerSpecProvider(daoProvider.get(), ENVIRONMENT, new ReloadingResourceGroupConfig()); ReloadingResourceGroupConfigurationManager manager = new ReloadingResourceGroupConfigurationManager((poolId, listener) -> {}, new ReloadingResourceGroupConfig(), dbManagerSpecProvider); - InternalResourceGroup missing = new InternalResourceGroup.RootInternalResourceGroup("missing", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker()); + InternalResourceGroup missing = new InternalResourceGroup.RootInternalResourceGroup("missing", (group, export) -> {}, directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); manager.configure(missing, new SelectionContext<>(missing.getId(), new VariableMap(ImmutableMap.of("USER", "user")))); } @@ -122,7 +123,7 @@ public void testReconfig() ReloadingResourceGroupConfigurationManager manager = new ReloadingResourceGroupConfigurationManager((poolId, listener) -> {}, new ReloadingResourceGroupConfig(), dbManagerSpecProvider); manager.start(); AtomicBoolean exported = new AtomicBoolean(); - InternalResourceGroup global = new InternalResourceGroup.RootInternalResourceGroup("global", (group, export) -> exported.set(export), directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker()); + InternalResourceGroup global = new InternalResourceGroup.RootInternalResourceGroup("global", (group, export) -> exported.set(export), directExecutor(), ignored -> Optional.empty(), rg -> false, new InMemoryNodeManager(), createClusterResourceChecker(), QueryPacingContext.NOOP); manager.configure(global, new SelectionContext<>(global.getId(), new VariableMap(ImmutableMap.of("USER", "user")))); InternalResourceGroup globalSub = global.getOrCreateSubGroup("sub", true); manager.configure(globalSub, new SelectionContext<>(globalSub.getId(), new VariableMap(ImmutableMap.of("USER", "user")))); diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index b7176785fedd4..f8fc0be809bd6 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -41,6 +41,12 @@ + + + + + + @@ -281,12 +287,6 @@ - - - - - - @@ -371,10 +371,4 @@ - - - - - - diff --git a/presto-singlestore/pom.xml b/presto-singlestore/pom.xml index a99c620ec2d04..4f1f017b9e164 100644 --- a/presto-singlestore/pom.xml +++ b/presto-singlestore/pom.xml @@ -107,7 +107,7 @@ org.testcontainers - jdbc + testcontainers-jdbc test diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java index 066fcb82c0d47..4d58c20f5e46b 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkSessionContext.java @@ -29,6 +29,8 @@ import com.google.common.collect.ImmutableSet; import jakarta.annotation.Nullable; +import java.security.cert.X509Certificate; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -77,7 +79,8 @@ public static PrestoSparkSessionContext createFromSessionInfo( extraCredentials.build(), extraTokenAuthenticators.build(), Optional.empty(), - Optional.empty()), + Optional.empty(), + prestoSparkSession.getCertificates()), prestoSparkSession.getCatalog().orElse(null), prestoSparkSession.getSchema().orElse(null), prestoSparkSession.getSource().orElse(null), @@ -128,6 +131,12 @@ public Identity getIdentity() return identity; } + @Override + public List getCertificates() + { + return identity.getCertificates(); + } + @Nullable @Override public String getCatalog() diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/accesscontrol/PrestoSparkAccessControlCheckerExecution.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/accesscontrol/PrestoSparkAccessControlCheckerExecution.java index 7f0720017fa70..a33656a1a1d33 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/accesscontrol/PrestoSparkAccessControlCheckerExecution.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/accesscontrol/PrestoSparkAccessControlCheckerExecution.java @@ -20,6 +20,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spark.classloader_interface.IPrestoSparkQueryExecution; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; @@ -83,12 +84,14 @@ public List> execute() preparedQuery.getParameters(), parameterExtractor(preparedQuery.getStatement(), preparedQuery.getParameters()), warningCollector, - query); + query, + new ViewDefinitionReferences()); queryStateTimer.beginSemanticAnalyzing(); Analysis analysis = analyzer.analyzeSemantic(preparedQuery.getStatement(), false); queryStateTimer.beginColumnAccessPermissionChecking(); - checkAccessPermissions(analysis.getAccessControlReferences(), query, session.getPreparedStatements()); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); + queryStateTimer.endColumnAccessPermissionChecking(); List> results = new ArrayList<>(); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java index 452ec6394e357..5505e7b6b65ea 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.UpdateInfo; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.plan.OutputNode; import com.facebook.presto.spi.plan.PlanNode; @@ -69,6 +70,7 @@ import static com.facebook.presto.sql.analyzer.utils.ParameterUtils.parameterExtractor; import static com.facebook.presto.sql.analyzer.utils.StatementUtils.getQueryType; import static com.facebook.presto.sql.planner.PlanNodeCanonicalInfo.getCanonicalInfo; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; import static java.util.Objects.requireNonNull; public class PrestoSparkQueryPlanner @@ -117,9 +119,11 @@ public PlanAndMore createQueryPlan(Session session, BuiltInPreparedQuery prepare preparedQuery.getParameters(), parameterExtractor(preparedQuery.getStatement(), preparedQuery.getParameters()), warningCollector, - query); + query, + new ViewDefinitionReferences()); - Analysis analysis = analyzer.analyze(preparedQuery.getStatement()); + Analysis analysis = analyzer.analyzeSemantic(preparedQuery.getStatement(), false); + checkAccessPermissions(analysis.getAccessControlReferences(), analysis.getViewDefinitionReferences(), query, session.getPreparedStatements(), session.getIdentity(), accessControl, session.getAccessControlContext()); LogicalPlanner logicalPlanner = new LogicalPlanner( session, diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkRddFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkRddFactory.java index 9853b3a5c8289..74879e56dee1a 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkRddFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkRddFactory.java @@ -72,6 +72,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static com.facebook.presto.SystemSessionProperties.isNativeExecutionEnabled; import static com.facebook.presto.spark.util.PrestoSparkUtils.classTag; import static com.facebook.presto.spark.util.PrestoSparkUtils.serializeZstdCompressed; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -250,7 +251,7 @@ else if (rddInputs.size() == 0) { taskSourceRdd = Optional.empty(); } - if (featuresConfig.isNativeExecutionEnabled()) { + if (isNativeExecutionEnabled(session)) { return JavaPairRDD.fromRDD( PrestoSparkNativeTaskRdd.create( sparkContext.sc(), diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index b279e11f825cc..a85aa71ee7de2 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -339,15 +339,15 @@ public PrestoSparkQueryRunner( // Sql-Standard Access Control Checker // needs us to specify our role .setIdentity( - new Identity( - "hive", - Optional.empty(), - ImmutableMap.of(defaultCatalog, - new SelectedRole(Type.ROLE, Optional.of("admin"))), - ImmutableMap.of(), - ImmutableMap.of(), - Optional.empty(), - Optional.empty())) + new Identity( + "hive", + Optional.empty(), + ImmutableMap.of(defaultCatalog, + new SelectedRole(Type.ROLE, Optional.of("admin"))), + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty(), + Optional.empty())) .build(); transactionManager = injector.getInstance(TransactionManager.class); @@ -659,6 +659,7 @@ private static PrestoSparkSession createSessionInfo(Session session) session.getIdentity().getUser(), session.getIdentity().getPrincipal(), session.getIdentity().getExtraCredentials(), + session.getIdentity().getCertificates(), session.getCatalog(), session.getSchema(), session.getSource(), diff --git a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java index 5fd5e354e0595..351c42af2ff1f 100644 --- a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java +++ b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkSession.java @@ -14,12 +14,16 @@ package com.facebook.presto.spark.classloader_interface; import java.security.Principal; +import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; @@ -33,6 +37,7 @@ public class PrestoSparkSession private final String user; private final Optional principal; private final Map extraCredentials; + private final List certificates; private final Optional catalog; private final Optional schema; private final Optional source; @@ -49,6 +54,7 @@ public PrestoSparkSession( String user, Optional principal, Map extraCredentials, + List certificates, Optional catalog, Optional schema, Optional source, @@ -65,6 +71,7 @@ public PrestoSparkSession( this.user = requireNonNull(user, "user is null"); this.principal = requireNonNull(principal, "principal is null"); this.extraCredentials = unmodifiableMap(new HashMap<>(requireNonNull(extraCredentials, "extraCredentials is null"))); + this.certificates = unmodifiableList(new ArrayList<>(requireNonNull(certificates, "certificates is null"))); this.catalog = requireNonNull(catalog, "catalog is null"); this.schema = requireNonNull(schema, "schema is null"); this.source = requireNonNull(source, "source is null"); @@ -94,6 +101,11 @@ public Map getExtraCredentials() return extraCredentials; } + public List getCertificates() + { + return certificates; + } + public Optional getCatalog() { return catalog; diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java index c3e8c08c8db45..5e7e4e8535be4 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java @@ -14,6 +14,7 @@ package com.facebook.presto.spark.launcher; import com.facebook.presto.spark.classloader_interface.PrestoSparkConfInitializer; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.airlift.airline.Command; @@ -67,7 +68,7 @@ public void run() Optional.empty(), clientOptions.sessionPropertyConfig == null ? Optional.empty() : Optional.of( loadProperties(checkFile(new File(clientOptions.sessionPropertyConfig)))), - Optional.empty(), + Optional.empty(), Optional.empty()); try (PrestoSparkRunner runner = new PrestoSparkRunner(distribution)) { @@ -75,6 +76,7 @@ public void run() "test", Optional.empty(), ImmutableMap.of(), + ImmutableList.of(), clientOptions.catalog, clientOptions.schema, Optional.empty(), diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java index 4c8af446e8b40..9f1b4926bb7d9 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java @@ -35,6 +35,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.security.Principal; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -84,6 +85,7 @@ public void run( String user, Optional principal, Map extraCredentials, + List certificates, String catalog, String schema, Optional source, @@ -106,6 +108,7 @@ public void run( user, principal, extraCredentials, + certificates, catalog, schema, source, @@ -154,6 +157,7 @@ private void execute(IPrestoSparkQueryExecutionFactory queryExecutionFactory, Pr prestoSparkRunnerContext.getUser(), prestoSparkRunnerContext.getPrincipal(), prestoSparkRunnerContext.getExtraCredentials(), + prestoSparkRunnerContext.getCertificates(), Optional.ofNullable(prestoSparkRunnerContext.getCatalog()), Optional.ofNullable(prestoSparkRunnerContext.getSchema()), prestoSparkRunnerContext.getSource(), diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java index 1c0d8541e8bb8..0736c5252c645 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunnerContext.java @@ -16,6 +16,7 @@ import com.facebook.presto.spark.classloader_interface.ExecutionStrategy; import java.security.Principal; +import java.security.cert.X509Certificate; import java.util.List; import java.util.Map; import java.util.Optional; @@ -28,6 +29,7 @@ public class PrestoSparkRunnerContext private final String user; private final Optional principal; private final Map extraCredentials; + private final List certificates; private final String catalog; private final String schema; private final Optional source; @@ -50,6 +52,7 @@ public PrestoSparkRunnerContext( String user, Optional principal, Map extraCredentials, + List certificates, String catalog, String schema, Optional source, @@ -71,6 +74,7 @@ public PrestoSparkRunnerContext( this.user = user; this.principal = principal; this.extraCredentials = extraCredentials; + this.certificates = certificates; this.catalog = catalog; this.schema = schema; this.source = source; @@ -105,6 +109,11 @@ public Map getExtraCredentials() return extraCredentials; } + public List getCertificates() + { + return certificates; + } + public String getCatalog() { return catalog; @@ -195,6 +204,7 @@ public static class Builder private String user; private Optional principal; private Map extraCredentials; + private List certificates; private String catalog; private String schema; private Optional source; @@ -218,6 +228,7 @@ public Builder(PrestoSparkRunnerContext prestoSparkRunnerContext) this.user = prestoSparkRunnerContext.getUser(); this.principal = prestoSparkRunnerContext.getPrincipal(); this.extraCredentials = prestoSparkRunnerContext.getExtraCredentials(); + this.certificates = prestoSparkRunnerContext.getCertificates(); this.catalog = prestoSparkRunnerContext.getCatalog(); this.schema = prestoSparkRunnerContext.getSchema(); this.source = prestoSparkRunnerContext.getSource(); @@ -249,6 +260,7 @@ public PrestoSparkRunnerContext build() user, principal, extraCredentials, + certificates, catalog, schema, source, diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c703a051062d4..a84c67a450657 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -22,6 +22,12 @@ com.facebook.presto presto-common + + + com.google.guava + guava + + @@ -73,6 +79,12 @@ presto-common test-jar test + + + com.google.guava + guava + + diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewDefinition.java b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewDefinition.java index f31a31aa79c47..091f175fd1d4c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewDefinition.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewDefinition.java @@ -293,7 +293,7 @@ public static final class TableColumn private final String columnName; // This signifies whether the mapping is direct or not. // Mapping is always direct in inner join case. In the outer join case, only the mapping from a column to its source column in the join input table is direct. - // For e.g. in case of SELECT t1_a as t1.a, t2_a as t2.a FROM t1 LEFT JOIN t2 ON t1.a = t2.a + // For e.g. in case of SELECT t1.a as t1_a, t2.a as t2_a FROM t1 LEFT JOIN t2 ON t1.a = t2.a // t1_a -> t1.a is direct mapped // t1_a -> t2.a is NOT direct mapped(as t1,t2 are in outer join) // t2_a -> t2.a is direct mapped(value can become null but column mapping is not altered) @@ -311,6 +311,13 @@ public TableColumn( this.isDirectMapped = requireNonNull(isDirectMapped, "isDirectMapped is null"); } + public TableColumn( + SchemaTableName tableName, + String columnName) + { + this(tableName, columnName, Optional.empty()); + } + public TableColumn( SchemaTableName tableName, String columnName, @@ -355,8 +362,7 @@ public boolean equals(Object o) TableColumn that = (TableColumn) o; return Objects.equals(this.columnName, that.columnName) && - Objects.equals(this.tableName, that.tableName) && - Objects.equals(this.isDirectMapped, that.isDirectMapped); + Objects.equals(this.tableName, that.tableName); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewStaleReadBehavior.java b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewStaleReadBehavior.java index 84500db5d40dd..f7ff66fd9c8ff 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewStaleReadBehavior.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/MaterializedViewStaleReadBehavior.java @@ -17,4 +17,5 @@ public enum MaterializedViewStaleReadBehavior { FAIL, USE_VIEW_QUERY, + USE_STITCHING, } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java index d075c9e79ce03..bcd3bd44e184b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardWarningCode.java @@ -27,6 +27,9 @@ public enum StandardWarningCode SAMPLED_FIELDS(0x0000_0009), MULTIPLE_TABLE_METADATA(0x0000_0010), UTILIZED_COLUMN_ANALYSIS_FAILED(0x0000_0011), + MATERIALIZED_VIEW_ACCESS_CONTROL_FALLBACK(0x0000_0012), + MATERIALIZED_VIEW_STITCHING_FALLBACK(0x0000_0013), + MATERIALIZED_VIEW_STALE_DATA(0x0000_0014), /**/; private final WarningCode warningCode; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/AccessControlReferences.java b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/AccessControlReferences.java index 91ac7123d5d76..16bd163077b2f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/AccessControlReferences.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/AccessControlReferences.java @@ -15,7 +15,6 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.Subfield; -import com.facebook.presto.spi.MaterializedViewDefinition; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -29,16 +28,11 @@ public class AccessControlReferences { private final Map> tableReferences; private final Map>> tableColumnAndSubfieldReferencesForAccessControl; - private AccessControlInfo queryAccessControlInfo; - private final Map viewDefinitions; - private final Map materializedViewDefinitions; public AccessControlReferences() { tableReferences = new LinkedHashMap<>(); tableColumnAndSubfieldReferencesForAccessControl = new LinkedHashMap<>(); - viewDefinitions = new LinkedHashMap<>(); - materializedViewDefinitions = new LinkedHashMap<>(); } public Map> getTableReferences() @@ -60,36 +54,4 @@ public void addTableColumnAndSubfieldReferencesForAccessControl(Map getViewDefinitions() - { - return unmodifiableMap(new LinkedHashMap<>(viewDefinitions)); - } - - public Map getMaterializedViewDefinitions() - { - return unmodifiableMap(new LinkedHashMap<>(materializedViewDefinitions)); - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/QueryAnalysis.java b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/QueryAnalysis.java index 4edab5e2bc378..d2d43663a810a 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/QueryAnalysis.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/QueryAnalysis.java @@ -46,6 +46,11 @@ public interface QueryAnalysis */ AccessControlReferences getAccessControlReferences(); + /** + * Returns all view definitions accessed in the query + */ + ViewDefinitionReferences getViewDefinitionReferences(); + /** * Returns whether the QueryAnalysis represents an "EXPLAIN ANALYZE" query. */ diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/ViewDefinitionReferences.java b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/ViewDefinitionReferences.java new file mode 100644 index 0000000000000..4219f345b36f8 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/analyzer/ViewDefinitionReferences.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.analyzer; + +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.MaterializedViewDefinition; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static java.util.Collections.unmodifiableMap; + +public class ViewDefinitionReferences +{ + private final Map viewDefinitions; + private final Map materializedViewDefinitions; + + public ViewDefinitionReferences() + { + viewDefinitions = new LinkedHashMap<>(); + materializedViewDefinitions = new LinkedHashMap<>(); + } + + public void addViewDefinitionReference(QualifiedObjectName viewDefinitionName, ViewDefinition viewDefinition) + { + viewDefinitions.put(viewDefinitionName, viewDefinition); + } + + public void addMaterializedViewDefinitionReference(QualifiedObjectName viewDefinitionName, MaterializedViewDefinition materializedViewDefinition) + { + materializedViewDefinitions.put(viewDefinitionName, materializedViewDefinition); + } + + public Map getViewDefinitions() + { + return unmodifiableMap(new LinkedHashMap<>(viewDefinitions)); + } + + public Map getMaterializedViewDefinitions() + { + return unmodifiableMap(new LinkedHashMap<>(materializedViewDefinitions)); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java index e014ee82681aa..d0635e928f079 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java @@ -32,7 +32,15 @@ public interface Connector { - ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly); + default ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + throw new UnsupportedOperationException(); + } + + default ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean autoCommitContext, boolean readOnly) + { + return beginTransaction(isolationLevel, readOnly); + } /** * Guaranteed to be called at most once per transaction. The returned metadata will only be accessed diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAccessControl.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAccessControl.java index f3fccb0840d64..cc2e55cbede1b 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAccessControl.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorAccessControl.java @@ -31,9 +31,11 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddColumn; import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateRole; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -447,6 +449,26 @@ default void checkCanDropBranch(ConnectorTransactionHandle transactionHandle, Co denyDropBranch(tableName.toString()); } + /** + * Check if identity is allowed to create branch from the specified table in this catalog. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + default void checkCanCreateBranch(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + denyCreateBranch(tableName.toString()); + } + + /** + * Check if identity is allowed to create tag on the specified table in this catalog. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + default void checkCanCreateTag(ConnectorTransactionHandle transactionHandle, ConnectorIdentity identity, AccessControlContext context, SchemaTableName tableName) + { + denyCreateTag(tableName.toString()); + } + /** * Check if identity is allowed to drop tag from the specified table in this catalog. * diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java index 65ce146a9fa3f..5540734c7ed47 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java @@ -968,6 +968,38 @@ default void dropBranch(ConnectorSession session, ConnectorTableHandle tableHand throw new PrestoException(NOT_SUPPORTED, "This connector does not support dropping table branches"); } + /** + * Create a branch for the specified table + */ + default void createBranch( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + throw new PrestoException(NOT_SUPPORTED, "This connector does not support creating table branches"); + } + + /** + * Create a tag for the specified table + */ + default void createTag( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays) + { + throw new PrestoException(NOT_SUPPORTED, "This connector does not support creating table tags"); + } + /** * Drop the specified tag */ diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java index a1e2333806d3b..63e0f082b788d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java @@ -876,6 +876,38 @@ public void dropBranch(ConnectorSession session, ConnectorTableHandle tableHandl } } + @Override + public void createBranch( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String branchName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays, + Optional minSnapshotsToKeep, + Optional maxSnapshotAgeDays) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + delegate.createBranch(session, tableHandle, branchName, replace, ifNotExists, tableVersion, retainDays, minSnapshotsToKeep, maxSnapshotAgeDays); + } + } + + @Override + public void createTag( + ConnectorSession session, + ConnectorTableHandle tableHandle, + String tagName, + boolean replace, + boolean ifNotExists, + Optional tableVersion, + Optional retainDays) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + delegate.createTag(session, tableHandle, tagName, replace, ifNotExists, tableVersion, retainDays); + } + } + @Override public void dropTag(ConnectorSession session, ConnectorTableHandle tableHandle, String tagName, boolean tagExists) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java index 2bf40c8e525ce..342c421dc8106 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java @@ -169,4 +169,9 @@ public R visitMaterializedViewScan(MaterializedViewScanNode node, C context) { return visitPlan(node, context); } + + public R visitTopNRowNumber(TopNRowNumberNode node, C context) + { + return visitPlan(node, context); + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNRowNumberNode.java similarity index 75% rename from presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNRowNumberNode.java index f5fc3bde590a6..4c156e3a91c65 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TopNRowNumberNode.java @@ -11,32 +11,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; import com.facebook.presto.spi.SourceLocation; -import com.facebook.presto.spi.plan.DataOrganizationSpecification; -import com.facebook.presto.spi.plan.OrderingScheme; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.errorprone.annotations.Immutable; +import java.util.ArrayList; import java.util.List; import java.util.Optional; -import static com.google.common.base.Preconditions.checkArgument; +import static com.facebook.presto.common.Utils.checkArgument; +import static java.util.Collections.singletonList; +import static java.util.Collections.unmodifiableList; import static java.util.Objects.requireNonNull; @Immutable public final class TopNRowNumberNode - extends InternalPlanNode + extends PlanNode { + public enum RankingFunction + { + ROW_NUMBER, + RANK, + DENSE_RANK + } + private final PlanNode source; private final DataOrganizationSpecification specification; + private final RankingFunction rankingFunction; private final VariableReferenceExpression rowNumberVariable; private final int maxRowCountPerPartition; private final boolean partial; @@ -48,12 +53,13 @@ public TopNRowNumberNode( @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, @JsonProperty("specification") DataOrganizationSpecification specification, + @JsonProperty("rankingType") RankingFunction rankingFunction, @JsonProperty("rowNumberVariable") VariableReferenceExpression rowNumberVariable, @JsonProperty("maxRowCountPerPartition") int maxRowCountPerPartition, @JsonProperty("partial") boolean partial, @JsonProperty("hashVariable") Optional hashVariable) { - this(sourceLocation, id, Optional.empty(), source, specification, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); + this(sourceLocation, id, Optional.empty(), source, specification, rankingFunction, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); } public TopNRowNumberNode( @@ -62,6 +68,7 @@ public TopNRowNumberNode( Optional statsEquivalentPlanNode, PlanNode source, DataOrganizationSpecification specification, + RankingFunction rankingFunction, VariableReferenceExpression rowNumberVariable, int maxRowCountPerPartition, boolean partial, @@ -75,9 +82,11 @@ public TopNRowNumberNode( requireNonNull(rowNumberVariable, "rowNumberVariable is null"); checkArgument(maxRowCountPerPartition > 0, "maxRowCountPerPartition must be > 0"); requireNonNull(hashVariable, "hashVariable is null"); + requireNonNull(rankingFunction, "rankingFunction is null"); this.source = source; this.specification = specification; + this.rankingFunction = rankingFunction; this.rowNumberVariable = rowNumberVariable; this.maxRowCountPerPartition = maxRowCountPerPartition; this.partial = partial; @@ -87,18 +96,17 @@ public TopNRowNumberNode( @Override public List getSources() { - return ImmutableList.of(source); + return singletonList(source); } @Override public List getOutputVariables() { - ImmutableList.Builder builder = ImmutableList.builder().addAll(source.getOutputVariables()); - + List outputVariables = new ArrayList<>(source.getOutputVariables()); if (!partial) { - builder.add(rowNumberVariable); + outputVariables.add(rowNumberVariable); } - return builder.build(); + return unmodifiableList(outputVariables); } @JsonProperty @@ -113,6 +121,12 @@ public DataOrganizationSpecification getSpecification() return specification; } + @JsonProperty + public RankingFunction getRankingFunction() + { + return rankingFunction; + } + public List getPartitionBy() { return specification.getPartitionBy(); @@ -148,7 +162,7 @@ public Optional getHashVariable() } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitTopNRowNumber(this, context); } @@ -156,12 +170,13 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new TopNRowNumberNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), Iterables.getOnlyElement(newChildren), specification, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); + checkArgument(newChildren.size() == 1, "expected newChildren to contain 1 node"); + return new TopNRowNumberNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), newChildren.get(0), specification, rankingFunction, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); } @Override public PlanNode assignStatsEquivalentPlanNode(Optional statsEquivalentPlanNode) { - return new TopNRowNumberNode(getSourceLocation(), getId(), statsEquivalentPlanNode, source, specification, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); + return new TopNRowNumberNode(getSourceLocation(), getId(), statsEquivalentPlanNode, source, specification, rankingFunction, rowNumberVariable, maxRowCountPerPartition, partial, hashVariable); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessControl.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessControl.java index ae63db812f477..e864e3ac0699a 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessControl.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessControl.java @@ -350,6 +350,20 @@ default AuthorizedIdentity selectAuthorizedIdentity(Identity identity, AccessCon */ void checkCanDropBranch(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName); + /** + * Check if identity is allowed to create branch for the specified table. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + void checkCanCreateBranch(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName); + + /** + * Check if identity is allowed to create tag for the specified table. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + void checkCanCreateTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName); + /** * Check if identity is allowed to drop tag from the specified table. * diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessDeniedException.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessDeniedException.java index f55b1b8e9e70b..03afcc8d4e97d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessDeniedException.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/AccessDeniedException.java @@ -407,6 +407,26 @@ public static void denySetRole(String role) throw new AccessDeniedException(format("Cannot set role %s", role)); } + public static void denyCreateBranch(String tableName) + { + denyCreateBranch(tableName, null); + } + + public static void denyCreateBranch(String tableName, String extraInfo) + { + throw new AccessDeniedException(format("Cannot create branch on table %s%s", tableName, formatExtraInfo(extraInfo))); + } + + public static void denyCreateTag(String tableName) + { + denyCreateTag(tableName, null); + } + + public static void denyCreateTag(String tableName, String extraInfo) + { + throw new AccessDeniedException(format("Cannot create tag on table %s%s", tableName, formatExtraInfo(extraInfo))); + } + public static void denyDropBranch(String tableName) { denyDropBranch(tableName, null); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/AllowAllAccessControl.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/AllowAllAccessControl.java index 987ab6bf1720a..5f5cdd14ef407 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/AllowAllAccessControl.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/AllowAllAccessControl.java @@ -255,6 +255,16 @@ public void checkCanDropBranch(TransactionId transactionId, Identity identity, A { } + @Override + public void checkCanCreateBranch(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + } + + @Override + public void checkCanCreateTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + } + @Override public void checkCanDropTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/DenyAllAccessControl.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/DenyAllAccessControl.java index 3d6030d91532d..39a66bb9adf4e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/DenyAllAccessControl.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/DenyAllAccessControl.java @@ -33,9 +33,11 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; import static com.facebook.presto.spi.security.AccessDeniedException.denyCatalogAccess; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateRole; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -344,6 +346,18 @@ public void checkCanDropBranch(TransactionId transactionId, Identity identity, A denyDropBranch(tableName.toString()); } + @Override + public void checkCanCreateBranch(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + denyCreateBranch(tableName.toString()); + } + + @Override + public void checkCanCreateTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) + { + denyCreateTag(tableName.toString()); + } + @Override public void checkCanDropTag(TransactionId transactionId, Identity identity, AccessControlContext context, QualifiedObjectName tableName) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/SystemAccessControl.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/SystemAccessControl.java index fdc937e996ba4..1090ed8d6a6e6 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/SystemAccessControl.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/SystemAccessControl.java @@ -33,8 +33,10 @@ import static com.facebook.presto.spi.security.AccessDeniedException.denyAddConstraint; import static com.facebook.presto.spi.security.AccessDeniedException.denyCallProcedure; import static com.facebook.presto.spi.security.AccessDeniedException.denyCatalogAccess; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateBranch; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateSchema; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTable; +import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateTag; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateView; import static com.facebook.presto.spi.security.AccessDeniedException.denyCreateViewWithSelect; import static com.facebook.presto.spi.security.AccessDeniedException.denyDeleteTable; @@ -413,6 +415,26 @@ default void checkCanRevokeTablePrivilege(Identity identity, AccessControlContex denyRevokeTablePrivilege(privilege.toString(), table.toString()); } + /** + * Check if identity is allowed to create branch on the specified table in a catalog. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + default void checkCanCreateBranch(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + denyCreateBranch(table.toString()); + } + + /** + * Check if identity is allowed to create tag on the specified table in a catalog. + * + * @throws com.facebook.presto.spi.security.AccessDeniedException if not allowed + */ + default void checkCanCreateTag(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + denyCreateTag(table.toString()); + } + /** * Check if identity is allowed to drop branch from the specified table in a catalog. * diff --git a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSimpleSamplingPercent.java b/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSimpleSamplingPercent.java deleted file mode 100644 index a710391760714..0000000000000 --- a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSimpleSamplingPercent.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.scalar.sql; - -import com.facebook.presto.spi.function.Description; -import com.facebook.presto.spi.function.SqlInvokedScalarFunction; -import com.facebook.presto.spi.function.SqlParameter; -import com.facebook.presto.spi.function.SqlType; - -public class NativeSimpleSamplingPercent -{ - private NativeSimpleSamplingPercent() {} - - @SqlInvokedScalarFunction(value = "key_sampling_percent", deterministic = true, calledOnNullInput = false) - @Description("Returns a value between 0.0 and 1.0 using the hash of the given input string") - @SqlParameter(name = "input", type = "varchar") - @SqlType("double") - public static String keySamplingPercent() - { - return "return (abs(from_ieee754_64(xxhash64(cast(input as varbinary)))) % 100) / 100. "; - } -} diff --git a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSqlInvokedFunctionsPlugin.java b/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSqlInvokedFunctionsPlugin.java index 69d7ff1e78522..48857c7300216 100644 --- a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSqlInvokedFunctionsPlugin.java +++ b/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/src/main/java/com/facebook/presto/scalar/sql/NativeSqlInvokedFunctionsPlugin.java @@ -27,7 +27,6 @@ public Set> getSqlInvokedFunctions() return ImmutableSet.>builder() .add(NativeArraySqlFunctions.class) .add(NativeMapSqlFunctions.class) - .add(NativeSimpleSamplingPercent.class) .build(); } } diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index e907fa0868449..f1b49251de559 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -264,6 +264,11 @@ drift-transport-netty + + io.netty + netty-buffer + + jakarta.servlet jakarta.servlet-api diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index dfe949a7a148f..3a6c366e4e191 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -90,6 +90,7 @@ import static com.facebook.presto.SystemSessionProperties.REWRITE_EXPRESSION_WITH_CONSTANT_EXPRESSION; import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_NULL_FILTER_TO_SEMI_JOIN; import static com.facebook.presto.SystemSessionProperties.REWRITE_MIN_MAX_BY_TO_TOP_N; +import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_COALESCE_OVER_JOIN_KEYS; import static com.facebook.presto.SystemSessionProperties.SIMPLIFY_PLAN_WITH_EMPTY_INPUT; import static com.facebook.presto.SystemSessionProperties.USE_DEFAULTS_FOR_CORRELATED_AGGREGATION_PUSHDOWN_THROUGH_OUTER_JOINS; import static com.facebook.presto.common.type.BigintType.BIGINT; @@ -2715,19 +2716,41 @@ public void testExplainValidateOfExplain() @Test public void testExplainDdl() { + // CREATE TABLE assertExplainDdl("CREATE TABLE foo (pk bigint)", "CREATE TABLE foo"); + assertExplainDdl("CREATE TABLE IF NOT EXISTS foo (pk bigint)", "CREATE TABLE IF NOT EXISTS foo"); + assertExplainDdl("CREATE TABLE mycatalog.myschema.foo (pk bigint)", "CREATE TABLE mycatalog.myschema.foo"); + assertExplainDdl("CREATE TABLE IF NOT EXISTS mycatalog.myschema.foo (pk bigint)", "CREATE TABLE IF NOT EXISTS mycatalog.myschema.foo"); + + // DROP TABLE + assertExplainDdl("DROP TABLE orders"); + assertExplainDdl("DROP TABLE IF EXISTS orders"); + assertExplainDdl("DROP TABLE IF EXISTS mycatalog.myschema.orders"); + + // CREATE VIEW assertExplainDdl("CREATE VIEW foo AS SELECT * FROM orders", "CREATE VIEW foo"); + + // DROP VIEW + assertExplainDdl("DROP VIEW view"); + + // CREATE/ALTER/DROP FUNCTION assertExplainDdl("CREATE OR REPLACE FUNCTION testing.default.tan (x int) RETURNS double COMMENT 'tangent trigonometric function' LANGUAGE SQL DETERMINISTIC CALLED ON NULL INPUT RETURN sin(x) / cos(x)", "CREATE FUNCTION testing.default.tan"); assertExplainDdl("ALTER FUNCTION testing.default.tan CALLED ON NULL INPUT", "ALTER FUNCTION testing.default.tan"); assertExplainDdl("DROP FUNCTION IF EXISTS testing.default.tan (int)", "DROP FUNCTION testing.default.tan"); - assertExplainDdl("DROP TABLE orders"); - assertExplainDdl("DROP VIEW view"); + + // ALTER TABLE assertExplainDdl("ALTER TABLE orders RENAME TO new_name"); assertExplainDdl("ALTER TABLE orders RENAME COLUMN orderkey TO new_column_name"); + + // SESSION assertExplainDdl("SET SESSION foo = 'bar'"); + assertExplainDdl("RESET SESSION foo"); + + // PREPARE/DEALLOCATE assertExplainDdl("PREPARE my_query FROM SELECT * FROM orders", "PREPARE my_query"); assertExplainDdl("DEALLOCATE PREPARE my_query"); - assertExplainDdl("RESET SESSION foo"); + + // TRANSACTION assertExplainDdl("START TRANSACTION"); assertExplainDdl("COMMIT"); assertExplainDdl("ROLLBACK"); @@ -8199,6 +8222,97 @@ private List getNativeWorkerSessionProperties(List 0) FROM (SELECT 5 AS x FROM orders)", + disabled); + + // Non-constant columns should not be affected + assertQueryWithSameQueryRunner(enabled, + "SELECT MIN(custkey), MAX(custkey) FROM orders", + disabled); + + // Constant via WHERE equality (after constant propagation) + assertQueryWithSameQueryRunner(enabled, + "SELECT MIN(orderkey) FROM orders WHERE orderkey = 7", + disabled); + + // Constant via WHERE equality with multiple aggregations + assertQueryWithSameQueryRunner(enabled, + "SELECT MIN(orderkey), MAX(orderkey), SUM(orderkey) FROM orders WHERE orderkey = 7", + disabled); + + // Constant via CAST expression in projection + assertQueryWithSameQueryRunner(enabled, + "SELECT MIN(x) FROM (SELECT CAST(7 AS BIGINT) AS x FROM orders)", + disabled); + + // WHERE clause that eliminates all rows — result should be NULL + assertQueryWithSameQueryRunner(enabled, + "SELECT MIN(orderkey) FROM orders WHERE orderkey = -1", + disabled); + + // GROUP BY with constant via WHERE + assertQueryWithSameQueryRunner(enabled, + "SELECT orderstatus, MIN(orderkey) FROM orders WHERE orderkey = 7 GROUP BY orderstatus", + disabled); + } + /** * Returns a date expression, casting to DATE if storageFormat is DWRF. */ @@ -8207,4 +8321,50 @@ protected String getDateExpression(String storageFormat, String columnExpression // DWRF does not support date type. return storageFormat.equals("DWRF") ? "cast(" + columnExpression + " as DATE)" : columnExpression; } + + @Test + public void testSimplifyCoalesceOverJoinKeys() + { + Session enabledSession = Session.builder(getSession()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "true") + .build(); + Session disabledSession = Session.builder(getSession()) + .setSystemProperty(SIMPLIFY_COALESCE_OVER_JOIN_KEYS, "false") + .build(); + + // LEFT JOIN: COALESCE(l.x, r.y) should be simplified to l.x + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(n.nationkey, r.regionkey) FROM nation n LEFT JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // LEFT JOIN: COALESCE(r.y, l.x) should also simplify to l.x + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(r.regionkey, n.nationkey) FROM nation n LEFT JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // RIGHT JOIN: COALESCE(l.x, r.y) should simplify to r.y + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(n.nationkey, r.regionkey) FROM nation n RIGHT JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // INNER JOIN: COALESCE(l.x, r.y) should simplify to l.x (first arg) + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(n.nationkey, r.regionkey) FROM nation n INNER JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // FULL JOIN: COALESCE should NOT be simplified — verify results still match + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(n.nationkey, r.regionkey) FROM nation n FULL JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // Multiple columns with COALESCE on join key plus other columns + assertQueryWithSameQueryRunner(enabledSession, + "SELECT COALESCE(n.nationkey, r.regionkey), n.name FROM nation n LEFT JOIN region r ON n.nationkey = r.regionkey", + disabledSession); + + // JOIN USING produces COALESCE automatically + assertQueryWithSameQueryRunner(enabledSession, + "SELECT regionkey FROM nation LEFT JOIN region USING (regionkey)", + disabledSession); + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index 44b1c60c82956..ae31e009d9ecd 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -28,6 +28,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.ViewDefinitionReferences; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.security.AccessDeniedException; import com.facebook.presto.spi.security.AllowAllAccessControl; @@ -185,19 +186,16 @@ protected void assertQuery(@Language("SQL") String actual, @Language("SQL") Stri protected void assertQueryWithSameQueryRunner(@Language("SQL") String actual, @Language("SQL") String expected) { - checkArgument(!actual.equals(expected)); QueryAssertions.assertQuery(queryRunner, getSession(), actual, queryRunner, expected, false, false); } protected void assertQueryWithSameQueryRunner(Session session, @Language("SQL") String actual, @Language("SQL") String expected) { - checkArgument(!actual.equals(expected)); QueryAssertions.assertQuery(queryRunner, session, actual, queryRunner, expected, false, false); } protected void assertQueryOrderedWithSameQueryRunner(@Language("SQL") String actual, @Language("SQL") String expected) { - checkArgument(!actual.equals(expected)); QueryAssertions.assertQuery(queryRunner, getSession(), actual, queryRunner, expected, true, false); } @@ -208,7 +206,6 @@ protected void assertQueryWithSameQueryRunner(Session actualSession, @Language(" protected void assertQueryWithSameQueryRunner(Session actualSession, @Language("SQL") String actual, Session expectedSession, @Language("SQL") String expected) { - checkArgument(!actual.equals(expected)); QueryAssertions.assertQuery(queryRunner, actualSession, actual, queryRunner, expectedSession, expected, false, false); } @@ -499,7 +496,7 @@ public String getExplainPlan(String explainCommandText, String query, ExplainTyp return transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(queryRunner.getDefaultSession(), session -> { - return explainer.getPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), false, WarningCollector.NOOP, query); + return explainer.getPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), false, WarningCollector.NOOP, query, new ViewDefinitionReferences()); }); } @@ -509,7 +506,7 @@ public String getGraphvizExplainPlan(String explainCommandText, String query, Ex return transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(queryRunner.getDefaultSession(), session -> { - return explainer.getGraphvizPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), WarningCollector.NOOP, query); + return explainer.getGraphvizPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), WarningCollector.NOOP, query, new ViewDefinitionReferences()); }); } @@ -519,7 +516,7 @@ public String getJsonExplainPlan(String explainCommandText, String query, Explai return transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(queryRunner.getDefaultSession(), session -> { - return explainer.getJsonPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), WarningCollector.NOOP, query); + return explainer.getJsonPlan(session, sqlParser.createStatement(explainCommandText.replaceAll(".", " ") + query, createParsingOptions(session)), planType, emptyList(), WarningCollector.NOOP, query, new ViewDefinitionReferences()); }); } @@ -539,7 +536,7 @@ protected void assertPlan(Session session, @Language("SQL") String query, PlanMa transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(session, transactionSession -> { - Plan actualPlan = explainer.getLogicalPlan(transactionSession, sqlParser.createStatement(query, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, query); + Plan actualPlan = explainer.getLogicalPlan(transactionSession, sqlParser.createStatement(query, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, query, new ViewDefinitionReferences()); PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getStatsCalculator(), actualPlan, pattern); planValidator.accept(actualPlan); return null; @@ -553,7 +550,7 @@ protected Plan plan(@Language("SQL") String sql, Session session) return transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(session, transactionSession -> { - return explainer.getLogicalPlan(transactionSession, sqlParser.createStatement(sql, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, sql); + return explainer.getLogicalPlan(transactionSession, sqlParser.createStatement(sql, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, sql, new ViewDefinitionReferences()); }); } catch (RuntimeException e) { @@ -573,7 +570,7 @@ protected SubPlan subplan(String sql, Session session) return transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) .singleStatement() .execute(session, transactionSession -> { - return explainer.getDistributedPlan(transactionSession, sqlParser.createStatement(sql, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, sql); + return explainer.getDistributedPlan(transactionSession, sqlParser.createStatement(sql, createParsingOptions(transactionSession)), emptyList(), WarningCollector.NOOP, sql, new ViewDefinitionReferences()); }); } catch (RuntimeException e) { diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index b9f88018332f8..dd9333fab1ad8 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -23,6 +23,7 @@ import com.facebook.presto.Session; import com.facebook.presto.Session.SessionBuilder; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.Type; import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.execution.QueryInfo; import com.facebook.presto.execution.QueryManager; @@ -52,6 +53,7 @@ import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.MaterializedRow; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.testing.TestingAccessControlManager; import com.facebook.presto.transaction.TransactionManager; @@ -91,6 +93,7 @@ import static com.facebook.airlift.json.JsonCodec.jsonCodec; import static com.facebook.airlift.units.Duration.nanosSince; import static com.facebook.presto.client.PrestoHeaders.PRESTO_USER; +import static com.facebook.presto.common.type.TypeUtils.isNumericType; import static com.facebook.presto.spi.NodePoolType.INTERMEDIATE; import static com.facebook.presto.spi.NodePoolType.LEAF; import static com.facebook.presto.testing.TestingSession.TESTING_CATALOG; @@ -874,6 +877,52 @@ public MaterializedResult execute(Session session, @Language("SQL") String sql) return execute(getRandomCoordinatorIndex(), session, sql); } + @Override + public MaterializedResult execute(Session session, @Language("SQL") String sql, List resultTypes) + { + MaterializedResult result = execute(session, sql); + List actualTypes = result.getTypes(); + if (actualTypes.equals(resultTypes)) { + return result; + } + checkState(actualTypes.size() == resultTypes.size(), + "Expected %s result types but got %s", resultTypes.size(), actualTypes.size()); + + List coercedRows = result.getMaterializedRows().stream() + .map(row -> { + List coercedValues = new java.util.ArrayList<>(); + for (int i = 0; i < row.getFieldCount(); i++) { + Object value = row.getField(i); + Type fromType = actualTypes.get(i); + Type toType = resultTypes.get(i); + if (value == null || fromType.equals(toType)) { + coercedValues.add(value); + } + else if (value instanceof Number && isNumericType(fromType) && isNumericType(toType)) { + coercedValues.add(coerceNumeric((Number) value, toType)); + } + else { + coercedValues.add(value); + } + } + return new MaterializedRow(row.getPrecision(), coercedValues); + }) + .collect(ImmutableList.toImmutableList()); + return new MaterializedResult(coercedRows, resultTypes); + } + + private static Object coerceNumeric(Number value, Type toType) + { + Class javaType = toType.getJavaType(); + if (javaType == long.class) { + return value.longValue(); + } + if (javaType == double.class) { + return value.doubleValue(); + } + return value; + } + public ResultWithQueryId executeWithQueryId(Session session, @Language("SQL") String sql) { return executeWithQueryId(getRandomCoordinatorIndex(), session, sql); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/H2FunctionNamespaceManagerFactory.java b/presto-tests/src/main/java/com/facebook/presto/tests/H2FunctionNamespaceManagerFactory.java index 420044f21ca41..b8a1c227f01ae 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/H2FunctionNamespaceManagerFactory.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/H2FunctionNamespaceManagerFactory.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.function.SqlFunctionHandle; import com.google.inject.Injector; +import io.netty.buffer.PooledByteBufAllocator; import java.util.Map; @@ -53,7 +54,7 @@ public FunctionNamespaceManager create(String catalogName, Map TABLE(VALUES (1, 'a'), (2, 'b'), (3, 'c')) t(x, y)))\n" + + ")\n" + + "SELECT *\n" + + "FROM TABLE(system.identity_function(input => TABLE(step1)))", + "VALUES (1, 'a'), (2, 'b'), (3, 'c')"); + + // Multiple levels of nesting with CTE + assertQuery("WITH step1 AS (\n" + + " SELECT *\n" + + " FROM TABLE(system.repeat(TABLE(VALUES (1, 'x')) t(a, b), 2))\n" + + "),\n" + + "step2 AS (\n" + + " SELECT *\n" + + " FROM TABLE(system.repeat(TABLE(step1), 2))\n" + + ")\n" + + "SELECT * FROM TABLE(system.identity_function(input => TABLE(step2)))", + "VALUES (1, 'x'), (1, 'x'), (1, 'x'), (1, 'x')"); + } + + @Test + public void testNestedTableFunctionsWithSubquery() + { + // Test nested table function calls using subquery + // This approach also works for chaining table function operations + assertQuery("SELECT *\n" + + "FROM TABLE(system.identity_function(input => TABLE(\n" + + " SELECT *\n" + + " FROM TABLE(system.identity_function(input => TABLE(VALUES (1, 'a'), (2, 'b'), (3, 'c')) t(x, y)))\n" + + ")))", + "VALUES (1, 'a'), (2, 'b'), (3, 'c')"); + + // Multiple levels of nesting with subquery + assertQuery("SELECT *\n" + + "FROM TABLE(system.identity_function(input => TABLE(\n" + + " SELECT *\n" + + " FROM TABLE(system.repeat(TABLE(\n" + + " SELECT *\n" + + " FROM TABLE(system.repeat(TABLE(VALUES (1, 'x')) t(a, b), 2))\n" + + " ), 2))\n" + + ")))", + "VALUES (1, 'x'), (1, 'x'), (1, 'x'), (1, 'x')"); + + // Combining with pass-through function + assertQuery("SELECT *\n" + + "FROM TABLE(system.identity_pass_through_function(input => TABLE(\n" + + " SELECT *\n" + + " FROM TABLE(system.identity_function(input => TABLE(VALUES (5, 'test')) t(num, str)))\n" + + ")))", + "VALUES (5, 'test')"); + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java index 904c61831f0fa..db2cafbda26d4 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/TestingPrestoClient.java @@ -70,6 +70,8 @@ import static com.facebook.presto.common.type.DateType.DATE; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.IpAddressType.IPADDRESS; +import static com.facebook.presto.common.type.IpPrefixType.IPPREFIX; import static com.facebook.presto.common.type.JsonType.JSON; import static com.facebook.presto.common.type.RealType.REAL; import static com.facebook.presto.common.type.SmallintType.SMALLINT; @@ -82,7 +84,6 @@ import static com.facebook.presto.testing.MaterializedResult.DEFAULT_PRECISION; import static com.facebook.presto.type.IntervalDayTimeType.INTERVAL_DAY_TIME; import static com.facebook.presto.type.IntervalYearMonthType.INTERVAL_YEAR_MONTH; -import static com.facebook.presto.type.IpAddressType.IPADDRESS; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.transform; @@ -246,6 +247,9 @@ else if (INTERVAL_YEAR_MONTH.equals(type)) { else if (IPADDRESS.equals(type)) { return value; } + else if (IPPREFIX.equals(type)) { + return value; + } else if (type instanceof ArrayType) { return ((List) value).stream() .map(element -> convertToRowValue(((ArrayType) type).getElementType(), element)) diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java index e1e6e1c63646a..28785a9327457 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestHistoryBasedStatsTracking.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.TopNRowNumberNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.statistics.CostBasedSourceInfo; import com.facebook.presto.spi.statistics.HistoryBasedPlanStatisticsProvider; @@ -38,7 +39,6 @@ import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.testing.InMemoryHistoryBasedPlanStatisticsProvider; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; @@ -120,7 +120,7 @@ public void testHistoryBasedStatsCalculator() anyTree(node(ProjectNode.class, node(FilterNode.class, any())).withOutputRowCount(12.5))); assertPlan( "SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", - anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(Double.NaN))); + anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(5).withOutputSize(90))); // HBO Statistics executeAndTrackHistory("SELECT max(nationkey) FROM nation where name < 'D' group by regionkey"); @@ -227,7 +227,7 @@ public void testHistoryBasedStatsCalculatorEnforceTimeOut() assertPlan( sessionWithDefaultTimeoutLimit, "SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", - anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(Double.NaN))); + anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(5).withOutputSize(90))); // Write HBO statistics failed as we set timeout limit to be 0 executeAndNoHistoryWritten("SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", sessionWithZeroTimeoutLimit); @@ -239,7 +239,7 @@ public void testHistoryBasedStatsCalculatorEnforceTimeOut() assertPlan( sessionWithDefaultTimeoutLimit, "SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", - anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(Double.NaN))); + anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(5).withOutputSize(90))); // Write HBO Statistics is successful, as we use the default 10 seconds timeout limit executeAndTrackHistory("SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", sessionWithDefaultTimeoutLimit); @@ -261,7 +261,7 @@ public void testHistoryBasedStatsCalculatorEnforceTimeOut() assertPlan( sessionWithZeroTimeoutLimit, "SELECT max(nationkey) FROM nation where name < 'D' group by regionkey", - anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(Double.NaN))); + anyTree(node(AggregationNode.class, node(ExchangeNode.class, anyTree(any()))).withOutputRowCount(5))); } @Test diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java index a1395ae886ca5..d55f661667f4e 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java @@ -53,5 +53,9 @@ public void testCheckQueryIntegrityCalls() assertAccessDenied("explain (type validate) select * from orders", ".*Query integrity check failed.*"); assertAccessDenied("CREATE TABLE test_empty (a BIGINT)", ".*Query integrity check failed.*"); assertAccessDenied("use tpch.tiny", ".*Query integrity check failed.*"); + assertAccessDenied("create view test_orders_view as select * from orders", ".*Query integrity check failed.*"); + assertAccessDenied("create function tpch.tiny.square2(x int) returns int return x * x", ".*Query integrity check failed.*"); + assertAccessDenied("alter function tpch.tiny.tan(double) called on null input", ".*Query integrity check failed.*"); + assertAccessDenied("drop function tpch.tiny.tan(double)", ".*Query integrity check failed.*"); } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java index e9f361f019dd3..41ed1e54689eb 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestMetadataManager.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.inject.Key; import org.intellij.lang.annotations.Language; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -89,7 +90,7 @@ public Iterable getConnectorFactories() } }); queryRunner.createCatalog("upper_case_schema_catalog", "mock"); - metadataManager = (MetadataManager) queryRunner.getMetadata(); + metadataManager = queryRunner.getCoordinator().getInstance(Key.get(MetadataManager.class)); } @AfterClass(alwaysRun = true) diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java index 4c62a887cced1..b0bb921fa1e9f 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestQueryTaskLimit.java @@ -94,7 +94,7 @@ public void testQueuingWhenTaskLimitExceeds() { ImmutableMap extraProperties = ImmutableMap.builder() .put("experimental.spill-enabled", "false") - .put("experimental.max-total-running-task-count-to-not-execute-new-query", "2") + .put("max-total-running-task-count-to-not-execute-new-query", "2") .build(); try (DistributedQueryRunner queryRunner = createQueryRunner(defaultSession, extraProperties)) { diff --git a/presto-thrift-connector/pom.xml b/presto-thrift-connector/pom.xml index d05e9106c59fb..69eece3a9bfbf 100644 --- a/presto-thrift-connector/pom.xml +++ b/presto-thrift-connector/pom.xml @@ -56,14 +56,13 @@ - com.google.guava - guava + io.netty + netty-buffer - com.google.errorprone - error_prone_annotations - true + com.google.guava + guava diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java index 5f690b6cb690c..bb8b36becd16a 100644 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java +++ b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/ThriftConnectorFactory.java @@ -16,7 +16,7 @@ import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.drift.transport.netty.client.DriftNettyClientModule; import com.facebook.presto.common.type.TypeManager; -import com.facebook.presto.connector.thrift.util.RebindSafeMBeanServer; +import com.facebook.presto.common.util.RebindSafeMBeanServer; import com.facebook.presto.spi.ConnectorHandleResolver; import com.facebook.presto.spi.ConnectorSystemConfig; import com.facebook.presto.spi.connector.Connector; @@ -25,6 +25,7 @@ import com.facebook.presto.spi.relation.RowExpressionService; import com.google.inject.Injector; import com.google.inject.Module; +import io.netty.buffer.PooledByteBufAllocator; import org.weakref.jmx.guice.MBeanModule; import javax.management.MBeanServer; @@ -65,7 +66,7 @@ public Connector create(String catalogName, Map config, Connecto try { Bootstrap app = new Bootstrap( new MBeanModule(), - new DriftNettyClientModule(), + new DriftNettyClientModule(PooledByteBufAllocator.DEFAULT), binder -> { binder.bind(MBeanServer.class).toInstance(new RebindSafeMBeanServer(getPlatformMBeanServer())); binder.bind(TypeManager.class).toInstance(context.getTypeManager()); diff --git a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java b/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java deleted file mode 100644 index be2e9f01fb5a3..0000000000000 --- a/presto-thrift-connector/src/main/java/com/facebook/presto/connector/thrift/util/RebindSafeMBeanServer.java +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.connector.thrift.util; - -import com.facebook.airlift.log.Logger; -import com.google.errorprone.annotations.ThreadSafe; - -import javax.management.Attribute; -import javax.management.AttributeList; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceAlreadyExistsException; -import javax.management.InstanceNotFoundException; -import javax.management.IntrospectionException; -import javax.management.InvalidAttributeValueException; -import javax.management.ListenerNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanInfo; -import javax.management.MBeanRegistrationException; -import javax.management.MBeanServer; -import javax.management.NotCompliantMBeanException; -import javax.management.NotificationFilter; -import javax.management.NotificationListener; -import javax.management.ObjectInstance; -import javax.management.ObjectName; -import javax.management.OperationsException; -import javax.management.QueryExp; -import javax.management.ReflectionException; -import javax.management.loading.ClassLoaderRepository; - -import java.io.ObjectInputStream; -import java.util.Set; - -// TODO: move this to airlift or jmxutils - -/** - * MBeanServer wrapper that a ignores calls to registerMBean when there is already - * a MBean registered with the specified object name. - */ -@ThreadSafe -public class RebindSafeMBeanServer - implements MBeanServer -{ - private static final Logger log = Logger.get(RebindSafeMBeanServer.class); - - private final MBeanServer mbeanServer; - - public RebindSafeMBeanServer(MBeanServer mbeanServer) - { - this.mbeanServer = mbeanServer; - } - - /** - * Delegates to the wrapped mbean server, but if a mbean is already registered - * with the specified name, the existing instance is returned. - */ - @Override - public ObjectInstance registerMBean(Object object, ObjectName name) - throws MBeanRegistrationException, NotCompliantMBeanException - { - while (true) { - try { - // try to register the mbean - return mbeanServer.registerMBean(object, name); - } - catch (InstanceAlreadyExistsException ignored) { - } - - try { - // a mbean is already installed, try to return the already registered instance - ObjectInstance objectInstance = mbeanServer.getObjectInstance(name); - log.debug("%s already bound to %s", name, objectInstance); - return objectInstance; - } - catch (InstanceNotFoundException ignored) { - // the mbean was removed before we could get the reference - // start the whole process over again - } - } - } - - @Override - public void unregisterMBean(ObjectName name) - throws InstanceNotFoundException, MBeanRegistrationException - { - mbeanServer.unregisterMBean(name); - } - - @Override - public ObjectInstance getObjectInstance(ObjectName name) - throws InstanceNotFoundException - { - return mbeanServer.getObjectInstance(name); - } - - @Override - public Set queryMBeans(ObjectName name, QueryExp query) - { - return mbeanServer.queryMBeans(name, query); - } - - @Override - public Set queryNames(ObjectName name, QueryExp query) - { - return mbeanServer.queryNames(name, query); - } - - @Override - public boolean isRegistered(ObjectName name) - { - return mbeanServer.isRegistered(name); - } - - @Override - public Integer getMBeanCount() - { - return mbeanServer.getMBeanCount(); - } - - @Override - public Object getAttribute(ObjectName name, String attribute) - throws MBeanException, AttributeNotFoundException, InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttribute(name, attribute); - } - - @Override - public AttributeList getAttributes(ObjectName name, String[] attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.getAttributes(name, attributes); - } - - @Override - public void setAttribute(ObjectName name, Attribute attribute) - throws InstanceNotFoundException, AttributeNotFoundException, InvalidAttributeValueException, MBeanException, ReflectionException - { - mbeanServer.setAttribute(name, attribute); - } - - @Override - public AttributeList setAttributes(ObjectName name, AttributeList attributes) - throws InstanceNotFoundException, ReflectionException - { - return mbeanServer.setAttributes(name, attributes); - } - - @Override - public Object invoke(ObjectName name, String operationName, Object[] params, String[] signature) - throws InstanceNotFoundException, MBeanException, ReflectionException - { - return mbeanServer.invoke(name, operationName, params, signature); - } - - @Override - public String getDefaultDomain() - { - return mbeanServer.getDefaultDomain(); - } - - @Override - public String[] getDomains() - { - return mbeanServer.getDomains(); - } - - @Override - public void addNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void addNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException - { - mbeanServer.addNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, ObjectName listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener); - } - - @Override - public void removeNotificationListener(ObjectName name, NotificationListener listener, NotificationFilter filter, Object context) - throws InstanceNotFoundException, ListenerNotFoundException - { - mbeanServer.removeNotificationListener(name, listener, filter, context); - } - - @Override - public MBeanInfo getMBeanInfo(ObjectName name) - throws InstanceNotFoundException, IntrospectionException, ReflectionException - { - return mbeanServer.getMBeanInfo(name); - } - - @Override - public boolean isInstanceOf(ObjectName name, String className) - throws InstanceNotFoundException - { - return mbeanServer.isInstanceOf(name, className); - } - - @Override - public Object instantiate(String className) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className); - } - - @Override - public Object instantiate(String className, ObjectName loaderName) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName); - } - - @Override - public Object instantiate(String className, Object[] params, String[] signature) - throws ReflectionException, MBeanException - { - return mbeanServer.instantiate(className, params, signature); - } - - @Override - public Object instantiate(String className, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, MBeanException, InstanceNotFoundException - { - return mbeanServer.instantiate(className, loaderName, params, signature); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(ObjectName name, byte[] data) - throws OperationsException - { - return mbeanServer.deserialize(name, data); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(String className, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, data); - } - - @SuppressWarnings("deprecation") - @Override - @Deprecated - public ObjectInputStream deserialize(String className, ObjectName loaderName, byte[] data) - throws OperationsException, ReflectionException - { - return mbeanServer.deserialize(className, loaderName, data); - } - - @Override - public ClassLoader getClassLoaderFor(ObjectName mbeanName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoaderFor(mbeanName); - } - - @Override - public ClassLoader getClassLoader(ObjectName loaderName) - throws InstanceNotFoundException - { - return mbeanServer.getClassLoader(loaderName); - } - - @Override - public ClassLoaderRepository getClassLoaderRepository() - { - return mbeanServer.getClassLoaderRepository(); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException - { - return mbeanServer.createMBean(className, name, params, signature); - } - - @Override - public ObjectInstance createMBean(String className, ObjectName name, ObjectName loaderName, Object[] params, String[] signature) - throws ReflectionException, InstanceAlreadyExistsException, MBeanException, NotCompliantMBeanException, InstanceNotFoundException - { - return mbeanServer.createMBean(className, name, loaderName, params, signature); - } -} diff --git a/presto-ui/bin/check_webui.sh b/presto-ui/bin/check_webui.sh index 92e56ebcdfde8..247dc0f740cc6 100755 --- a/presto-ui/bin/check_webui.sh +++ b/presto-ui/bin/check_webui.sh @@ -4,7 +4,9 @@ # # 1. Validate that the generated files that have been checked in to the webapp folder are in sync # with the source. -# 2. Make sure there are no type checker warnings reported by Flow +# 2. Make sure there are no type checker warnings reported by TypeScript +# 3. Run ESLint to check for code quality issues +# 4. Run Jest tests to ensure code functionality set -euo pipefail @@ -41,4 +43,11 @@ fi if ! yarn --cwd ${WEBUI_ROOT}/ run lint --quiet; then echo "ERROR: ESlint errors found" exit 1 +fi + +# Fail on test failures only (coverage thresholds disabled for now) + +if ! yarn --cwd ${WEBUI_ROOT}/ run test:ci; then + echo "ERROR: Tests failed" + exit 1 fi \ No newline at end of file diff --git a/presto-ui/src/.gitignore b/presto-ui/src/.gitignore new file mode 100644 index 0000000000000..6035f2075a480 --- /dev/null +++ b/presto-ui/src/.gitignore @@ -0,0 +1,6 @@ +# Test coverage +coverage/ +*.lcov + +# Jest +.jest-cache/ \ No newline at end of file diff --git a/presto-ui/src/__tests__/README.md b/presto-ui/src/__tests__/README.md new file mode 100644 index 0000000000000..18d5a0e4f2bea --- /dev/null +++ b/presto-ui/src/__tests__/README.md @@ -0,0 +1,612 @@ +# Presto UI Testing Guide + +This guide explains the testing infrastructure and patterns used in the Presto UI project. + +## Table of Contents + +1. [Quick Start](#quick-start) +2. [Test Structure](#test-structure) +3. [Test Utilities](#test-utilities) +4. [Setup Helpers](#setup-helpers) +5. [Mocks](#mocks) +6. [Fixtures](#fixtures) +7. [Testing Patterns](#testing-patterns) +8. [Configuration](#configuration) +9. [Best Practices](#best-practices) + +## Quick Start + +### Running Tests + +```bash +# Run all tests +yarn test + +# Run tests in watch mode +yarn test:watch + +# Run tests with coverage +yarn test:coverage + +# Run specific test file +yarn test QueryList.test.jsx +``` + +### Writing Your First Test + +```jsx +import { render, screen } from "../__tests__/utils/testUtils"; +import { setupCommonMocks } from "../__tests__/utils/setupHelpers"; +import { createMockQuery } from "../__tests__/fixtures"; +import MyComponent from "./MyComponent"; + +describe("MyComponent", () => { + setupCommonMocks(); + + it("renders correctly", () => { + const query = createMockQuery(); + render(); + expect(screen.getByText(/expected text/i)).toBeInTheDocument(); + }); +}); +``` + +## Test Structure + +### Directory Layout + +``` +src/ +├── __tests__/ +│ ├── fixtures/ # Test data factories +│ │ ├── index.ts # Barrel export +│ │ ├── queryFixtures.js +│ │ ├── infoFixtures.ts +│ │ ├── stageFixtures.ts +│ │ └── clusterFixtures.ts +│ ├── mocks/ # Mocking utilities +│ │ ├── index.ts # Barrel export +│ │ ├── apiMocks.ts # Fetch API mocks +│ │ ├── jqueryMock.ts # jQuery mocks +│ │ └── browserMocks.ts # Browser API mocks +│ ├── utils/ # Test utilities +│ │ ├── testUtils.tsx # Common test helpers +│ │ └── setupHelpers.ts # Setup patterns +│ └── README.md # This file +├── components/ +│ └── MyComponent.test.tsx +└── router/ + └── QueryList.test.jsx +``` + +## Test Utilities + +The `testUtils.tsx` file provides common helpers for testing. Import from `__tests__/utils/testUtils`. + +### Loading State Helpers + +```javascript +import { waitForLoadingToFinish, expectLoading } from "../__tests__/utils/testUtils"; + +// Wait for loading to finish +render(); +await waitForLoadingToFinish(); + +// Assert loading is present +render(); +expectLoading(); +``` + +### Text Content Matchers + +```javascript +import { findByTextContent, getByTextContent, queryByTextContent } from "../__tests__/utils/testUtils"; + +// Find by exact text content (async) +const element = await findByTextContent("Total: 100"); + +// Get by exact text content (sync) +const element = getByTextContent("Total: 100"); + +// Query by exact text content (returns null if not found) +const element = queryByTextContent("Total: 100"); +``` + +### Dropdown Helpers + +```javascript +import { clickDropdownOption, selectDropdownByRole } from "../__tests__/utils/testUtils"; + +// Click dropdown and select option +await clickDropdownOption("Sort By", "Name"); + +// Select by role +await selectDropdownByRole("combobox", "Option 1"); +``` + +### Form Helpers + +```javascript +import { + typeIntoInput, + setInputValue, + clickButton, + clickButtonSync, + appendToInput, +} from "../__tests__/utils/testUtils"; + +// Type into input (clears first, realistic typing) +await typeIntoInput(/search/i, "test query"); + +// Set input value directly (faster, for non-interactive tests) +setInputValue(/search/i, "test query"); + +// Click button (realistic interaction) +await clickButton(/submit/i); + +// Click button synchronously (faster, for non-interactive tests) +clickButtonSync(/submit/i); + +// Append to input (doesn't clear) +await appendToInput(/search/i, " more text"); +``` + +**When to use async vs sync helpers:** + +- Use **async helpers** (`clickButton`, `typeIntoInput`) when testing user interactions that need realistic behavior +- Use **sync helpers** (`clickButtonSync`, `setInputValue`) for faster tests that don't need interaction simulation + +### Timer Helpers + +```javascript +import { advanceTimersAndWait, typeWithDebounce } from "../__tests__/utils/testUtils"; + +// Advance timers and wait for updates +await typeIntoInput(/search/i, "test"); +await advanceTimersAndWait(300); // Wait for debounce + +// Type with automatic debounce handling +const input = screen.getByPlaceholderText(/search/i); +await typeWithDebounce(input, "test", 300); +``` + +### Interaction Helpers + +```javascript +import { clickAndWait, clickAndWaitForRemoval } from "../__tests__/utils/testUtils"; + +// Click and wait for text to appear +await clickAndWait(button, /success/i); + +// Click and wait for element to disappear +await clickAndWaitForRemoval(deleteButton); +``` + +## Setup Helpers + +The `setupHelpers.ts` file provides component-specific setup patterns. Import from `__tests__/utils/setupHelpers`. + +### QueryList Setup + +```javascript +import { setupQueryListTest } from "../__tests__/utils/setupHelpers"; +import { createRunningQuery, createFinishedQuery } from "../__tests__/fixtures"; + +const { mockQueries } = setupQueryListTest([ + createRunningQuery({ queryId: "q1" }), + createFinishedQuery({ queryId: "q2" }), +]); + +render(); +``` + +### QueryDetail Setup + +```javascript +import { setupQueryDetailTest } from "../__tests__/utils/setupHelpers"; +import { createMockQuery } from "../__tests__/fixtures"; + +const { mockQuery } = setupQueryDetailTest(createMockQuery({ queryId: "test_123" }), { includeStages: true }); + +render(); +``` + +### ClusterHUD Setup + +```javascript +import { setupClusterHUDTest } from "../__tests__/utils/setupHelpers"; + +const { clusterData } = setupClusterHUDTest({ + clusterData: { runningQueries: 10 }, + workerData: [{ workerId: "w1", state: "ACTIVE" }], +}); + +render(); +``` + +### Common Setup Patterns + +```javascript +import { setupCommonMocks, setupFakeTimers, setupIntegrationTest } from "../__tests__/utils/setupHelpers"; + +// Setup common mocks (use in describe block) +describe("MyComponent", () => { + setupCommonMocks(); + + it("test", () => { + /* ... */ + }); +}); + +// Setup fake timers (use in describe block) +describe("MyComponent", () => { + setupFakeTimers(); + + it("test with timers", () => { + /* ... */ + }); +}); + +// Setup for integration tests (combines both) +describe("MyComponent Integration", () => { + setupIntegrationTest(); + + it("integration test", async () => { + /* ... */ + }); +}); +``` + +## Mocks + +Mocks simulate external dependencies. Import from `__tests__/mocks`. + +### jQuery Mocks + +```javascript +import { mockJQueryGet, mockJQueryAjax } from "../__tests__/mocks"; + +// Mock $.get +mockJQueryGet("/api/query", { queryId: "123", state: "RUNNING" }); + +// Mock $.ajax +mockJQueryAjax({ + url: "/api/query", + success: (data) => data, + error: (xhr, status, error) => error, +}); +``` + +### Fetch API Mocks + +```javascript +import { mockFetchByUrl } from "../__tests__/mocks"; +import { createMockQuery } from "../__tests__/fixtures"; + +mockFetchByUrl({ + "/v1/query": [createMockQuery()], + "/v1/info": { runningQueries: 5 }, +}); +``` + +### Browser API Mocks + +```javascript +import { setupAllBrowserMocks } from "../__tests__/mocks"; + +beforeEach(() => { + setupAllBrowserMocks(); +}); +``` + +## Fixtures + +Fixtures provide test data with sensible defaults. Import from `__tests__/fixtures`. + +### Query Fixtures + +```javascript +import { createMockQuery, createRunningQuery, createFinishedQuery, createFailedQuery } from "../__tests__/fixtures"; + +// Basic query +const query = createMockQuery(); + +// Running query with overrides +const runningQuery = createRunningQuery({ + queryId: "custom_123", + query: "SELECT * FROM users", +}); + +// Finished query +const finishedQuery = createFinishedQuery({ + elapsedTime: "5.2s", +}); + +// Failed query +const failedQuery = createFailedQuery({ + errorType: "USER_ERROR", + errorCode: { name: "SYNTAX_ERROR" }, +}); +``` + +### Other Fixtures + +```javascript +import { createMockInfo, createMockStage, createMockCluster } from "../__tests__/fixtures"; + +const info = createMockInfo({ runningQueries: 10 }); +const stage = createMockStage({ stageId: "0" }); +const cluster = createMockCluster({ activeWorkers: 5 }); +``` + +## Testing Patterns + +### Unit Testing + +```jsx +import { render, screen } from "../__tests__/utils/testUtils"; +import { setupCommonMocks } from "../__tests__/utils/setupHelpers"; +import { createMockQuery } from "../__tests__/fixtures"; +import QueryListItem from "./QueryListItem"; + +describe("QueryListItem", () => { + setupCommonMocks(); + + it("displays query ID", () => { + const query = createMockQuery({ queryId: "test_123" }); + render(); + expect(screen.getByText("test_123")).toBeInTheDocument(); + }); +}); +``` + +### Integration Testing + +```jsx +import { render, screen } from "../__tests__/utils/testUtils"; +import { setupIntegrationTest, setupQueryListTest } from "../__tests__/utils/setupHelpers"; +import { clickButton } from "../__tests__/utils/testUtils"; +import { createRunningQuery, createFinishedQuery } from "../__tests__/fixtures"; +import QueryList from "./QueryList"; + +describe("QueryList Integration", () => { + setupIntegrationTest(); + + it("filters queries by state", async () => { + setupQueryListTest([createRunningQuery({ queryId: "q1" }), createFinishedQuery({ queryId: "q2" })]); + + render(); + await clickButton(/FINISHED/i); + + expect(screen.getByText("q2")).toBeInTheDocument(); + expect(screen.queryByText("q1")).not.toBeInTheDocument(); + }); +}); +``` + +### Testing with Timers + +```jsx +import { render, screen } from "../__tests__/utils/testUtils"; +import { setupFakeTimers } from "../__tests__/utils/setupHelpers"; +import { typeIntoInput, advanceTimersAndWait } from "../__tests__/utils/testUtils"; + +describe("SearchComponent", () => { + setupFakeTimers(); + + it("debounces search input", async () => { + render(); + + await typeIntoInput(/search/i, "test"); + await advanceTimersAndWait(300); + + expect(screen.getByText(/results for "test"/i)).toBeInTheDocument(); + }); +}); +``` + +## Configuration + +### Warning Suppression + +The test setup automatically suppresses harmless Jest fake timer warnings in `setupTests.ts`: + +```typescript +// Suppress harmless fake timer warnings +// These occur when async helpers call jest.advanceTimersByTime() +// but fake timers ARE properly set up in beforeEach +const originalWarn = console.warn; +console.warn = (...args: any[]) => { + const message = args[0]?.toString() || ""; + + // Suppress fake timer warnings - they're harmless + if (message.includes("A function to advance timers was called but the timers APIs are not replaced")) { + return; + } + + // Allow all other warnings + originalWarn.apply(console, args); +}; +``` + +This ensures clean test output without suppressing legitimate warnings. + +### Test Environment + +- **Jest 29.7.0** with ts-jest and babel-jest +- **React Testing Library 16.1.0** for component testing +- **@testing-library/user-event 14.5.2** for user interaction simulation +- **Fake Timers** enabled by default for timer-dependent tests +- **ESLint + Prettier** for code quality and formatting + +## Best Practices + +### 1. Use Setup Helpers + +✅ **Good**: Use setup helpers + +```javascript +describe("QueryList", () => { + setupCommonMocks(); + + it("test", () => { + setupQueryListTest([createMockQuery()]); + render(); + }); +}); +``` + +⌠**Bad**: Manual setup + +```javascript +it("test", () => { + jest.clearAllMocks(); + setupAllBrowserMocks(); + mockFetchByUrl({ "/v1/query": [createMockQuery()] }); + render(); +}); +``` + +### 2. Use Test Utilities + +✅ **Good**: Use helper functions + +```javascript +await clickButton(/submit/i); +await typeIntoInput(/search/i, "test"); +``` + +⌠**Bad**: Manual interactions + +```javascript +const button = screen.getByRole("button", { name: /submit/i }); +await userEvent.click(button); +const input = screen.getByPlaceholderText(/search/i); +await userEvent.type(input, "test"); +``` + +### 3. Use Fixtures + +✅ **Good**: Use fixtures + +```javascript +const query = createRunningQuery({ queryId: "test" }); +``` + +⌠**Bad**: Manual objects + +```javascript +const query = { + queryId: "test", + state: "RUNNING", + // ... 50 more properties +}; +``` + +### 4. Test User Behavior + +✅ **Good**: Test what users see + +```javascript +expect(screen.getByText("RUNNING")).toBeInTheDocument(); +``` + +⌠**Bad**: Test implementation + +```javascript +expect(component.state.queries[0].state).toBe("RUNNING"); +``` + +### 5. Choose Appropriate Helpers + +✅ **Good**: Use sync helpers for speed when interaction realism isn't needed + +```javascript +clickButtonSync(/submit/i); +setInputValue(/search/i, "test"); +await advanceTimersAndWait(300); +``` + +⌠**Bad**: Using slow async helpers unnecessarily + +```javascript +await clickButton(/submit/i); // Slower, not needed for simple state changes +await typeIntoInput(/search/i, "test"); // Much slower +``` + +**Rule of thumb**: Use sync helpers (`clickButtonSync`, `setInputValue`) unless you're specifically testing user interaction behavior. + +### 6. Descriptive Test Names + +✅ **Good**: Clear intent + +```javascript +it("displays error message when API call fails", () => { + // test +}); +``` + +⌠**Bad**: Vague description + +```javascript +it("works", () => { + // test +}); +``` + +## Example: Before vs After + +### Before (Without Helpers) + +```javascript +it("filters queries by state", async () => { + const queries = [createRunningQuery({ queryId: "q1" }), createFinishedQuery({ queryId: "q2" })]; + + jest.clearAllMocks(); + setupAllBrowserMocks(); + mockFetchByUrl({ "/v1/query": queries }); + + render(); + + const finishedButton = screen.getByRole("button", { name: /FINISHED/i }); + await userEvent.click(finishedButton); + + expect(screen.getByText("q2")).toBeInTheDocument(); + expect(screen.queryByText("q1")).not.toBeInTheDocument(); +}); +``` + +### After (With Helpers) + +```javascript +describe("QueryList", () => { + setupCommonMocks(); + + it("filters queries by state", async () => { + setupQueryListTest([createRunningQuery({ queryId: "q1" }), createFinishedQuery({ queryId: "q2" })]); + + render(); + await clickButton(/FINISHED/i); + + expect(screen.getByText("q2")).toBeInTheDocument(); + expect(screen.queryByText("q1")).not.toBeInTheDocument(); + }); +}); +``` + +**Improvements:** + +- 30% less code +- Clearer intent +- Easier to maintain +- Reusable patterns + +## Additional Resources + +- [Jest Documentation](https://jestjs.io/docs/getting-started) +- [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) +- [Testing Library Queries](https://testing-library.com/docs/queries/about) + +--- + +**Made with Bob** 🤖 diff --git a/presto-ui/src/__tests__/fixtures/clusterFixtures.ts b/presto-ui/src/__tests__/fixtures/clusterFixtures.ts new file mode 100644 index 0000000000000..27c59cdbbbb77 --- /dev/null +++ b/presto-ui/src/__tests__/fixtures/clusterFixtures.ts @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared test fixtures for cluster-related tests + * Provides factory functions to create mock cluster info objects with sensible defaults + */ + +/** + * Base mock cluster info object with all required fields + */ +export const baseMockClusterInfo = { + runningQueries: 5, + queuedQueries: 2, + blockedQueries: 1, + activeWorkers: 10, + runningDrivers: 50, + reservedMemory: 1073741824, // 1GB in bytes + totalInputRows: 1000000, + totalInputBytes: 1073741824, + totalCpuTimeSecs: 100, +}; + +/** + * Create a mock cluster info object with custom overrides + * @param overrides - Properties to override in the base cluster info + * @returns Mock cluster info object + * @example + * const busyCluster = createMockClusterInfo({ runningQueries: 50, activeWorkers: 100 }); + */ +export const createMockClusterInfo = (overrides: Partial = {}) => ({ + ...baseMockClusterInfo, + ...overrides, +}); + +/** + * Create a mock cluster info with high load + */ +export const createHighLoadCluster = (overrides: Partial = {}) => + createMockClusterInfo({ + runningQueries: 50, + queuedQueries: 20, + blockedQueries: 10, + activeWorkers: 100, + ...overrides, + }); + +/** + * Create a mock cluster info with low load + */ +export const createLowLoadCluster = (overrides: Partial = {}) => + createMockClusterInfo({ + runningQueries: 1, + queuedQueries: 0, + blockedQueries: 0, + activeWorkers: 5, + ...overrides, + }); + +/** + * Create a mock cluster info with no activity + */ +export const createIdleCluster = (overrides: Partial = {}) => + createMockClusterInfo({ + runningQueries: 0, + queuedQueries: 0, + blockedQueries: 0, + activeWorkers: 0, + runningDrivers: 0, + ...overrides, + }); + +// Made with Bob diff --git a/presto-ui/src/__tests__/fixtures/index.ts b/presto-ui/src/__tests__/fixtures/index.ts new file mode 100644 index 0000000000000..75577d4a490af --- /dev/null +++ b/presto-ui/src/__tests__/fixtures/index.ts @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Barrel export for all test fixtures + * Import all fixtures from a single location for convenience + * + * @example + * import { createMockQuery, createMockStage, createMockClusterInfo } from '../__tests__/fixtures'; + */ + +// Query fixtures +export * from "./queryFixtures"; + +// Info/cluster API fixtures +export * from "./infoFixtures"; + +// Stage fixtures +export * from "./stageFixtures"; + +// Cluster info fixtures +export * from "./clusterFixtures"; + +// Made with Bob diff --git a/presto-ui/src/__tests__/fixtures/infoFixtures.ts b/presto-ui/src/__tests__/fixtures/infoFixtures.ts new file mode 100644 index 0000000000000..40e1837d795b3 --- /dev/null +++ b/presto-ui/src/__tests__/fixtures/infoFixtures.ts @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared test fixtures for info/cluster API responses + * Used primarily by PageTitle component tests + */ + +import { mockFetchByUrl } from "../mocks/apiMocks"; + +/** + * Base mock info response + */ +export const baseMockInfo = { + nodeVersion: { version: "1.0.0" }, + environment: "test", + uptime: "1h", +}; + +/** + * Base mock cluster response + */ +export const baseMockCluster = { + clusterTag: "", +}; + +/** + * Create a mock info object with custom overrides + * @param overrides - Properties to override in the base info + * @returns Mock info object + */ +export const createMockInfo = (overrides: Partial = {}) => ({ + ...baseMockInfo, + ...overrides, + nodeVersion: { + ...baseMockInfo.nodeVersion, + ...(overrides.nodeVersion || {}), + }, +}); + +/** + * Create a mock cluster object with custom overrides + * @param overrides - Properties to override in the base cluster + * @returns Mock cluster object + */ +export const createMockCluster = (overrides: Partial = {}) => ({ + ...baseMockCluster, + ...overrides, +}); + +/** + * Setup PageTitle test with standard API mocks + * This is the most common pattern in PageTitle tests + * @param infoOverrides - Custom info response data + * @param clusterOverrides - Custom cluster response data + */ +export const setupPageTitleTest = ( + infoOverrides: Partial = {}, + clusterOverrides: Partial = {} +) => { + mockFetchByUrl({ + "/v1/info": createMockInfo(infoOverrides), + "/v1/cluster": createMockCluster(clusterOverrides), + }); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/fixtures/queryFixtures.js b/presto-ui/src/__tests__/fixtures/queryFixtures.js new file mode 100644 index 0000000000000..ab7b5b3964c13 --- /dev/null +++ b/presto-ui/src/__tests__/fixtures/queryFixtures.js @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared test fixtures for query-related tests + * Provides factory functions to create mock query objects with sensible defaults + */ + +/** + * Base mock query object with all required fields + */ +export const baseMockQuery = { + queryId: "test_query_123", + state: "RUNNING", + query: "SELECT * FROM table WHERE id = 1", + session: { + user: "testuser", + source: "presto-ui", + }, + queryStats: { + createTime: "2024-01-01T10:00:00.000Z", + elapsedTime: "5.2s", + executionTime: "4.8s", + totalCpuTime: "10.5s", + totalDrivers: 15, + queuedDrivers: 0, + runningDrivers: 5, + completedDrivers: 10, + cumulativeUserMemory: 1073741824, + userMemoryReservation: "1GB", + totalMemoryReservation: "2GB", + peakUserMemoryReservation: "1.5GB", + peakTotalMemoryReservation: "1.5GB", + rawInputDataSize: "500MB", + rawInputPositions: 1000000, + }, + errorType: null, + errorCode: null, + coordinatorUri: "", + scheduled: true, + fullyBlocked: false, + blockedReasons: [], + memoryPool: "general", +}; + +/** + * Create a mock query with custom overrides + * @param {Object} overrides - Properties to override in the base query + * @returns {Object} Mock query object + * @example + * const finishedQuery = createMockQuery({ state: "FINISHED", queryId: "query_456" }); + */ +export const createMockQuery = (overrides = {}) => { + // Deep merge for nested objects like queryStats + const merged = { ...baseMockQuery, ...overrides }; + if (overrides.queryStats) { + merged.queryStats = { ...baseMockQuery.queryStats, ...overrides.queryStats }; + } + if (overrides.session) { + merged.session = { ...baseMockQuery.session, ...overrides.session }; + } + return merged; +}; + +/** + * Create a mock query in RUNNING state + */ +export const createRunningQuery = (overrides = {}) => createMockQuery({ state: "RUNNING", ...overrides }); + +/** + * Create a mock query in FINISHED state + */ +export const createFinishedQuery = (overrides = {}) => createMockQuery({ state: "FINISHED", ...overrides }); + +/** + * Create a mock query in FAILED state + */ +export const createFailedQuery = (overrides = {}) => + createMockQuery({ + state: "FAILED", + errorType: "USER_ERROR", + errorCode: null, + ...overrides, + }); + +/** + * Create a mock query in QUEUED state + */ +export const createQueuedQuery = (overrides = {}) => createMockQuery({ state: "QUEUED", ...overrides }); + +/** + * Create multiple mock queries with sequential IDs + * @param {number} count - Number of queries to create + * @param {Object} baseOverrides - Base overrides to apply to all queries + * @returns {Array} Array of mock query objects + */ +export const createMockQueries = (count, baseOverrides = {}) => { + return Array.from({ length: count }, (_, i) => + createMockQuery({ + queryId: `test_query_${i + 1}`, + ...baseOverrides, + }) + ); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/fixtures/stageFixtures.ts b/presto-ui/src/__tests__/fixtures/stageFixtures.ts new file mode 100644 index 0000000000000..ed117d3a59721 --- /dev/null +++ b/presto-ui/src/__tests__/fixtures/stageFixtures.ts @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Shared test fixtures for stage-related tests + * Provides factory functions to create mock stage objects with sensible defaults + */ + +/** + * Base mock stage object with all required fields + */ +export const baseMockStage = { + stageId: "0", + state: "RUNNING", + stageStats: { + totalDrivers: 10, + queuedDrivers: 0, + runningDrivers: 5, + completedDrivers: 5, + fullyBlocked: false, + }, +}; + +/** + * Create a mock stage with custom overrides + * @param overrides - Properties to override in the base stage + * @returns Mock stage object + * @example + * const finishedStage = createMockStage({ state: "FINISHED", stageId: "1" }); + */ +export const createMockStage = (overrides: Partial = {}) => { + const merged = { ...baseMockStage, ...overrides }; + if (overrides.stageStats) { + merged.stageStats = { ...baseMockStage.stageStats, ...overrides.stageStats }; + } + return merged; +}; + +/** + * Create a mock stage in RUNNING state + */ +export const createRunningStage = (overrides: Partial = {}) => + createMockStage({ state: "RUNNING", ...overrides }); + +/** + * Create a mock stage in FINISHED state + */ +export const createFinishedStage = (overrides: Partial = {}) => + createMockStage({ state: "FINISHED", ...overrides }); + +/** + * Create a mock stage in FAILED state + */ +export const createFailedStage = (overrides: Partial = {}) => + createMockStage({ state: "FAILED", ...overrides }); + +/** + * Create multiple mock stages with sequential IDs + * @param count - Number of stages to create + * @param baseOverrides - Base overrides to apply to all stages + * @returns Array of mock stage objects + */ +export const createMockStages = (count: number, baseOverrides: Partial = {}) => { + return Array.from({ length: count }, (_, i) => + createMockStage({ + stageId: `${i}`, + ...baseOverrides, + }) + ); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/mocks/apiMocks.ts b/presto-ui/src/__tests__/mocks/apiMocks.ts new file mode 100644 index 0000000000000..57a10987dbc79 --- /dev/null +++ b/presto-ui/src/__tests__/mocks/apiMocks.ts @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * API Mocking Utilities + * Provides helpers for mocking fetch API calls in tests + * + * NOTE: This file contains only mocking mechanisms (functions). + * For test data, import from __tests__/fixtures: + * - queryFixtures.js - Query objects + * - infoFixtures.ts - Info/cluster data + * - stageFixtures.ts - Stage data + * - clusterFixtures.ts - Cluster data + */ + +// ============================================================================ +// Response Creation Helpers +// ============================================================================ + +/** + * Create a successful fetch response object + */ +const createSuccessResponse = (data: any): Partial => ({ + ok: true, + status: 200, + json: async () => data, + text: async () => JSON.stringify(data), +}); + +/** + * Create a failed fetch response object + */ +const createErrorResponse = (status: number, message: string): Partial => ({ + ok: false, + status, + statusText: message, + json: async () => { + throw new Error(message); + }, + text: async () => message, +}); + +// ============================================================================ +// Fetch Mock Setup +// ============================================================================ + +/** + * Initialize fetch mock (call in beforeEach) + */ +export const setupFetchMock = () => { + global.fetch = jest.fn(); +}; + +/** + * Reset fetch mock (call in afterEach) + */ +export const resetFetchMock = () => { + (global.fetch as jest.Mock).mockReset(); +}; + +// ============================================================================ +// URL-Based Mocking (Recommended) +// ============================================================================ + +/** + * Mock fetch responses for specific URLs + * @param urlMap - Map of URL to response data + * @example + * mockFetchByUrl({ + * "/v1/query": [mockQuery], + * "/v1/info": mockInfo, + * }); + */ +export const mockFetchByUrl = (urlMap: Record) => { + (global.fetch as jest.Mock).mockImplementation((url: string) => { + const data = urlMap[url]; + if (data !== undefined) { + return Promise.resolve(createSuccessResponse(data)); + } + // Return empty success response for unmapped URLs + return Promise.resolve(createSuccessResponse({})); + }); +}; + +/** + * Mock fetch error for a specific URL + * @param url - The URL to mock + * @param status - HTTP status code (default: 500) + * @param message - Error message (default: "Internal Server Error") + */ +export const mockFetchErrorByUrl = (url: string, status: number = 500, message: string = "Internal Server Error") => { + (global.fetch as jest.Mock).mockImplementation((fetchUrl: string) => { + if (fetchUrl === url) { + return Promise.resolve(createErrorResponse(status, message)); + } + // Return success for other URLs + return Promise.resolve(createSuccessResponse({})); + }); +}; + +// ============================================================================ +// Legacy Single-Call Mocking (For Backward Compatibility) +// ============================================================================ + +/** + * Mock a single successful fetch call (legacy) + * @deprecated Use mockFetchByUrl for better control + */ +export const mockFetchSuccess = (data: any) => { + (global.fetch as jest.Mock).mockResolvedValueOnce(createSuccessResponse(data)); +}; + +/** + * Mock a single failed fetch call (legacy) + * @deprecated Use mockFetchErrorByUrl for better control + */ +export const mockFetchError = (message: string, _status: number = 500) => { + (global.fetch as jest.Mock).mockRejectedValueOnce(new Error(message)); +}; + +/** + * Mock fetch with custom response (legacy) + * @deprecated Use mockFetchByUrl or create custom implementation + */ +export const mockFetchResponse = (response: Partial) => { + (global.fetch as jest.Mock).mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({}), + text: async () => "", + ...response, + }); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/mocks/browserMocks.ts b/presto-ui/src/__tests__/mocks/browserMocks.ts new file mode 100644 index 0000000000000..4d78607a7ec6c --- /dev/null +++ b/presto-ui/src/__tests__/mocks/browserMocks.ts @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Browser API mocks for testing + * Provides mock implementations of browser APIs used throughout the application + */ + +/** + * Mock window.matchMedia + * Used for responsive design and media query testing + */ +export const setupMatchMediaMock = () => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: jest.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: jest.fn(), + removeListener: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + dispatchEvent: jest.fn(), + })), + }); +}; + +/** + * NOTE: Fetch mocking is handled in apiMocks.ts + * This keeps all HTTP/API mocking logic in one place. + * Import setupFetchMock from apiMocks.ts if needed. + */ + +/** + * Mock highlight.js (hljs) + * Used for syntax highlighting in code blocks + */ +export const setupHljsMock = () => { + (global as any).hljs = { + highlightBlock: jest.fn(), + highlightElement: jest.fn(), + highlight: jest.fn().mockReturnValue({ value: "" }), + }; +}; + +/** + * Mock Clipboard API + * Used for copy-to-clipboard functionality + */ +export const setupClipboardMock = () => { + (global as any).Clipboard = jest.fn(); +}; + +/** + * Mock document.body extensions + * Some legacy code may use non-standard document.body methods + */ +export const setupDocumentMocks = () => { + if (!(document.body as any).createTextRange) { + (document.body as any).createTextRange = jest.fn(); + } +}; + +/** + * Setup console error suppression for known React warnings + * Suppresses specific warnings that are expected in test environment + */ +export const setupConsoleErrorSuppression = () => { + const originalError = console.error; + + beforeAll(() => { + console.error = (...args: any[]) => { + // Suppress specific React warnings that are expected in tests + if ( + typeof args[0] === "string" && + (args[0].includes("Warning: ReactDOM.render") || + args[0].includes("Warning: useLayoutEffect") || + args[0].includes("Not implemented: HTMLFormElement.prototype.submit")) + ) { + return; + } + originalError.call(console, ...args); + }; + }); + + afterAll(() => { + console.error = originalError; + }); +}; + +/** + * Setup all browser mocks at once + * Convenience function to initialize all browser API mocks + * + * NOTE: Fetch mocking is NOT included here - it's handled separately + * in apiMocks.ts to keep all HTTP/API mocking logic centralized. + */ +export const setupAllBrowserMocks = () => { + setupMatchMediaMock(); + setupHljsMock(); + setupClipboardMock(); + setupDocumentMocks(); + setupConsoleErrorSuppression(); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/mocks/index.ts b/presto-ui/src/__tests__/mocks/index.ts new file mode 100644 index 0000000000000..23e2e84860afd --- /dev/null +++ b/presto-ui/src/__tests__/mocks/index.ts @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Barrel export for all test mocks + * Import all mocks from a single location for convenience + * + * @example + * import { mockJQueryGet, mockFetchByUrl, setupAllBrowserMocks } from '../__tests__/mocks'; + */ + +// jQuery mocks +export * from "./jqueryMock"; + +// API mocks (fetch) - Centralized in apiMocks.ts +export { + mockFetchByUrl, + mockFetchErrorByUrl, + mockFetchSuccess, + mockFetchError, + mockFetchResponse, + setupFetchMock, + resetFetchMock, +} from "./apiMocks"; + +// Browser API mocks (matchMedia, hljs, clipboard, document, console) +// Note: Fetch mocking is NOT included here - it's in apiMocks.ts +export { + setupMatchMediaMock, + setupHljsMock, + setupClipboardMock, + setupDocumentMocks, + setupConsoleErrorSuppression, + setupAllBrowserMocks, +} from "./browserMocks"; + +// Made with Bob diff --git a/presto-ui/src/__tests__/mocks/jqueryMock.ts b/presto-ui/src/__tests__/mocks/jqueryMock.ts new file mode 100644 index 0000000000000..993dfb3b163f1 --- /dev/null +++ b/presto-ui/src/__tests__/mocks/jqueryMock.ts @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * jQuery Setup for Tests + * + * Uses the SAME jQuery version as production (3.7.1 from npm package) + * with selective mocking of only the problematic parts: + * - jQuery plugins (sparkline, modal, tooltip) that require browser APIs + * - Animation methods (made instant for test speed) + * + * Benefits: + * - Uses exact same jQuery version as production (3.7.1) + * - Real jQuery behavior (DOM manipulation, selectors, utilities) + * - Utility functions work correctly ($.extend, $.isArray, $.each, etc.) + * - Better test confidence + * - Less maintenance burden + * - Proper TypeScript types + */ + +// Import jQuery 3.7.1 from npm - same version as production +import $ from "jquery"; + +/** + * Setup jQuery with selective mocking + * Call this once in setupTests.ts to configure jQuery for the test environment + */ +/** + * Helper to create instant animation mock that calls callback immediately + * @param displayAction - The display action to perform (show/hide) + */ +const createInstantAnimationMock = (displayAction: "show" | "hide") => { + return function (this: JQuery, ...args: any[]): JQuery { + // Perform the display action immediately + this[displayAction](); + + // Extract and call the callback if provided + const callback = args[args.length - 1]; + if (typeof callback === "function") { + callback.call(this); + } + + return this; + }; +}; + +export const setupJQuery = (): JQueryStatic => { + // Mock jQuery plugins that require browser APIs or external dependencies + // These are used in the application but don't need real implementations in tests + + // Sparkline plugin - used for rendering charts + // @ts-expect-error - Adding plugin method to jQuery + $.fn.sparkline = jest.fn().mockReturnThis(); + + // Bootstrap modal plugin - used for showing/hiding modals + // @ts-expect-error - Adding plugin method to jQuery + $.fn.modal = jest.fn().mockReturnThis(); + + // Bootstrap tooltip plugin - used for tooltips + // @ts-expect-error - Adding plugin method to jQuery + $.fn.tooltip = jest.fn().mockReturnThis(); + + // Mock animation methods to be instant in tests (for speed) + // Override animate to skip animation and call complete callback immediately + $.fn.animate = function (this: JQuery, ...args: any[]): JQuery { + const options = args[1]; + if (typeof options === "object" && options?.complete) { + options.complete.call(this); + } + return this; + }; + + // Override fade/slide methods to be instant using helper + $.fn.fadeIn = createInstantAnimationMock("show"); + $.fn.fadeOut = createInstantAnimationMock("hide"); + $.fn.slideDown = createInstantAnimationMock("show"); + $.fn.slideUp = createInstantAnimationMock("hide"); + + // Note: We do NOT mock AJAX methods ($.ajax, $.get, $.post, etc.) + // because Presto UI uses the fetch() API instead of jQuery AJAX. + // The fetch() API is already mocked in apiMocks.ts. + + // Set jQuery as global for components that use it + (global as any).$ = $; + (global as any).jQuery = $; + + return $; +}; + +/** + * Mock jQuery.get() for tests + * This is a test helper that mocks jQuery AJAX GET requests + * Maps URLs to response data for testing + * + * Supports both callback and promise-style usage: + * - $.get(url, callback) - callback style + * - $.get(url).done(callback) - promise style + * + * Uses jQuery's native Deferred object for proper promise behavior + * + * @param urlToDataMap - Map of URL to response data + * + * @example + * mockJQueryGet({ + * '/v1/query': [query1, query2], + * '/v1/info': { version: '1.0.0' } + * }); + */ +export const mockJQueryGet = (urlToDataMap: Record): void => { + // Mock $.get using jQuery's native Deferred for proper promise behavior + ($ as any).get = jest.fn((url: string, callback?: (data: any) => void) => { + const data = urlToDataMap[url]; + + const deferred = $.Deferred(); + if (data !== undefined) { + deferred.resolve(data); + + // callback style + if (typeof callback === "function") { + callback(data); + } + } else { + deferred.reject(); + } + + return deferred.promise(); + }); +}; + +// Made with Bob diff --git a/presto-ui/src/__tests__/utils/setupHelpers.ts b/presto-ui/src/__tests__/utils/setupHelpers.ts new file mode 100644 index 0000000000000..98d615c385535 --- /dev/null +++ b/presto-ui/src/__tests__/utils/setupHelpers.ts @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Component-Specific Setup Helpers + * Provides reusable setup patterns for common test scenarios + */ + +import { setupAllBrowserMocks } from "../mocks"; + +// ============================================================================ +// Common Test Setup Patterns +// ============================================================================ + +/** + * Setup fake timers with automatic cleanup + * Use in describe block to apply to all tests + * + * @example + * describe('MyComponent', () => { + * setupFakeTimers(); + * + * it('handles debounced input', () => { + * // timers are automatically set up and cleaned up + * }); + * }); + */ +export const setupFakeTimers = () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); +}; + +/** + * Setup common mocks for all tests + * Clears mocks and sets up browser APIs + * + * @example + * describe('MyComponent', () => { + * setupCommonMocks(); + * + * it('renders correctly', () => { + * // mocks are ready to use + * }); + * }); + */ +export const setupCommonMocks = () => { + beforeEach(() => { + jest.clearAllMocks(); + setupAllBrowserMocks(); + }); +}; + +/** + * Setup for integration tests + * Combines fake timers and common mocks + * + * @example + * describe('MyComponent Integration', () => { + * setupIntegrationTest(); + * + * it('handles user interactions', async () => { + * // ready for integration testing + * }); + * }); + */ +export const setupIntegrationTest = () => { + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); +}; + +// ============================================================================ +// Custom Setup Builders +// ============================================================================ + +// Made with Bob diff --git a/presto-ui/src/__tests__/utils/testUtils.tsx b/presto-ui/src/__tests__/utils/testUtils.tsx new file mode 100644 index 0000000000000..f8a5a5bb21784 --- /dev/null +++ b/presto-ui/src/__tests__/utils/testUtils.tsx @@ -0,0 +1,276 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Enhanced Test Utilities + * Provides common helpers for testing React components + */ + +import { waitFor, screen, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; + +// ============================================================================ +// Loading State Helpers +// ============================================================================ + +/** + * Wait for loading indicator to disappear + * @example + * render(); + * await waitForLoadingToFinish(); + */ +export const waitForLoadingToFinish = () => + waitFor(() => expect(screen.queryByText(/loading/i)).not.toBeInTheDocument()); + +/** + * Assert that loading indicator is present + * @example + * render(); + * expectLoading(); + */ +export const expectLoading = () => expect(screen.getByText(/loading/i)).toBeInTheDocument(); + +// ============================================================================ +// Text Content Matchers +// ============================================================================ + +/** + * Find element by exact text content (async) + * Useful when text is split across multiple elements + * @example + * const element = await findByTextContent('Total: 100'); + */ +export const findByTextContent = (text: string) => + screen.findByText((content, element) => { + return element?.textContent === text; + }); + +/** + * Get element by exact text content (sync) + * @example + * const element = getByTextContent('Total: 100'); + */ +export const getByTextContent = (text: string) => + screen.getByText((content, element) => { + return element?.textContent === text; + }); + +/** + * Query element by exact text content (returns null if not found) + * @example + * const element = queryByTextContent('Total: 100'); + * expect(element).toBeNull(); + */ +export const queryByTextContent = (text: string) => + screen.queryByText((content, element) => { + return element?.textContent === text; + }); + +// ============================================================================ +// Dropdown Interaction Helpers +// ============================================================================ + +/** + * Click a dropdown and select an option by text + * @example + * await clickDropdownOption('Sort By', 'Name'); + */ +export const clickDropdownOption = async (dropdownText: string, optionText: string) => { + const dropdown = screen.getByText(dropdownText); + await userEvent.click(dropdown); + const option = screen.getByText(optionText); + await userEvent.click(option); +}; + +/** + * Select dropdown option by role + * @example + * await selectDropdownByRole('combobox', 'Option 1'); + */ +export const selectDropdownByRole = async (role: string, optionText: string) => { + const dropdown = screen.getByRole(role); + await userEvent.click(dropdown); + const option = screen.getByText(optionText); + await userEvent.click(option); +}; + +// ============================================================================ +// Form Interaction Helpers +// ============================================================================ + +/** + * Type into an input field by placeholder (uses userEvent for realistic typing) + * @example + * await typeIntoInput(/search/i, 'test query'); + */ +export const typeIntoInput = async (placeholder: string | RegExp, text: string) => { + const input = screen.getByPlaceholderText(placeholder); + await userEvent.clear(input); + await userEvent.type(input, text); + return input; +}; + +/** + * Set input value directly (faster, uses fireEvent.change) + * Use for tests that don't need realistic typing simulation + * @example + * setInputValue(/search/i, 'test query'); + */ +export const setInputValue = (placeholder: string | RegExp, value: string) => { + const input = screen.getByPlaceholderText(placeholder); + fireEvent.change(input, { target: { value } }); + return input; +}; + +/** + * Click a button by accessible name (uses userEvent for realistic interaction) + * @example + * await clickButton(/submit/i); + */ +export const clickButton = async (text: string | RegExp) => { + const button = screen.getByRole("button", { name: text }); + await userEvent.click(button); + return button; +}; + +/** + * Click a button synchronously (uses fireEvent, faster) + * Use for tests that don't need realistic click simulation + * @example + * clickButtonSync(/submit/i); + */ +export const clickButtonSync = (text: string | RegExp) => { + const button = screen.getByRole("button", { name: text }); + fireEvent.click(button); + return button; +}; + +/** + * Type into input without clearing first + * @example + * await appendToInput(/search/i, ' more text'); + */ +export const appendToInput = async (placeholder: string | RegExp, text: string) => { + const input = screen.getByPlaceholderText(placeholder); + await userEvent.type(input, text); + return input; +}; + +// ============================================================================ +// Timer Helpers +// ============================================================================ + +/** + * Advance timers and wait for updates + * Useful for debounced operations + * + * IMPORTANT: Requires Jest fake timers to be enabled. + * Use setupFakeTimers() or setupIntegrationTest() in your test suite. + * + * @param ms - Milliseconds to advance + * @throws Error with helpful message if fake timers are not enabled + * @example + * describe('MyComponent', () => { + * setupFakeTimers(); // or setupIntegrationTest() + * + * it('handles debounced input', async () => { + * await typeIntoInput(/search/i, 'test'); + * await advanceTimersAndWait(300); // Wait for debounce + * }); + * }); + */ +export const advanceTimersAndWait = async (ms: number) => { + try { + jest.advanceTimersByTime(ms); + } catch (error) { + // Provide a clearer error when fake timers are not enabled + const originalMessage = error instanceof Error ? error.message : String(error); + throw new Error( + [ + "advanceTimersAndWait requires Jest fake timers to be enabled.", + "Make sure to call setupFakeTimers() or setupIntegrationTest() before using this helper.", + `Original Jest error: ${originalMessage}`, + ].join(" ") + ); + } + await waitFor(() => {}, { timeout: 0 }); +}; + +/** + * Simulate user typing with debounce + * @example + * const input = screen.getByPlaceholderText(/search/i); + * await typeWithDebounce(input, 'test', 300); + */ +export const typeWithDebounce = async (input: HTMLElement, text: string, debounceMs = 300) => { + await userEvent.type(input, text); + jest.advanceTimersByTime(debounceMs); + await waitFor(() => {}, { timeout: 0 }); +}; + +// ============================================================================ +// Interaction Helpers +// ============================================================================ + +/** + * Click element and wait for specific text to appear + * @example + * await clickAndWait(button, /success/i); + */ +export const clickAndWait = async (element: HTMLElement, waitForText?: string | RegExp) => { + await userEvent.click(element); + if (waitForText) { + await screen.findByText(waitForText); + } +}; + +/** + * Click element and wait for it to disappear + * @example + * await clickAndWaitForRemoval(deleteButton); + */ +export const clickAndWaitForRemoval = async (element: HTMLElement) => { + await userEvent.click(element); + await waitFor(() => expect(element).not.toBeInTheDocument()); +}; + +// ============================================================================ +// Query Helpers +// ============================================================================ + +/** + * Get all elements matching text pattern + * @example + * const items = getAllByTextPattern(/query_\d+/); + */ +export const getAllByTextPattern = (pattern: RegExp) => screen.getAllByText(pattern); + +/** + * Check if element with text exists + * @example + * expect(hasText('Success')).toBe(true); + */ +export const hasText = (text: string | RegExp) => screen.queryByText(text) !== null; + +// ============================================================================ +// Exports +// ============================================================================ + +// Export all from @testing-library/react +export * from "@testing-library/react"; + +// Export userEvent for convenience +export { userEvent }; + +// Made with Bob diff --git a/presto-ui/src/babel.config.js b/presto-ui/src/babel.config.js new file mode 100644 index 0000000000000..a46f1d681965f --- /dev/null +++ b/presto-ui/src/babel.config.js @@ -0,0 +1,9 @@ +// eslint-disable-next-line no-undef +module.exports = { + presets: [ + ["@babel/preset-env", { targets: { node: "current" } }], + ["@babel/preset-react", { runtime: "automatic" }], + ], +}; + +// Made with Bob diff --git a/presto-ui/src/components/PageTitle.test.tsx b/presto-ui/src/components/PageTitle.test.tsx new file mode 100644 index 0000000000000..7dab1bb2b1429 --- /dev/null +++ b/presto-ui/src/components/PageTitle.test.tsx @@ -0,0 +1,192 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from "react"; +import { render, screen, waitFor } from "../__tests__/utils/testUtils"; +import { PageTitle } from "./PageTitle"; +import { setupPageTitleTest } from "../__tests__/fixtures/infoFixtures"; + +describe("PageTitle", () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + + // Mock window.location using Object.defineProperty for better type safety + // Currently only mocking 'protocol' as that's all the component checks + // Add other properties (href, hostname, pathname, etc.) if component needs them + Object.defineProperty(window, "location", { + value: { + protocol: "http:", + // Add other properties here as needed + }, + writable: true, + configurable: true, + }); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + describe("Rendering", () => { + it("renders nothing initially when not offline", () => { + // Set up fetch mock to return pending promise + (global.fetch as jest.Mock).mockImplementation(() => new Promise(() => {})); + + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it("renders navbar after fetching info", async () => { + setupPageTitleTest(); + + render(); + + await waitFor(() => { + expect(screen.getByText("1.0.0")).toBeInTheDocument(); + }); + + expect(screen.getByText("test")).toBeInTheDocument(); + expect(screen.getByText("1h")).toBeInTheDocument(); + }); + + it("renders multiple navigation titles", async () => { + setupPageTitleTest({ environment: "prod", uptime: "5d" }, {}); + + render( + + ); + + await waitFor(() => { + expect(screen.getByText("Cluster")).toBeInTheDocument(); + }); + + expect(screen.getByText("Queries")).toBeInTheDocument(); + expect(screen.getByText("Workers")).toBeInTheDocument(); + }); + + it("renders cluster tag when available", async () => { + setupPageTitleTest( + { nodeVersion: { version: "2.0.0" }, environment: "staging", uptime: "3h" }, + { clusterTag: "my-cluster-tag" } + ); + + render(); + + await waitFor(() => { + expect(screen.getByText("my-cluster-tag")).toBeInTheDocument(); + }); + }); + + it("uses custom path for logo", async () => { + setupPageTitleTest(); + + render(); + + await waitFor(() => { + const img = screen.getByRole("img"); + expect(img).toHaveAttribute("src", "/custom/assets/logo.png"); + }); + }); + + it("displays version information", async () => { + setupPageTitleTest({ nodeVersion: { version: "1.2.3" }, environment: "production", uptime: "10d 5h" }, {}); + + render(); + + await waitFor(() => { + expect(screen.getByText("1.2.3")).toBeInTheDocument(); + expect(screen.getByText("production")).toBeInTheDocument(); + expect(screen.getByText("10d 5h")).toBeInTheDocument(); + }); + }); + }); + + describe("Offline Mode", () => { + it("detects offline protocol", () => { + (window as any).location.protocol = "file:"; + + // Component should handle offline mode + const { container } = render(); + expect(container).toBeInTheDocument(); + }); + }); + + describe("Connection Status", () => { + it("shows green status light when connected", async () => { + setupPageTitleTest(); + + render(); + + await waitFor(() => { + const statusLight = document.getElementById("status-indicator"); + expect(statusLight).toHaveClass("status-light-green"); + }); + }); + + it("calls fetch for info endpoint", async () => { + const fetchSpy = jest.spyOn(global, "fetch"); + setupPageTitleTest(); + + render(); + + await waitFor(() => { + expect(fetchSpy).toHaveBeenCalledWith("/v1/info"); + }); + }); + + it("calls fetch for cluster endpoint", async () => { + const fetchSpy = jest.spyOn(global, "fetch"); + setupPageTitleTest({}, { clusterTag: "test" }); + + render(); + + await waitFor(() => { + expect(fetchSpy).toHaveBeenCalledWith("/v1/cluster"); + }); + }); + }); + + describe("Navigation", () => { + it("renders logo link", async () => { + setupPageTitleTest(); + + render(); + + await waitFor(() => { + const links = screen.getAllByRole("link"); + const logoLink = links.find((link) => link.getAttribute("href") === "/ui/"); + expect(logoLink).toBeDefined(); + }); + }); + + it("renders navbar toggle button", async () => { + setupPageTitleTest(); + + render(); + + await waitFor(() => { + const toggleButton = screen.getByLabelText("Toggle navigation"); + expect(toggleButton).toBeInTheDocument(); + }); + }); + }); +}); + +// Made with Bob diff --git a/presto-ui/src/components/SQLInput.tsx b/presto-ui/src/components/SQLInput.tsx index d8d2bca8a49b1..ad9a06d84c9fe 100644 --- a/presto-ui/src/components/SQLInput.tsx +++ b/presto-ui/src/components/SQLInput.tsx @@ -67,22 +67,26 @@ class UpperCaseCharStream extends antlr4.CharStream { class SelectListener extends SqlBaseListener { limit = -1; fetchFirstNRows = -1; - isSelect = false; + isTopLevelSelect = false; constructor() { super(); } - enterQueryNoWith(ctx) { - super.enterQueryNoWith(ctx); - this.isSelect = true; + enterStatement(ctx) { + // Top-level SELECT only (not CTAS / INSERT) + if (ctx.query()) { + this.isTopLevelSelect = true; + } } + exitQueryNoWith(ctx) { super.exitQueryNoWith(ctx); this.limit = ctx.limit ? ctx.limit.text : -1; this.fetchFirstNRows = ctx.fetchFirstNRows ? ctx.fetchFirstNRows.text : -1; } } + class SyntaxError extends antlr4.ErrorListener { error = undefined; @@ -147,7 +151,7 @@ const sqlCleaning = (sql, errorHandler) => { errorHandler(syntaxError.error); return false; } - if (selectDetector.isSelect) { + if (selectDetector.isTopLevelSelect) { if (typeof selectDetector.limit === "string" || selectDetector.limit > 100) { cleanSql = cleanSql.replace(limitRE, "limit 100"); } else if (selectDetector.fetchFirstNRows > 100) { diff --git a/presto-ui/src/d3utils.test.ts b/presto-ui/src/d3utils.test.ts new file mode 100644 index 0000000000000..3ac3235e93335 --- /dev/null +++ b/presto-ui/src/d3utils.test.ts @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Mock dagre-d3-es and d3 before importing d3utils +jest.mock("dagre-d3-es", () => { + const mockGraph = { + setGraph: jest.fn().mockReturnThis(), + setDefaultEdgeLabel: jest.fn().mockReturnThis(), + defaultEdgeLabelFn: jest.fn().mockReturnValue({}), + graph: jest.fn().mockReturnValue({ rankdir: "BT" }), + isCompound: jest.fn().mockReturnValue(true), + }; + return { + graphlib: { + Graph: jest.fn().mockImplementation(() => mockGraph), + }, + render: jest.fn(), + }; +}); + +jest.mock("d3", () => ({ + select: jest.fn().mockReturnValue({ + append: jest.fn().mockReturnThis(), + attr: jest.fn().mockReturnThis(), + call: jest.fn().mockReturnThis(), + empty: jest.fn().mockReturnValue(false), + node: jest.fn().mockReturnValue(null), + }), +})); + +import { initializeGraph, initializeSvg } from "./d3utils"; +import * as _dagreD3 from "dagre-d3-es"; + +describe("d3utils", () => { + describe("initializeGraph", () => { + it("creates a graph instance", () => { + const graph = initializeGraph(); + expect(graph).toBeDefined(); + expect(graph.setGraph).toBeDefined(); + expect(graph.setDefaultEdgeLabel).toBeDefined(); + }); + + it("sets graph direction to bottom-to-top", () => { + const graph = initializeGraph(); + expect(graph.graph()).toEqual({ rankdir: "BT" }); + }); + + it("sets default edge label function", () => { + const graph = initializeGraph(); + // @ts-expect-error - defaultEdgeLabelFn is a private method in the type definition + const edgeLabel = graph.defaultEdgeLabelFn(); + expect(edgeLabel).toEqual({}); + }); + + it("creates a compound graph", () => { + const graph = initializeGraph(); + // Compound graphs support parent-child relationships + expect(graph.isCompound()).toBe(true); + }); + }); + + describe("initializeSvg", () => { + beforeEach(() => { + // Create a test SVG element in the document + document.body.innerHTML = ''; + }); + + afterEach(() => { + // Clean up + document.body.innerHTML = ""; + }); + + it("selects SVG element by selector", () => { + const svg = initializeSvg("#test-svg"); + expect(svg).toBeDefined(); + expect(svg.empty()).toBe(false); + }); + + it("appends a g element to the SVG", () => { + const svg = initializeSvg("#test-svg"); + // Verify append was called with 'g' + expect(svg.append).toHaveBeenCalledWith("g"); + }); + + it("returns the SVG selection", () => { + const svg = initializeSvg("#test-svg"); + // Verify the selection has the expected methods + expect(svg.node).toBeDefined(); + expect(typeof svg.node).toBe("function"); + }); + }); +}); + +// Made with Bob diff --git a/presto-ui/src/eslint.config.mjs b/presto-ui/src/eslint.config.mjs index d7f491e414234..2fb6827b13db1 100644 --- a/presto-ui/src/eslint.config.mjs +++ b/presto-ui/src/eslint.config.mjs @@ -10,7 +10,14 @@ export default [ js.configs.recommended, reactHooks.configs["recommended-latest"], { - ignores: ["**/vendor/**", "**/node_modules/**", "**/sql-parser/**", "webpack.config.js"], + ignores: [ + "**/vendor/**", + "**/node_modules/**", + "**/sql-parser/**", + "webpack.config.js", + "jest.config.js", + "coverage/**", + ], }, { languageOptions: { @@ -86,5 +93,19 @@ export default [ }, }, }, + // Test files + { + files: ["**/*.test.{js,jsx,ts,tsx}", "**/*.spec.{js,jsx,ts,tsx}", "**/setupTests.ts", "**/__tests__/**"], + languageOptions: { + globals: { + ...globals.jest, + }, + }, + rules: { + "@typescript-eslint/no-explicit-any": "off", + "react/display-name": "off", + "no-undef": "off", // Jest globals are defined + }, + }, prettierEslint, ]; diff --git a/presto-ui/src/jest.config.js b/presto-ui/src/jest.config.js new file mode 100644 index 0000000000000..f63e749635bce --- /dev/null +++ b/presto-ui/src/jest.config.js @@ -0,0 +1,52 @@ +module.exports = { + preset: "ts-jest", + testEnvironment: "jsdom", + roots: [""], + testMatch: ["**/__tests__/**/*.+(ts|tsx|js|jsx)", "**/?(*.)+(spec|test).+(ts|tsx|js|jsx)"], + transform: { + "^.+\\.(ts|tsx)$": "ts-jest", + "^.+\\.(js|jsx)$": "babel-jest", + }, + moduleNameMapper: { + "\\.(css|less|scss|sass)$": "identity-obj-proxy", + }, + transformIgnorePatterns: ["node_modules/(?!(dagre-d3-es|d3|d3-.*|internmap|delaunator|robust-predicates)/)"], + setupFilesAfterEnv: ["/setupTests.ts"], + collectCoverageFrom: [ + "**/*.{js,jsx,ts,tsx}", + "!**/*.d.ts", + "!sql-parser/**", + "!static/**", + "!templates/**", + "!webpack.config.js", + "!**/*.test.{js,jsx,ts,tsx}", + "!**/*.spec.{js,jsx,ts,tsx}", + "!jest.config.js", + "!setupTests.ts", + "!__tests__/**", + "!coverage/**", // Exclude coverage output directory from being scanned + ], + // Coverage thresholds disabled - focus on test pass rate + // Can be re-enabled later when more tests are added + // coverageThreshold: { + // global: { + // branches: 80, + // functions: 80, + // lines: 80, + // statements: 80, + // }, + // }, + coverageReporters: ["text", "lcov", "html"], + testPathIgnorePatterns: [ + "/node_modules/", + "/sql-parser/", + "/static/", + "/templates/", + "/__tests__/utils/", + "/__tests__/mocks/", + "/__tests__/fixtures/", + ], + moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json"], +}; + +// Made with Bob diff --git a/presto-ui/src/package.json b/presto-ui/src/package.json index 5793c3b12d167..e7af8db02290f 100644 --- a/presto-ui/src/package.json +++ b/presto-ui/src/package.json @@ -8,13 +8,20 @@ }, "devDependencies": { "@babel/core": "^7.24.5", - "@babel/preset-env": "^7.24.5", - "@babel/preset-react": "^7.24.1", + "@babel/preset-env": "^7.29.0", + "@babel/preset-react": "^7.28.5", + "@testing-library/dom": "^10.4.0", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.1.0", + "@testing-library/user-event": "^14.5.2", + "@types/jest": "^29.5.14", "@types/jquery": "^3.5.33", + "jquery": "3.7.1", "@types/react": "^19.2.2", "@types/react-dom": "^19.2.2", "@typescript-eslint/eslint-plugin": "^8.46.3", "@typescript-eslint/parser": "^8.46.3", + "babel-jest": "^30.2.0", "babel-loader": "9.1.3", "css-loader": "^7.1.1", "eslint": "^9.32.0", @@ -26,8 +33,12 @@ "hermes-eslint": "^0.31.0", "html-inline-script-webpack-plugin": "^3.2.1", "html-webpack-plugin": "^5.6.0", + "identity-obj-proxy": "^3.0.0", + "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", "prettier": "^3.6.2", "style-loader": "^4.0.0", + "ts-jest": "^29.2.5", "ts-loader": "^9.5.4", "typescript": "^5.9.3", "webpack": "^5.91.0", @@ -63,7 +74,11 @@ "lint:fix": "eslint . --ext .js,.jsx,.mjs,.cjs,.ts,.tsx --fix", "format": "prettier \"**/*.{js,jsx,mjs,cjs,ts,tsx}\"", "format:check": "prettier --check \"**/*.{js,jsx,mjs,cjs,ts,tsx}\"", - "format:fix": "prettier --write \"**/*.{js,jsx,mjs,cjs,ts,tsx}\"" + "format:fix": "prettier --write \"**/*.{js,jsx,mjs,cjs,ts,tsx}\"", + "test": "jest", + "test:watch": "jest --watch", + "test:coverage": "jest --coverage", + "test:ci": "jest --ci --coverage --maxWorkers=2" }, "resolutions": { "d3-color": "3.1.0" diff --git a/presto-ui/src/router/QueryList.integration.test.jsx b/presto-ui/src/router/QueryList.integration.test.jsx new file mode 100644 index 0000000000000..d925475195981 --- /dev/null +++ b/presto-ui/src/router/QueryList.integration.test.jsx @@ -0,0 +1,620 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from "react"; +import { + render, + screen, + waitFor, + fireEvent, + clickButtonSync, + advanceTimersAndWait, + setInputValue, +} from "../__tests__/utils/testUtils"; +import { setupIntegrationTest } from "../__tests__/utils/setupHelpers"; +import { QueryList } from "./QueryList"; +import { mockJQueryGet } from "../__tests__/mocks/jqueryMock"; +import { + createRunningQuery, + createFinishedQuery, + createFailedQuery, + createQueuedQuery, +} from "../__tests__/fixtures/queryFixtures"; + +/** + * Integration tests for QueryList component + * Tests user interactions including filter buttons, search, and dropdown menus + */ +describe("QueryList Integration Tests", () => { + setupIntegrationTest(); + + describe("Filter Button Interactions", () => { + it("clicking FINISHED filter button shows finished queries", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const finishedQuery = createFinishedQuery({ queryId: "finished_1" }); + mockJQueryGet({ + "/v1/query": [runningQuery, finishedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.queryByText("finished_1")).not.toBeInTheDocument(); + }); + + clickButtonSync(/Finished/i); + + await waitFor(() => { + expect(screen.getByText("finished_1")).toBeInTheDocument(); + }); + }); + + it("toggling RUNNING filter removes running queries", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const queuedQuery = createQueuedQuery({ queryId: "queued_1" }); + mockJQueryGet({ + "/v1/query": [runningQuery, queuedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.getByText("queued_1")).toBeInTheDocument(); + }); + + clickButtonSync(/Running/i); + + await waitFor(() => { + expect(screen.queryByText("running_1")).not.toBeInTheDocument(); + expect(screen.getByText("queued_1")).toBeInTheDocument(); + }); + }); + + it("toggling QUEUED filter removes queued queries", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const queuedQuery = createQueuedQuery({ queryId: "queued_1" }); + mockJQueryGet({ + "/v1/query": [runningQuery, queuedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.getByText("queued_1")).toBeInTheDocument(); + }); + + clickButtonSync(/Queued/i); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.queryByText("queued_1")).not.toBeInTheDocument(); + }); + }); + + it("clicking multiple filters shows combined results", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const finishedQuery = createFinishedQuery({ queryId: "finished_1" }); + const queuedQuery = createQueuedQuery({ queryId: "queued_1" }); + mockJQueryGet({ + "/v1/query": [runningQuery, finishedQuery, queuedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.getByText("queued_1")).toBeInTheDocument(); + expect(screen.queryByText("finished_1")).not.toBeInTheDocument(); + }); + + clickButtonSync(/Finished/i); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + expect(screen.getByText("queued_1")).toBeInTheDocument(); + expect(screen.getByText("finished_1")).toBeInTheDocument(); + }); + }); + }); + + describe("Search Functionality", () => { + it("typing in search box filters queries by query ID", async () => { + const query1 = createRunningQuery({ queryId: "query_abc_123" }); + const query2 = createRunningQuery({ queryId: "query_xyz_456" }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("query_abc_123")).toBeInTheDocument(); + expect(screen.getByText("query_xyz_456")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "abc"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("query_abc_123")).toBeInTheDocument(); + expect(screen.queryByText("query_xyz_456")).not.toBeInTheDocument(); + }); + }); + + it("search filters by user name", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + session: { user: "alice", source: "cli" }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + session: { user: "bob", source: "cli" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "alice"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.queryByText("q2")).not.toBeInTheDocument(); + }); + }); + + it("search filters by source", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + session: { user: "test", source: "jdbc-driver" }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + session: { user: "test", source: "presto-cli" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "jdbc"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.queryByText("q2")).not.toBeInTheDocument(); + }); + }); + + it("search filters by resource group", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + resourceGroupId: ["global", "adhoc"], + }); + const query2 = createRunningQuery({ + queryId: "q2", + resourceGroupId: ["global", "pipeline"], + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "adhoc"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.queryByText("q2")).not.toBeInTheDocument(); + }); + }); + + it("search filters by query text", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + query: "SELECT * FROM users WHERE name LIKE '%test%'", + }); + const query2 = createRunningQuery({ + queryId: "q2", + query: "SELECT * FROM orders WHERE status = 'pending'", + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "users"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.queryByText("q2")).not.toBeInTheDocument(); + }); + }); + + it("clearing search shows all queries again", async () => { + const query1 = createRunningQuery({ queryId: "q1" }); + const query2 = createRunningQuery({ queryId: "q2" }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "q1"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.queryByText("q2")).not.toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, ""); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q2")).toBeInTheDocument(); + }); + }); + + it("shows 'No queries matched filters' when search has no results", async () => { + const query1 = createRunningQuery({ queryId: "q1" }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "nonexistent_query"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("No queries matched filters")).toBeInTheDocument(); + }); + }); + }); + + describe("Combined Filters and Search", () => { + it("applies both state filters and search together", async () => { + const runningQuery1 = createRunningQuery({ + queryId: "running_alice", + session: { user: "alice", source: "cli" }, + }); + const runningQuery2 = createRunningQuery({ + queryId: "running_bob", + session: { user: "bob", source: "cli" }, + }); + const finishedQuery = createFinishedQuery({ + queryId: "finished_alice", + session: { user: "alice", source: "cli" }, + }); + mockJQueryGet({ + "/v1/query": [runningQuery1, runningQuery2, finishedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_alice")).toBeInTheDocument(); + expect(screen.getByText("running_bob")).toBeInTheDocument(); + expect(screen.queryByText("finished_alice")).not.toBeInTheDocument(); + }); + + setInputValue(/User, source, query ID/i, "alice"); + await advanceTimersAndWait(300); + + await waitFor(() => { + expect(screen.getByText("running_alice")).toBeInTheDocument(); + expect(screen.queryByText("running_bob")).not.toBeInTheDocument(); + expect(screen.queryByText("finished_alice")).not.toBeInTheDocument(); + }); + + clickButtonSync(/Finished/i); + + await waitFor(() => { + expect(screen.getByText("running_alice")).toBeInTheDocument(); + expect(screen.getByText("finished_alice")).toBeInTheDocument(); + expect(screen.queryByText("running_bob")).not.toBeInTheDocument(); + }); + }); + }); + + describe("Dropdown Menu Interactions", () => { + describe("Sort Dropdown", () => { + it("sorts queries by execution time when selecting from dropdown", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + queryStats: { executionTime: "10.5s" }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + queryStats: { executionTime: "5.2s" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + + const sortDropdown = screen.getByText(/Sort/); + fireEvent.click(sortDropdown); + + const executionOption = screen.getByText("Execution Time"); + fireEvent.click(executionOption); + + await waitFor(() => { + const queryElements = screen.getAllByText(/^q[12]$/); + expect(queryElements.length).toBeGreaterThan(0); + }); + }); + + it("sorts queries by CPU time when selecting from dropdown", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + queryStats: { totalCpuTime: "5.0s" }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + queryStats: { totalCpuTime: "15.0s" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + + const sortDropdown = screen.getByText(/Sort/); + fireEvent.click(sortDropdown); + + const cpuOption = screen.getByText("CPU Time"); + fireEvent.click(cpuOption); + + await waitFor(() => { + const queryElements = screen.getAllByText(/^q[12]$/); + expect(queryElements.length).toBeGreaterThan(0); + }); + }); + + it("sorts queries by cumulative memory when selecting from dropdown", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + queryStats: { cumulativeUserMemory: 1000000 }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + queryStats: { cumulativeUserMemory: 5000000 }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + + const sortDropdown = screen.getByText(/Sort/); + fireEvent.click(sortDropdown); + + const memoryOption = screen.getByText("Cumulative User Memory"); + fireEvent.click(memoryOption); + + await waitFor(() => { + const queryElements = screen.getAllByText(/^q[12]$/); + expect(queryElements.length).toBeGreaterThan(0); + }); + }); + + it("sorts queries by current memory when selecting from dropdown", async () => { + const query1 = createRunningQuery({ + queryId: "q1", + queryStats: { userMemoryReservation: "100MB" }, + }); + const query2 = createRunningQuery({ + queryId: "q2", + queryStats: { userMemoryReservation: "500MB" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + + const sortDropdown = screen.getByText(/Sort/); + fireEvent.click(sortDropdown); + + const currentMemoryOption = screen.getByText("Current Memory"); + fireEvent.click(currentMemoryOption); + + await waitFor(() => { + const queryElements = screen.getAllByText(/^q[12]$/); + expect(queryElements.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Error Type Filter Dropdown", () => { + it("filters by user error when selecting from dropdown", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const userErrorQuery = createFailedQuery({ + queryId: "user_error_1", + state: "FAILED", + errorType: "USER_ERROR", + }); + const internalErrorQuery = createFailedQuery({ + queryId: "internal_error_1", + state: "FAILED", + errorType: "INTERNAL_ERROR", + }); + mockJQueryGet({ + "/v1/query": [runningQuery, userErrorQuery, internalErrorQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + }); + + const failedDropdown = screen.getByText(/Failed/); + fireEvent.click(failedDropdown); + + const userErrorOption = screen.getByText("User Error"); + fireEvent.click(userErrorOption); + + await waitFor(() => { + expect(screen.getByText("user_error_1")).toBeInTheDocument(); + }); + }); + + it("filters by internal error when selecting from dropdown", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const userErrorQuery = createFailedQuery({ + queryId: "user_error_1", + state: "FAILED", + errorType: "USER_ERROR", + }); + const internalErrorQuery = createFailedQuery({ + queryId: "internal_error_1", + state: "FAILED", + errorType: "INTERNAL_ERROR", + }); + mockJQueryGet({ + "/v1/query": [runningQuery, userErrorQuery, internalErrorQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + }); + + const failedDropdown = screen.getByText(/Failed/); + fireEvent.click(failedDropdown); + + const internalErrorOption = screen.getByText("Internal Error"); + fireEvent.click(internalErrorOption); + + await waitFor(() => { + expect(screen.queryByText("internal_error_1")).not.toBeInTheDocument(); + }); + }); + }); + + describe("Max Queries Dropdown", () => { + it("changes max displayed queries when selecting from dropdown", async () => { + // Create 25 queries + const queries = Array.from({ length: 25 }, (_, i) => createRunningQuery({ queryId: `q${i + 1}` })); + mockJQueryGet({ + "/v1/query": queries, + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + expect(screen.getByText("q20")).toBeInTheDocument(); + }); + + expect(screen.getByText("q25")).toBeInTheDocument(); + + const maxQueriesButtons = screen.getAllByRole("button"); + const maxQueriesButton = maxQueriesButtons.find( + (btn) => btn.textContent.includes("20 queries") || btn.textContent.includes("queries") + ); + + if (maxQueriesButton) { + fireEvent.click(maxQueriesButton); + + const max100Option = screen.getByText("100 queries"); + fireEvent.click(max100Option); + + await waitFor(() => { + expect(screen.getByText("q25")).toBeInTheDocument(); + }); + } + }); + }); + }); +}); diff --git a/presto-ui/src/router/QueryList.jsx b/presto-ui/src/router/QueryList.jsx index d5ef070e863a9..4819fa55da0f1 100755 --- a/presto-ui/src/router/QueryList.jsx +++ b/presto-ui/src/router/QueryList.jsx @@ -339,7 +339,17 @@ export const QueryList = () => { const term = searchString.toLowerCase(); if ( query.queryId.toLowerCase().indexOf(term) !== -1 || - getHumanReadableState(query).toLowerCase().indexOf(term) !== -1 || + getHumanReadableState( + query.state, + query.scheduled, + query.fullyBlocked, + query.blockedReasons, + query.memoryPool, + query.errorType, + query.errorCode?.name + ) + .toLowerCase() + .indexOf(term) !== -1 || query.query.toLowerCase().indexOf(term) !== -1 ) { return true; diff --git a/presto-ui/src/router/QueryList.test.jsx b/presto-ui/src/router/QueryList.test.jsx new file mode 100644 index 0000000000000..7cb850b5e36a2 --- /dev/null +++ b/presto-ui/src/router/QueryList.test.jsx @@ -0,0 +1,780 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from "react"; +import { render, screen, waitFor, advanceTimersAndWait } from "../__tests__/utils/testUtils"; +import { QueryList } from "./QueryList"; +import { mockJQueryGet } from "../__tests__/mocks/jqueryMock"; +import { + baseMockQuery, + createMockQuery, + createRunningQuery, + createFinishedQuery, + createFailedQuery, + createQueuedQuery, +} from "../__tests__/fixtures/queryFixtures"; + +describe("QueryList", () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + describe("QueryList Component", () => { + it("renders empty state initially", () => { + mockJQueryGet({ "/v1/query": [] }); + const { container } = render(); + expect(container).toBeInTheDocument(); + }); + + it("fetches and displays queries", async () => { + mockJQueryGet({ + "/v1/query": [baseMockQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("test_query_123")).toBeInTheDocument(); + }); + }); + + it("displays multiple queries", async () => { + const query2 = createMockQuery({ queryId: "test_query_456" }); + mockJQueryGet({ + "/v1/query": [baseMockQuery, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("test_query_123")).toBeInTheDocument(); + expect(screen.getByText("test_query_456")).toBeInTheDocument(); + }); + }); + + it("calls jQuery.get on mount", async () => { + mockJQueryGet({ "/v1/query": [] }); + // Get reference to the mocked $.get function after mockJQueryGet sets it up + const getSpy = global.$.get; + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(getSpy).toHaveBeenCalledWith("/v1/query", expect.any(Function)); + }); + }); + + it("renders filter controls", async () => { + mockJQueryGet({ "/v1/query": [baseMockQuery] }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText(/Running/)).toBeInTheDocument(); + }); + }); + + it("renders sort controls", async () => { + mockJQueryGet({ "/v1/query": [baseMockQuery] }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + const sortButtons = screen.getAllByRole("button"); + expect(sortButtons.length).toBeGreaterThan(0); + }); + }); + + it("handles empty query list", async () => { + mockJQueryGet({ "/v1/query": [] }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.queryByText("test_query_123")).not.toBeInTheDocument(); + }); + }); + + it("displays query statistics", async () => { + mockJQueryGet({ "/v1/query": [baseMockQuery] }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText(/testuser/)).toBeInTheDocument(); + expect(screen.getByText(/5.2s/)).toBeInTheDocument(); + }); + }); + }); + + describe("Query Filtering", () => { + it("filters running queries", async () => { + const runningQuery = createRunningQuery(); + const finishedQuery = createFinishedQuery({ queryId: "finished_query" }); + + mockJQueryGet({ + "/v1/query": [runningQuery, finishedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("test_query_123")).toBeInTheDocument(); + }); + }); + + it("filters queued queries", async () => { + const queuedQuery = createQueuedQuery(); + + mockJQueryGet({ + "/v1/query": [queuedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("test_query_123")).toBeInTheDocument(); + }); + }); + }); + + describe("Query Sorting", () => { + it("sorts queries by creation time", async () => { + const query1 = createMockQuery({ + queryId: "query1", + queryStats: { createTime: "2024-01-01T10:00:00.000Z" }, + }); + const query2 = createMockQuery({ + queryId: "query2", + queryStats: { createTime: "2024-01-01T11:00:00.000Z" }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("query1")).toBeInTheDocument(); + expect(screen.getByText("query2")).toBeInTheDocument(); + }); + }); + describe("Query Search and Filtering", () => { + it("displays queries with different query IDs", async () => { + const query1 = createMockQuery({ queryId: "search_test_123" }); + const query2 = createMockQuery({ queryId: "other_query_456" }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("search_test_123")).toBeInTheDocument(); + expect(screen.getByText("other_query_456")).toBeInTheDocument(); + }); + }); + + it("displays queries with different users", async () => { + const query1 = createMockQuery({ + queryId: "query1", + session: { user: "alice", source: "presto-ui" }, + }); + const query2 = createMockQuery({ + queryId: "query2", + session: { user: "bob", source: "presto-ui" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("query1")).toBeInTheDocument(); + expect(screen.getByText("query2")).toBeInTheDocument(); + }); + }); + + it("displays queries with different sources", async () => { + const query1 = createMockQuery({ + queryId: "jdbc_query", + session: { user: "testuser", source: "jdbc" }, + }); + const query2 = createMockQuery({ + queryId: "cli_query", + session: { user: "testuser", source: "cli" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("jdbc_query")).toBeInTheDocument(); + expect(screen.getByText("cli_query")).toBeInTheDocument(); + }); + }); + + it("displays queries with different query text", async () => { + const query1 = createMockQuery({ + queryId: "users_query", + query: "SELECT * FROM users", + }); + const query2 = createMockQuery({ + queryId: "orders_query", + query: "SELECT * FROM orders", + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("users_query")).toBeInTheDocument(); + expect(screen.getByText("orders_query")).toBeInTheDocument(); + }); + }); + }); + + describe("Error Type Filtering", () => { + it("displays failed queries with internal errors (default filter)", async () => { + const failedQuery = createFailedQuery({ + queryId: "internal_error_query", + errorType: "INTERNAL_ERROR", + }); + mockJQueryGet({ + "/v1/query": [failedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("internal_error_query")).toBeInTheDocument(); + }); + }); + + it("displays failed queries with external errors (default filter)", async () => { + const failedQuery = createFailedQuery({ + queryId: "external_error_query", + errorType: "EXTERNAL", + }); + mockJQueryGet({ + "/v1/query": [failedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("external_error_query")).toBeInTheDocument(); + }); + }); + + it("handles queries with different error types", async () => { + const internalError = createFailedQuery({ + queryId: "internal_err", + errorType: "INTERNAL_ERROR", + }); + const externalError = createFailedQuery({ + queryId: "external_err", + errorType: "EXTERNAL", + }); + mockJQueryGet({ + "/v1/query": [internalError, externalError], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("internal_err")).toBeInTheDocument(); + expect(screen.getByText("external_err")).toBeInTheDocument(); + }); + }); + }); + + describe("Query Sorting by Different Metrics", () => { + it("sorts queries by elapsed time", async () => { + const query1 = createMockQuery({ + queryId: "fast_query", + queryStats: { elapsedTime: "1.0s" }, + }); + const query2 = createMockQuery({ + queryId: "slow_query", + queryStats: { elapsedTime: "10.0s" }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("fast_query")).toBeInTheDocument(); + expect(screen.getByText("slow_query")).toBeInTheDocument(); + }); + }); + + it("sorts queries by execution time", async () => { + const query1 = createMockQuery({ + queryId: "quick_exec", + queryStats: { executionTime: "0.5s" }, + }); + const query2 = createMockQuery({ + queryId: "long_exec", + queryStats: { executionTime: "5.0s" }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("quick_exec")).toBeInTheDocument(); + expect(screen.getByText("long_exec")).toBeInTheDocument(); + }); + }); + + it("sorts queries by CPU time", async () => { + const query1 = createMockQuery({ + queryId: "low_cpu", + queryStats: { totalCpuTime: "1.0s" }, + }); + const query2 = createMockQuery({ + queryId: "high_cpu", + queryStats: { totalCpuTime: "20.0s" }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("low_cpu")).toBeInTheDocument(); + expect(screen.getByText("high_cpu")).toBeInTheDocument(); + }); + }); + + it("sorts queries by memory usage", async () => { + const query1 = createMockQuery({ + queryId: "low_mem", + queryStats: { userMemoryReservation: "100MB" }, + }); + const query2 = createMockQuery({ + queryId: "high_mem", + queryStats: { userMemoryReservation: "5GB" }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("low_mem")).toBeInTheDocument(); + expect(screen.getByText("high_mem")).toBeInTheDocument(); + }); + }); + + describe("Advanced Sorting and Filtering", () => { + it("handles empty search string correctly", async () => { + const query1 = createMockQuery({ queryId: "query1" }); + const query2 = createMockQuery({ queryId: "query2" }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("query1")).toBeInTheDocument(); + expect(screen.getByText("query2")).toBeInTheDocument(); + }); + }); + + it("filters queries by INSUFFICIENT_RESOURCES error type", async () => { + const resourceError = createFailedQuery({ + queryId: "resource_err", + errorType: "INSUFFICIENT_RESOURCES", + }); + mockJQueryGet({ + "/v1/query": [resourceError], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("resource_err")).toBeInTheDocument(); + }); + }); + + it("sorts queries by cumulative memory", async () => { + const query1 = createMockQuery({ + queryId: "low_cumulative", + queryStats: { cumulativeUserMemory: 1000000 }, + }); + const query2 = createMockQuery({ + queryId: "high_cumulative", + queryStats: { cumulativeUserMemory: 5000000 }, + }); + + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("low_cumulative")).toBeInTheDocument(); + expect(screen.getByText("high_cumulative")).toBeInTheDocument(); + }); + }); + + it("searches queries by resourceGroupId", async () => { + const query1 = createMockQuery({ + queryId: "adhoc_query", + resourceGroupId: ["global", "adhoc"], + }); + const query2 = createMockQuery({ + queryId: "pipeline_query", + resourceGroupId: ["global", "pipeline"], + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("adhoc_query")).toBeInTheDocument(); + expect(screen.getByText("pipeline_query")).toBeInTheDocument(); + }); + }); + }); + }); + }); +}); + +describe("Interactive Filter and Sort Tests", () => { + it("handles filter button interactions", async () => { + const runningQuery = createRunningQuery({ queryId: "running_1" }); + const finishedQuery = createFinishedQuery({ queryId: "finished_1" }); + mockJQueryGet({ + "/v1/query": [runningQuery, finishedQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_1")).toBeInTheDocument(); + // Both queries should be visible initially (running and finished filters active by default) + }); + }); + + it("displays 'No queries' when query list is empty", async () => { + mockJQueryGet({ + "/v1/query": [], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("No queries")).toBeInTheDocument(); + }); + }); + + it("renders search input field", async () => { + mockJQueryGet({ + "/v1/query": [createMockQuery({ queryId: "test_q" })], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + const searchInput = screen.getByPlaceholderText(/User, source, query ID/i); + expect(searchInput).toBeInTheDocument(); + }); + }); + + describe("Coverage for Uncovered Code Paths", () => { + it("handles queries with FINISHED state (not shown by default)", async () => { + const finishedQuery = createFinishedQuery({ queryId: "finished_1" }); + const runningQuery = createRunningQuery({ queryId: "running_1" }); + mockJQueryGet({ + "/v1/query": [finishedQuery, runningQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + // FINISHED queries are filtered out by default, only RUNNING shown + expect(screen.getByText("running_1")).toBeInTheDocument(); + }); + }); + + it("handles queries with different elapsed times for sorting", async () => { + const query1 = createMockQuery({ + queryId: "fast", + queryStats: { elapsedTime: "100ms" }, + }); + const query2 = createMockQuery({ + queryId: "slow", + queryStats: { elapsedTime: "5.5s" }, + }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("fast")).toBeInTheDocument(); + expect(screen.getByText("slow")).toBeInTheDocument(); + }); + }); + + it("handles queries with different execution times", async () => { + const query1 = createMockQuery({ + queryId: "q1", + queryStats: { executionTime: "2.5s" }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with different CPU times", async () => { + const query1 = createMockQuery({ + queryId: "q1", + queryStats: { totalCpuTime: "15.2s" }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with cumulative memory values", async () => { + const query1 = createMockQuery({ + queryId: "q1", + queryStats: { cumulativeUserMemory: 5000000 }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with current memory reservation", async () => { + const query1 = createMockQuery({ + queryId: "q1", + queryStats: { userMemoryReservation: "500MB" }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with USER_ERROR error type (not shown by default)", async () => { + const userErrorQuery = createFailedQuery({ + queryId: "user_err", + errorType: "USER_ERROR", + }); + const internalErrorQuery = createFailedQuery({ + queryId: "internal_err", + errorType: "INTERNAL_ERROR", + }); + mockJQueryGet({ + "/v1/query": [userErrorQuery, internalErrorQuery], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + // USER_ERROR is filtered out by default, only INTERNAL_ERROR shown + expect(screen.getByText("internal_err")).toBeInTheDocument(); + }); + }); + + it("handles queries with INTERNAL_ERROR error type", async () => { + const query1 = createFailedQuery({ + queryId: "internal_err", + errorType: "INTERNAL_ERROR", + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("internal_err")).toBeInTheDocument(); + }); + }); + + it("handles queries with user in session", async () => { + const query1 = createMockQuery({ + queryId: "q1", + session: { user: "testuser", source: "cli" }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with source in session", async () => { + const query1 = createMockQuery({ + queryId: "q1", + session: { user: "test", source: "jdbc-driver" }, + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("handles queries with resourceGroupId array", async () => { + const query1 = createMockQuery({ + queryId: "q1", + resourceGroupId: ["global", "adhoc", "user_queries"], + }); + mockJQueryGet({ + "/v1/query": [query1], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("q1")).toBeInTheDocument(); + }); + }); + + it("renders with multiple query types", async () => { + const query1 = createRunningQuery({ queryId: "running_q" }); + const query2 = createQueuedQuery({ queryId: "queued_q" }); + mockJQueryGet({ + "/v1/query": [query1, query2], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByText("running_q")).toBeInTheDocument(); + expect(screen.getByText("queued_q")).toBeInTheDocument(); + }); + }); + }); + it("renders filter buttons", async () => { + mockJQueryGet({ + "/v1/query": [createMockQuery({ queryId: "test_q" })], + }); + + render(); + await advanceTimersAndWait(100); + + await waitFor(() => { + expect(screen.getByRole("button", { name: /Running/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Queued/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Finished/i })).toBeInTheDocument(); + }); + }); +}); +// Made with Bob diff --git a/presto-ui/src/router/QueryListItem.test.jsx b/presto-ui/src/router/QueryListItem.test.jsx new file mode 100644 index 0000000000000..5516081d7903e --- /dev/null +++ b/presto-ui/src/router/QueryListItem.test.jsx @@ -0,0 +1,173 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from "react"; +import { render, screen } from "../__tests__/utils/testUtils"; +import { QueryListItem } from "./QueryList"; +import { + baseMockQuery, + createFinishedQuery, + createFailedQuery, + createQueuedQuery, +} from "../__tests__/fixtures/queryFixtures"; + +describe("QueryListItem", () => { + it("renders query ID", () => { + render(); + expect(screen.getByText("test_query_123")).toBeInTheDocument(); + }); + + it("renders query state", () => { + const { container } = render(); + const progressBar = container.querySelector(".progress-bar"); + expect(progressBar).toBeInTheDocument(); + }); + + it("renders user information", () => { + render(); + expect(screen.getByText("testuser")).toBeInTheDocument(); + }); + + it("renders query text snippet", () => { + render(); + expect(screen.getByText(/SELECT \* FROM table/)).toBeInTheDocument(); + }); + + it("renders completed drivers count", () => { + render(); + expect(screen.getByText("10")).toBeInTheDocument(); + }); + + it("renders running drivers count", () => { + render(); + expect(screen.getByText("5")).toBeInTheDocument(); + }); + + it("renders queued drivers count", () => { + render(); + expect(screen.getByText("0")).toBeInTheDocument(); + }); + + it("renders elapsed time", () => { + render(); + expect(screen.getByText(/5.2s/)).toBeInTheDocument(); + }); + + it("renders CPU time", () => { + render(); + expect(screen.getByText(/10.5s/)).toBeInTheDocument(); + }); + + it("renders memory usage", () => { + render(); + expect(screen.getByText(/2GB/)).toBeInTheDocument(); + }); + + it("renders progress bar", () => { + const { container } = render(); + const progressBar = container.querySelector(".progress-bar"); + expect(progressBar).toBeInTheDocument(); + }); + + it("renders link to query detail page", () => { + const queryWithCoordinator = { ...baseMockQuery, coordinatorUri: "" }; + render(); + const link = screen.getByRole("link", { name: /test_query_123/ }); + expect(link).toHaveAttribute("href", expect.stringContaining("test_query_123")); + }); + + it("handles FINISHED state", () => { + const finishedQuery = createFinishedQuery(); + const { container } = render(); + const progressBar = container.querySelector(".progress-bar"); + expect(progressBar).toBeInTheDocument(); + // Progress bar shows 100% for finished queries + expect(progressBar).toHaveStyle({ width: "100%" }); + }); + + it("handles FAILED state", () => { + const failedQuery = createFailedQuery(); + const { container } = render(); + const progressBar = container.querySelector(".progress-bar"); + expect(progressBar).toBeInTheDocument(); + }); + + it("handles QUEUED state", () => { + const queuedQuery = createQueuedQuery(); + const { container } = render(); + const progressBar = container.querySelector(".progress-bar"); + expect(progressBar).toBeInTheDocument(); + }); + + describe("Query Text Formatting", () => { + it("handles query with no leading whitespace", () => { + const query = { + ...baseMockQuery, + query: "SELECT * FROM table", + }; + render(); + expect(screen.getByText(/SELECT \* FROM table/)).toBeInTheDocument(); + }); + + it("handles query with empty lines", () => { + const query = { + ...baseMockQuery, + query: "SELECT *\n\nFROM table\n\nWHERE id = 1", + }; + render(); + expect(screen.getByText(/SELECT/)).toBeInTheDocument(); + }); + + it("handles query with consistent indentation", () => { + const query = { + ...baseMockQuery, + query: " SELECT *\n FROM table\n WHERE id = 1", + }; + render(); + expect(screen.getByText(/SELECT/)).toBeInTheDocument(); + }); + + it("handles query with mixed indentation", () => { + const query = { + ...baseMockQuery, + query: " SELECT *\n FROM table\nWHERE id = 1", + }; + render(); + expect(screen.getByText(/SELECT/)).toBeInTheDocument(); + }); + + it("handles single line query", () => { + const query = { + ...baseMockQuery, + query: "SELECT * FROM table WHERE id = 1", + }; + render(); + expect(screen.getByText(/SELECT \* FROM table WHERE id = 1/)).toBeInTheDocument(); + }); + + it("handles very long query text (truncation)", () => { + const longQuery = "SELECT * FROM table WHERE " + "column = 'value' AND ".repeat(50); + const query = { + ...baseMockQuery, + query: longQuery, + }; + render(); + // Query should be truncated to 300 characters + const { container } = render(); + expect(container).toBeInTheDocument(); + }); + }); +}); + +// Made with Bob diff --git a/presto-ui/src/setupTests.ts b/presto-ui/src/setupTests.ts new file mode 100644 index 0000000000000..265ecfbfe1e87 --- /dev/null +++ b/presto-ui/src/setupTests.ts @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Global test setup file + * This file is executed before each test file and sets up the testing environment + */ + +import "@testing-library/jest-dom"; +import { setupJQuery } from "./__tests__/mocks/jqueryMock"; +import { setupAllBrowserMocks } from "./__tests__/mocks/browserMocks"; +import { setupFetchMock } from "./__tests__/mocks/apiMocks"; + +// Setup all browser API mocks (matchMedia, hljs, clipboard, document) +setupAllBrowserMocks(); + +// Setup fetch mock (centralized in apiMocks.ts for all HTTP/API mocking) +setupFetchMock(); + +// Setup jQuery with selective mocking (uses real jQuery with mocked plugins) +setupJQuery(); + +// Suppress specific harmless Jest fake timer warnings +// These occur when test helpers call jest.advanceTimersByTime() but fake timers +// ARE properly set up in beforeEach. This is a known Jest limitation with async helpers. +// +// IMPORTANT: This override is scoped to Jest environment only and uses exact string matching +// to avoid interfering with tests that spy on or assert console.warn. +const originalWarn = console.warn; +const JEST_FAKE_TIMER_WARNING = + "A function to advance timers was called but the timers APIs are not replaced with fake timers. Call `jest.useFakeTimers()` in this test file or enable fake timers for all tests by setting 'fakeTimers': {'enableGlobally': true} in Jest configuration file."; + +// Only apply the filter in Jest environment (when JEST_WORKER_ID is set) +if (process.env.JEST_WORKER_ID !== undefined) { + console.warn = (...args: any[]) => { + const message = args[0]?.toString() || ""; + + // Only suppress the Jest fake timer warning message (by checking if it starts with the warning) + // Jest includes stack trace after the message, so we check the prefix + // This is safe because: + // 1. We match a specific, complete warning message prefix (254 chars) + // 2. We only apply this in Jest environment (JEST_WORKER_ID check) + // 3. Fake timers ARE properly configured in our tests + // 4. This warning is a false positive from Jest's async helper interaction + // 5. Tests that spy on console.warn can still work (they see the wrapper) + if (message.startsWith(JEST_FAKE_TIMER_WARNING)) { + return; + } + + // Allow all other warnings to surface + originalWarn.apply(console, args); + }; +} + +// Made with Bob diff --git a/presto-ui/src/static/dev/index.html b/presto-ui/src/static/dev/index.html index 4bb57b880bb77..e2c4c3ca0791f 100644 --- a/presto-ui/src/static/dev/index.html +++ b/presto-ui/src/static/dev/index.html @@ -30,8 +30,8 @@ - - + + diff --git a/presto-ui/src/static/embedded_plan.html b/presto-ui/src/static/embedded_plan.html index a0c54f91cf10e..7ef37cc500e25 100644 --- a/presto-ui/src/static/embedded_plan.html +++ b/presto-ui/src/static/embedded_plan.html @@ -31,8 +31,8 @@ - - + + diff --git a/presto-ui/src/static/plan.html b/presto-ui/src/static/plan.html index 43a3d31923762..d9e92b1adf36d 100644 --- a/presto-ui/src/static/plan.html +++ b/presto-ui/src/static/plan.html @@ -31,8 +31,8 @@ - - + + diff --git a/presto-ui/src/static/query.html b/presto-ui/src/static/query.html index f6fbf26873f46..3333ea347a3b8 100644 --- a/presto-ui/src/static/query.html +++ b/presto-ui/src/static/query.html @@ -31,8 +31,8 @@ - - + + diff --git a/presto-ui/src/static/stage.html b/presto-ui/src/static/stage.html index 7e98efe6c7f26..98736e87c5186 100644 --- a/presto-ui/src/static/stage.html +++ b/presto-ui/src/static/stage.html @@ -31,8 +31,8 @@ - - + + diff --git a/presto-ui/src/static/vendor/highlightjs/10.1.2/highlight.min.js b/presto-ui/src/static/vendor/highlightjs/10.1.2/highlight.min.js new file mode 100644 index 0000000000000..1602651bdef59 --- /dev/null +++ b/presto-ui/src/static/vendor/highlightjs/10.1.2/highlight.min.js @@ -0,0 +1,44 @@ +/* + Highlight.js 10.1.2 (edd73d24) + License: BSD-3-Clause + Copyright (c) 2006-2020, Ivan Sagalaev +*/ +var hljs=function(){"use strict";function e(n){Object.freeze(n);var t="function"==typeof n;return Object.getOwnPropertyNames(n).forEach((function(r){!Object.hasOwnProperty.call(n,r)||null===n[r]||"object"!=typeof n[r]&&"function"!=typeof n[r]||t&&("caller"===r||"callee"===r||"arguments"===r)||Object.isFrozen(n[r])||e(n[r])})),n}class n{constructor(e){void 0===e.data&&(e.data={}),this.data=e.data}ignoreMatch(){this.ignore=!0}}function t(e){return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}function r(e,...n){var t={};for(const n in e)t[n]=e[n];return n.forEach((function(e){for(const n in e)t[n]=e[n]})),t}function a(e){return e.nodeName.toLowerCase()}var i=Object.freeze({__proto__:null,escapeHTML:t,inherit:r,nodeStream:function(e){var n=[];return function e(t,r){for(var i=t.firstChild;i;i=i.nextSibling)3===i.nodeType?r+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:r,node:i}),r=e(i,r),a(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:r,node:i}));return r}(e,0),n},mergeStreams:function(e,n,r){var i=0,s="",o=[];function l(){return e.length&&n.length?e[0].offset!==n[0].offset?e[0].offset"}function u(e){s+=""}function d(e){("start"===e.event?c:u)(e.node)}for(;e.length||n.length;){var g=l();if(s+=t(r.substring(i,g[0].offset)),i=g[0].offset,g===e){o.reverse().forEach(u);do{d(g.splice(0,1)[0]),g=l()}while(g===e&&g.length&&g[0].offset===i);o.reverse().forEach(c)}else"start"===g[0].event?o.push(g[0].node):o.pop(),d(g.splice(0,1)[0])}return s+t(r.substr(i))}});const s="",o=e=>!!e.kind;class l{constructor(e,n){this.buffer="",this.classPrefix=n.classPrefix,e.walk(this)}addText(e){this.buffer+=t(e)}openNode(e){if(!o(e))return;let n=e.kind;e.sublanguage||(n=`${this.classPrefix}${n}`),this.span(n)}closeNode(e){o(e)&&(this.buffer+=s)}value(){return this.buffer}span(e){this.buffer+=``}}class c{constructor(){this.rootNode={children:[]},this.stack=[this.rootNode]}get top(){return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){this.top.children.push(e)}openNode(e){const n={kind:e,children:[]};this.add(n),this.stack.push(n)}closeNode(){if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),n.children.forEach(n=>this._walk(e,n)),e.closeNode(n)),e}static _collapse(e){"string"!=typeof e&&e.children&&(e.children.every(e=>"string"==typeof e)?e.children=[e.children.join("")]:e.children.forEach(e=>{c._collapse(e)}))}}class u extends c{constructor(e){super(),this.options=e}addKeyword(e,n){""!==e&&(this.openNode(n),this.addText(e),this.closeNode())}addText(e){""!==e&&this.add(e)}addSublanguage(e,n){const t=e.root;t.kind=n,t.sublanguage=!0,this.add(t)}toHTML(){return new l(this,this.options).value()}finalize(){return!0}}function d(e){return e?"string"==typeof e?e:e.source:null}const g="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",h={begin:"\\\\[\\s\\S]",relevance:0},f={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[h]},p={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[h]},b={begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},m=function(e,n,t={}){var a=r({className:"comment",begin:e,end:n,contains:[]},t);return a.contains.push(b),a.contains.push({className:"doctag",begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),a},v=m("//","$"),x=m("/\\*","\\*/"),E=m("#","$");var _=Object.freeze({__proto__:null,IDENT_RE:"[a-zA-Z]\\w*",UNDERSCORE_IDENT_RE:"[a-zA-Z_]\\w*",NUMBER_RE:"\\b\\d+(\\.\\d+)?",C_NUMBER_RE:g,BINARY_NUMBER_RE:"\\b(0b[01]+)",RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",SHEBANG:(e={})=>{const n=/^#![ ]*\//;return e.binary&&(e.begin=function(...e){return e.map(e=>d(e)).join("")}(n,/.*\b/,e.binary,/\b.*/)),r({className:"meta",begin:n,end:/$/,relevance:0,"on:begin":(e,n)=>{0!==e.index&&n.ignoreMatch()}},e)},BACKSLASH_ESCAPE:h,APOS_STRING_MODE:f,QUOTE_STRING_MODE:p,PHRASAL_WORDS_MODE:b,COMMENT:m,C_LINE_COMMENT_MODE:v,C_BLOCK_COMMENT_MODE:x,HASH_COMMENT_MODE:E,NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?",relevance:0},C_NUMBER_MODE:{className:"number",begin:g,relevance:0},BINARY_NUMBER_MODE:{className:"number",begin:"\\b(0b[01]+)",relevance:0},CSS_NUMBER_MODE:{className:"number",begin:"\\b\\d+(\\.\\d+)?(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp",begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[h,{begin:/\[/,end:/\]/,relevance:0,contains:[h]}]}]},TITLE_MODE:{className:"title",begin:"[a-zA-Z]\\w*",relevance:0},UNDERSCORE_TITLE_MODE:{className:"title",begin:"[a-zA-Z_]\\w*",relevance:0},METHOD_GUARD:{begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:function(e){return Object.assign(e,{"on:begin":(e,n)=>{n.data._beginMatch=e[1]},"on:end":(e,n)=>{n.data._beginMatch!==e[1]&&n.ignoreMatch()}})}}),N="of and for in not or if then".split(" ");function w(e,n){return n?+n:function(e){return N.includes(e.toLowerCase())}(e)?0:1}const R=t,y=r,{nodeStream:O,mergeStreams:k}=i,M=Symbol("nomatch");return function(t){var a=[],i=Object.create(null),s=Object.create(null),o=[],l=!0,c=/(^(<[^>]+>|\t|)+|\n)/gm,g="Could not find the language '{}', did you forget to load/include a language module?";const h={disableAutodetect:!0,name:"Plain text",contains:[]};var f={noHighlightRe:/^(no-?highlight)$/i,languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:null,__emitter:u};function p(e){return f.noHighlightRe.test(e)}function b(e,n,t,r){var a={code:n,language:e};S("before:highlight",a);var i=a.result?a.result:m(a.language,a.code,t,r);return i.code=a.code,S("after:highlight",i),i}function m(e,t,a,s){var o=t;function c(e,n){var t=E.case_insensitive?n[0].toLowerCase():n[0];return Object.prototype.hasOwnProperty.call(e.keywords,t)&&e.keywords[t]}function u(){null!=y.subLanguage?function(){if(""!==A){var e=null;if("string"==typeof y.subLanguage){if(!i[y.subLanguage])return void k.addText(A);e=m(y.subLanguage,A,!0,O[y.subLanguage]),O[y.subLanguage]=e.top}else e=v(A,y.subLanguage.length?y.subLanguage:null);y.relevance>0&&(I+=e.relevance),k.addSublanguage(e.emitter,e.language)}}():function(){if(!y.keywords)return void k.addText(A);let e=0;y.keywordPatternRe.lastIndex=0;let n=y.keywordPatternRe.exec(A),t="";for(;n;){t+=A.substring(e,n.index);const r=c(y,n);if(r){const[e,a]=r;k.addText(t),t="",I+=a,k.addKeyword(n[0],e)}else t+=n[0];e=y.keywordPatternRe.lastIndex,n=y.keywordPatternRe.exec(A)}t+=A.substr(e),k.addText(t)}(),A=""}function h(e){return e.className&&k.openNode(e.className),y=Object.create(e,{parent:{value:y}})}function p(e){return 0===y.matcher.regexIndex?(A+=e[0],1):(L=!0,0)}var b={};function x(t,r){var i=r&&r[0];if(A+=t,null==i)return u(),0;if("begin"===b.type&&"end"===r.type&&b.index===r.index&&""===i){if(A+=o.slice(r.index,r.index+1),!l){const n=Error("0 width match regex");throw n.languageName=e,n.badRule=b.rule,n}return 1}if(b=r,"begin"===r.type)return function(e){var t=e[0],r=e.rule;const a=new n(r),i=[r.__beforeBegin,r["on:begin"]];for(const n of i)if(n&&(n(e,a),a.ignore))return p(t);return r&&r.endSameAsBegin&&(r.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")),r.skip?A+=t:(r.excludeBegin&&(A+=t),u(),r.returnBegin||r.excludeBegin||(A=t)),h(r),r.returnBegin?0:t.length}(r);if("illegal"===r.type&&!a){const e=Error('Illegal lexeme "'+i+'" for mode "'+(y.className||"")+'"');throw e.mode=y,e}if("end"===r.type){var s=function(e){var t=e[0],r=o.substr(e.index),a=function e(t,r,a){let i=function(e,n){var t=e&&e.exec(n);return t&&0===t.index}(t.endRe,a);if(i){if(t["on:end"]){const e=new n(t);t["on:end"](r,e),e.ignore&&(i=!1)}if(i){for(;t.endsParent&&t.parent;)t=t.parent;return t}}if(t.endsWithParent)return e(t.parent,r,a)}(y,e,r);if(!a)return M;var i=y;i.skip?A+=t:(i.returnEnd||i.excludeEnd||(A+=t),u(),i.excludeEnd&&(A=t));do{y.className&&k.closeNode(),y.skip||y.subLanguage||(I+=y.relevance),y=y.parent}while(y!==a.parent);return a.starts&&(a.endSameAsBegin&&(a.starts.endRe=a.endRe),h(a.starts)),i.returnEnd?0:t.length}(r);if(s!==M)return s}if("illegal"===r.type&&""===i)return 1;if(B>1e5&&B>3*r.index)throw Error("potential infinite loop, way more iterations than matches");return A+=i,i.length}var E=T(e);if(!E)throw console.error(g.replace("{}",e)),Error('Unknown language: "'+e+'"');var _=function(e){function n(n,t){return RegExp(d(n),"m"+(e.case_insensitive?"i":"")+(t?"g":""))}class t{constructor(){this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}addRule(e,n){n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),this.matchAt+=function(e){return RegExp(e.toString()+"|").exec("").length-1}(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null);const e=this.regexes.map(e=>e[1]);this.matcherRe=n(function(e,n="|"){for(var t=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,r=0,a="",i=0;i0&&(a+=n),a+="(";o.length>0;){var l=t.exec(o);if(null==l){a+=o;break}a+=o.substring(0,l.index),o=o.substring(l.index+l[0].length),"\\"===l[0][0]&&l[1]?a+="\\"+(+l[1]+s):(a+=l[0],"("===l[0]&&r++)}a+=")"}return a}(e),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex;const n=this.matcherRe.exec(e);if(!n)return null;const t=n.findIndex((e,n)=>n>0&&void 0!==e),r=this.matchIndexes[t];return n.splice(0,t),Object.assign(n,r)}}class a{constructor(){this.rules=[],this.multiRegexes=[],this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){if(this.multiRegexes[e])return this.multiRegexes[e];const n=new t;return this.rules.slice(e).forEach(([e,t])=>n.addRule(e,t)),n.compile(),this.multiRegexes[e]=n,n}considerAll(){this.regexIndex=0}addRule(e,n){this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){const n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex;const t=n.exec(e);return t&&(this.regexIndex+=t.position+1,this.regexIndex===this.count&&(this.regexIndex=0)),t}}function i(e,n){const t=e.input[e.index-1],r=e.input[e.index+e[0].length];"."!==t&&"."!==r||n.ignoreMatch()}if(e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.");return function t(s,o){const l=s;if(s.compiled)return l;s.compiled=!0,s.__beforeBegin=null,s.keywords=s.keywords||s.beginKeywords;let c=null;if("object"==typeof s.keywords&&(c=s.keywords.$pattern,delete s.keywords.$pattern),s.keywords&&(s.keywords=function(e,n){var t={};return"string"==typeof e?r("keyword",e):Object.keys(e).forEach((function(n){r(n,e[n])})),t;function r(e,r){n&&(r=r.toLowerCase()),r.split(" ").forEach((function(n){var r=n.split("|");t[r[0]]=[e,w(r[0],r[1])]}))}}(s.keywords,e.case_insensitive)),s.lexemes&&c)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ");return l.keywordPatternRe=n(s.lexemes||c||/\w+/,!0),o&&(s.beginKeywords&&(s.begin="\\b("+s.beginKeywords.split(" ").join("|")+")(?=\\b|\\s)",s.__beforeBegin=i),s.begin||(s.begin=/\B|\b/),l.beginRe=n(s.begin),s.endSameAsBegin&&(s.end=s.begin),s.end||s.endsWithParent||(s.end=/\B|\b/),s.end&&(l.endRe=n(s.end)),l.terminator_end=d(s.end)||"",s.endsWithParent&&o.terminator_end&&(l.terminator_end+=(s.end?"|":"")+o.terminator_end)),s.illegal&&(l.illegalRe=n(s.illegal)),void 0===s.relevance&&(s.relevance=1),s.contains||(s.contains=[]),s.contains=[].concat(...s.contains.map((function(e){return function(e){return e.variants&&!e.cached_variants&&(e.cached_variants=e.variants.map((function(n){return r(e,{variants:null},n)}))),e.cached_variants?e.cached_variants:function e(n){return!!n&&(n.endsWithParent||e(n.starts))}(e)?r(e,{starts:e.starts?r(e.starts):null}):Object.isFrozen(e)?r(e):e}("self"===e?s:e)}))),s.contains.forEach((function(e){t(e,l)})),s.starts&&t(s.starts,o),l.matcher=function(e){const n=new a;return e.contains.forEach(e=>n.addRule(e.begin,{rule:e,type:"begin"})),e.terminator_end&&n.addRule(e.terminator_end,{type:"end"}),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n}(l),l}(e)}(E),N="",y=s||_,O={},k=new f.__emitter(f);!function(){for(var e=[],n=y;n!==E;n=n.parent)n.className&&e.unshift(n.className);e.forEach(e=>k.openNode(e))}();var A="",I=0,S=0,B=0,L=!1;try{for(y.matcher.considerAll();;){B++,L?L=!1:(y.matcher.lastIndex=S,y.matcher.considerAll());const e=y.matcher.exec(o);if(!e)break;const n=x(o.substring(S,e.index),e);S=e.index+n}return x(o.substr(S)),k.closeAllNodes(),k.finalize(),N=k.toHTML(),{relevance:I,value:N,language:e,illegal:!1,emitter:k,top:y}}catch(n){if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{msg:n.message,context:o.slice(S-100,S+100),mode:n.mode},sofar:N,relevance:0,value:R(o),emitter:k};if(l)return{illegal:!1,relevance:0,value:R(o),emitter:k,language:e,top:y,errorRaised:n};throw n}}function v(e,n){n=n||f.languages||Object.keys(i);var t=function(e){const n={relevance:0,emitter:new f.__emitter(f),value:R(e),illegal:!1,top:h};return n.emitter.addText(e),n}(e),r=t;return n.filter(T).filter(I).forEach((function(n){var a=m(n,e,!1);a.language=n,a.relevance>r.relevance&&(r=a),a.relevance>t.relevance&&(r=t,t=a)})),r.language&&(t.second_best=r),t}function x(e){return f.tabReplace||f.useBR?e.replace(c,e=>"\n"===e?f.useBR?"
":e:f.tabReplace?e.replace(/\t/g,f.tabReplace):e):e}function E(e){let n=null;const t=function(e){var n=e.className+" ";n+=e.parentNode?e.parentNode.className:"";const t=f.languageDetectRe.exec(n);if(t){var r=T(t[1]);return r||(console.warn(g.replace("{}",t[1])),console.warn("Falling back to no-highlight mode for this block.",e)),r?t[1]:"no-highlight"}return n.split(/\s+/).find(e=>p(e)||T(e))}(e);if(p(t))return;S("before:highlightBlock",{block:e,language:t}),f.useBR?(n=document.createElement("div")).innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n"):n=e;const r=n.textContent,a=t?b(t,r,!0):v(r),i=O(n);if(i.length){const e=document.createElement("div");e.innerHTML=a.value,a.value=k(i,O(e),r)}a.value=x(a.value),S("after:highlightBlock",{block:e,result:a}),e.innerHTML=a.value,e.className=function(e,n,t){var r=n?s[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),e.includes(r)||a.push(r),a.join(" ").trim()}(e.className,t,a.language),e.result={language:a.language,re:a.relevance,relavance:a.relevance},a.second_best&&(e.second_best={language:a.second_best.language,re:a.second_best.relevance,relavance:a.second_best.relevance})}const N=()=>{if(!N.called){N.called=!0;var e=document.querySelectorAll("pre code");a.forEach.call(e,E)}};function T(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]}function A(e,{languageName:n}){"string"==typeof e&&(e=[e]),e.forEach(e=>{s[e]=n})}function I(e){var n=T(e);return n&&!n.disableAutodetect}function S(e,n){var t=e;o.forEach((function(e){e[t]&&e[t](n)}))}Object.assign(t,{highlight:b,highlightAuto:v,fixMarkup:x,highlightBlock:E,configure:function(e){f=y(f,e)},initHighlighting:N,initHighlightingOnLoad:function(){window.addEventListener("DOMContentLoaded",N,!1)},registerLanguage:function(e,n){var r=null;try{r=n(t)}catch(n){if(console.error("Language definition for '{}' could not be registered.".replace("{}",e)),!l)throw n;console.error(n),r=h}r.name||(r.name=e),i[e]=r,r.rawDefinition=n.bind(null,t),r.aliases&&A(r.aliases,{languageName:e})},listLanguages:function(){return Object.keys(i)},getLanguage:T,registerAliases:A,requireLanguage:function(e){var n=T(e);if(n)return n;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))},autoDetection:I,inherit:y,addPlugin:function(e){o.push(e)}}),t.debugMode=function(){l=!1},t.safeMode=function(){l=!0},t.versionString="10.1.2";for(const n in _)"object"==typeof _[n]&&e(_[n]);return Object.assign(t,_),t}({})}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs); +hljs.registerLanguage("apache",function(){"use strict";return function(e){var n={className:"number",begin:"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?"};return{name:"Apache config",aliases:["apacheconf"],case_insensitive:!0,contains:[e.HASH_COMMENT_MODE,{className:"section",begin:"",contains:[n,{className:"number",begin:":\\d{1,5}"},e.inherit(e.QUOTE_STRING_MODE,{relevance:0})]},{className:"attribute",begin:/\w+/,relevance:0,keywords:{nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername"},starts:{end:/$/,relevance:0,keywords:{literal:"on off all deny allow"},contains:[{className:"meta",begin:"\\s\\[",end:"\\]$"},{className:"variable",begin:"[\\$%]\\{",end:"\\}",contains:["self",{className:"number",begin:"[\\$%]\\d+"}]},n,{className:"number",begin:"\\d+"},e.QUOTE_STRING_MODE]}}],illegal:/\S/}}}()); +hljs.registerLanguage("bash",function(){"use strict";return function(e){const s={};Object.assign(s,{className:"variable",variants:[{begin:/\$[\w\d#@][\w\d_]*/},{begin:/\$\{/,end:/\}/,contains:[{begin:/:-/,contains:[s]}]}]});const t={className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},n={className:"string",begin:/"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,t]};t.contains.push(n);const a={begin:/\$\(\(/,end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,s]},i=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10}),c={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b-?[a-z\._]+\b/,keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[i,e.SHEBANG(),c,a,e.HASH_COMMENT_MODE,n,{className:"",begin:/\\"/},{className:"string",begin:/'/,end:/'/},s]}}}()); +hljs.registerLanguage("c-like",function(){"use strict";return function(e){function t(e){return"(?:"+e+")?"}var n="(decltype\\(auto\\)|"+t("[a-zA-Z_]\\w*::")+"[a-zA-Z_]\\w*"+t("<.*?>")+")",r={className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},a={className:"string",variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",end:"'",illegal:"."},e.END_SAME_AS_BEGIN({begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},i={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"},contains:[{begin:/\\\n/,relevance:0},e.inherit(a,{className:"meta-string"}),{className:"meta-string",begin:/<.*?>/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},o={className:"title",begin:t("[a-zA-Z_]\\w*::")+e.IDENT_RE,relevance:0},c=t("[a-zA-Z_]\\w*::")+e.IDENT_RE+"\\s*\\(",l={keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq",built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary",literal:"true false nullptr NULL"},d=[r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,i,a],_={variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],keywords:l,contains:d.concat([{begin:/\(/,end:/\)/,keywords:l,contains:d.concat(["self"]),relevance:0}]),relevance:0},u={className:"function",begin:"("+n+"[\\*&\\s]+)+"+c,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:l,illegal:/[^\w\s\*&:<>]/,contains:[{begin:"decltype\\(auto\\)",keywords:l,relevance:0},{begin:c,returnBegin:!0,contains:[o],relevance:0},{className:"params",begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r,{begin:/\(/,end:/\)/,keywords:l,relevance:0,contains:["self",e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,a,i,r]}]},r,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,s]};return{aliases:["c","cc","h","c++","h++","hpp","hh","hxx","cxx"],keywords:l,disableAutodetect:!0,illegal:"",keywords:l,contains:["self",r]},{begin:e.IDENT_RE+"::",keywords:l},{className:"class",beginKeywords:"class struct",end:/[{;:]/,contains:[{begin://,contains:["self"]},e.TITLE_MODE]}]),exports:{preprocessor:s,strings:a,keywords:l}}}}()); +hljs.registerLanguage("c",function(){"use strict";return function(e){var n=e.getLanguage("c-like").rawDefinition();return n.name="C",n.aliases=["c","h"],n}}()); +hljs.registerLanguage("coffeescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={keyword:e.concat(["then","unless","until","loop","by","when","and","or","is","isnt","not"]).filter((e=>n=>!e.includes(n))(["var","const","let","function","static"])).join(" "),literal:n.concat(["yes","no","on","off"]).join(" "),built_in:a.concat(["npm","print"]).join(" ")},i="[A-Za-z$_][0-9A-Za-z$_]*",s={className:"subst",begin:/#\{/,end:/}/,keywords:t},o=[r.BINARY_NUMBER_MODE,r.inherit(r.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/,contains:[r.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[r.BACKSLASH_ESCAPE]},{begin:/"""/,end:/"""/,contains:[r.BACKSLASH_ESCAPE,s]},{begin:/"/,end:/"/,contains:[r.BACKSLASH_ESCAPE,s]}]},{className:"regexp",variants:[{begin:"///",end:"///",contains:[s,r.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)",relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+i},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{begin:"```",end:"```"},{begin:"`",end:"`"}]}];s.contains=o;var c=r.inherit(r.TITLE_MODE,{begin:i}),l={className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/,end:/\)/,keywords:t,contains:["self"].concat(o)}]};return{name:"CoffeeScript",aliases:["coffee","cson","iced"],keywords:t,illegal:/\/\*/,contains:o.concat([r.COMMENT("###","###"),r.HASH_COMMENT_MODE,{className:"function",begin:"^\\s*"+i+"\\s*=\\s*(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[c,l]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function",begin:"(\\(.*\\))?\\s*\\B[-=]>",end:"[-=]>",returnBegin:!0,contains:[l]}]},{className:"class",beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[c]},c]},{begin:i+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}}()); +hljs.registerLanguage("cpp",function(){"use strict";return function(e){var t=e.getLanguage("c-like").rawDefinition();return t.disableAutodetect=!1,t.name="C++",t.aliases=["cc","c++","h++","hpp","hh","hxx","cxx"],t}}()); +hljs.registerLanguage("csharp",function(){"use strict";return function(e){var n={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let nameof on orderby partial remove select set value var when where yield",literal:"null false true"},i=e.inherit(e.TITLE_MODE,{begin:"[a-zA-Z](\\.?\\w)*"}),a={className:"number",variants:[{begin:"\\b(0b[01']+)"},{begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],relevance:0},s={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]},t=e.inherit(s,{illegal:/\n/}),l={className:"subst",begin:"{",end:"}",keywords:n},r=e.inherit(l,{illegal:/\n/}),c={className:"string",begin:/\$"/,end:'"',illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},e.BACKSLASH_ESCAPE,r]},o={className:"string",begin:/\$@"/,end:'"',contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},l]},g=e.inherit(o,{illegal:/\n/,contains:[{begin:"{{"},{begin:"}}"},{begin:'""'},r]});l.contains=[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.C_BLOCK_COMMENT_MODE],r.contains=[g,c,t,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.inherit(e.C_BLOCK_COMMENT_MODE,{illegal:/\n/})];var d={variants:[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},E={begin:"<",end:">",contains:[{beginKeywords:"in out"},i]},_=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",b={begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"],keywords:n,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0,contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{begin:"\x3c!--|--\x3e"},{begin:""}]}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#",end:"$",keywords:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},d,a,{beginKeywords:"class interface",end:/[{;=]/,illegal:/[^\s:,]/,contains:[{beginKeywords:"where class"},i,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace",end:/[{;=]/,illegal:/[^\s:]/,contains:[i,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta",begin:"^\\s*\\[",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{className:"meta-string",begin:/"/,end:/"/}]},{beginKeywords:"new return throw await else",relevance:0},{className:"function",begin:"("+_+"\\s+)+"+e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,end:/\s*[{;=]/,excludeEnd:!0,keywords:n,contains:[{begin:e.IDENT_RE+"\\s*(\\<.+\\>)?\\s*\\(",returnBegin:!0,contains:[e.TITLE_MODE,E],relevance:0},{className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,relevance:0,contains:[d,a,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},b]}}}()); +hljs.registerLanguage("css",function(){"use strict";return function(e){var n={begin:/(?:[A-Z\_\.\-]+|--[a-zA-Z0-9_-]+)\s*:/,returnBegin:!0,end:";",endsWithParent:!0,contains:[{className:"attribute",begin:/\S/,end:":",excludeEnd:!0,starts:{endsWithParent:!0,excludeEnd:!0,contains:[{begin:/[\w-]+\(/,returnBegin:!0,contains:[{className:"built_in",begin:/[\w-]+/},{begin:/\(/,end:/\)/,contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{className:"number",begin:"#[0-9A-Fa-f]+"},{className:"meta",begin:"!important"}]}}]};return{name:"CSS",case_insensitive:!0,illegal:/[=\/|'\$]/,contains:[e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/},{className:"selector-class",begin:/\.[A-Za-z0-9_-]+/},{className:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",illegal:/:/,returnBegin:!0,contains:[{className:"keyword",begin:/@\-?\w[\w]*(\-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0,relevance:0,keywords:"and or not only",contains:[{begin:/[a-z-]+:/,className:"attribute"},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.CSS_NUMBER_MODE]}]},{className:"selector-tag",begin:"[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0},{begin:"{",end:"}",illegal:/\S/,contains:[e.C_BLOCK_COMMENT_MODE,n]}]}}}()); +hljs.registerLanguage("diff",function(){"use strict";return function(e){return{name:"Diff",aliases:["patch"],contains:[{className:"meta",relevance:10,variants:[{begin:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{begin:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{begin:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{className:"comment",variants:[{begin:/Index: /,end:/$/},{begin:/={3,}/,end:/$/},{begin:/^\-{3}/,end:/$/},{begin:/^\*{3} /,end:/$/},{begin:/^\+{3}/,end:/$/},{begin:/^\*{15}$/}]},{className:"addition",begin:"^\\+",end:"$"},{className:"deletion",begin:"^\\-",end:"$"},{className:"addition",begin:"^\\!",end:"$"}]}}}()); +hljs.registerLanguage("go",function(){"use strict";return function(e){var n={keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune",literal:"true false iota nil",built_in:"append cap close complex copy imag len make new panic print println real recover delete"};return{name:"Go",aliases:["golang"],keywords:n,illegal:"e(n)).join("")}return function(a){var s={className:"number",relevance:0,variants:[{begin:/([\+\-]+)?[\d]+_[\d_]+/},{begin:a.NUMBER_RE}]},i=a.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];var t={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)}/}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={className:"string",contains:[a.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,s,"self"],relevance:0},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map(n=>e(n)).join("|")+")";return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr",starts:{end:/$/,contains:[i,c,r,t,l,s]}}]}}}()); +hljs.registerLanguage("java",function(){"use strict";function e(e){return e?"string"==typeof e?e:e.source:null}function n(e){return a("(",e,")?")}function a(...n){return n.map(n=>e(n)).join("")}function s(...n){return"("+n.map(n=>e(n)).join("|")+")"}return function(e){var t="false synchronized int abstract float private char boolean var static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",i={className:"meta",begin:"@[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",contains:[{begin:/\(/,end:/\)/,contains:["self"]}]},r=e=>a("[",e,"]+([",e,"_]*[",e,"]+)?"),c={className:"number",variants:[{begin:`\\b(0[bB]${r("01")})[lL]?`},{begin:`\\b(0${r("0-7")})[dDfFlL]?`},{begin:a(/\b0[xX]/,s(a(r("a-fA-F0-9"),/\./,r("a-fA-F0-9")),a(r("a-fA-F0-9"),/\.?/),a(/\./,r("a-fA-F0-9"))),/([pP][+-]?(\d+))?/,/[fFdDlL]?/)},{begin:a(/\b/,s(a(/\d*\./,r("\\d")),r("\\d")),/[eE][+-]?[\d]+[dDfF]?/)},{begin:a(/\b/,r(/\d/),n(/\.?/),n(r(/\d/)),/[dDfFlL]?/)}],relevance:0};return{name:"Java",aliases:["jsp"],keywords:t,illegal:/<\/|#/,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"class",beginKeywords:"class interface",end:/[{;=]/,excludeEnd:!0,keywords:"class interface",illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"new throw return else",relevance:0},{className:"function",begin:"([À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(<[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*(\\s*,\\s*[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*)*>)?\\s+)+"+e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:t,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"params",begin:/\(/,end:/\)/,keywords:t,relevance:0,contains:[i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE]},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},c,i]}}}()); +hljs.registerLanguage("javascript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);function s(e){return r("(?=",e,")")}function r(...e){return e.map(e=>(function(e){return e?"string"==typeof e?e:e.source:null})(e)).join("")}return function(t){var i="[A-Za-z$_][0-9A-Za-z$_]*",c={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/},o={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.join(" "),literal:n.join(" "),built_in:a.join(" ")},l={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:t.C_NUMBER_RE+"n?"}],relevance:0},E={className:"subst",begin:"\\$\\{",end:"\\}",keywords:o,contains:[]},d={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"xml"}},g={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[t.BACKSLASH_ESCAPE,E],subLanguage:"css"}},u={className:"string",begin:"`",end:"`",contains:[t.BACKSLASH_ESCAPE,E]};E.contains=[t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,l,t.REGEXP_MODE];var b=E.contains.concat([{begin:/\(/,end:/\)/,contains:["self"].concat(E.contains,[t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE])},t.C_BLOCK_COMMENT_MODE,t.C_LINE_COMMENT_MODE]),_={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:b};return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:o,contains:[t.SHEBANG({binary:"node",relevance:5}),{className:"meta",relevance:10,begin:/^\s*['"]use (strict|asm)['"]/},t.APOS_STRING_MODE,t.QUOTE_STRING_MODE,d,g,u,t.C_LINE_COMMENT_MODE,t.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+",contains:[{className:"type",begin:"\\{",end:"\\}",relevance:0},{className:"variable",begin:i+"(?=\\s*(-)|$)",endsParent:!0,relevance:0},{begin:/(?=[^\n])\s/,relevance:0}]}]}),t.C_BLOCK_COMMENT_MODE,l,{begin:r(/[{,\n]\s*/,s(r(/(((\/\/.*)|(\/\*(.|\n)*\*\/))\s*)*/,i+"\\s*:"))),relevance:0,contains:[{className:"attr",begin:i+s("\\s*:"),relevance:0}]},{begin:"("+t.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[t.C_LINE_COMMENT_MODE,t.C_BLOCK_COMMENT_MODE,t.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+t.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:t.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:o,contains:b}]}]},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{variants:[{begin:"<>",end:""},{begin:c.begin,end:c.end}],subLanguage:"xml",contains:[{begin:c.begin,end:c.end,skip:!0,contains:["self"]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/\{/,excludeEnd:!0,contains:[t.inherit(t.TITLE_MODE,{begin:i}),_],illegal:/\[|%/},{begin:/\$[(.]/},t.METHOD_GUARD,{className:"class",beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"\[\]]/,contains:[{beginKeywords:"extends"},t.UNDERSCORE_TITLE_MODE]},{beginKeywords:"constructor",end:/\{/,excludeEnd:!0},{begin:"(get|set)\\s+(?="+i+"\\()",end:/{/,keywords:"get set",contains:[t.inherit(t.TITLE_MODE,{begin:i}),{begin:/\(\)/},_]}],illegal:/#(?!!)/}}}()); +hljs.registerLanguage("json",function(){"use strict";return function(n){var e={literal:"true false null"},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],t=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],a={end:",",endsWithParent:!0,excludeEnd:!0,contains:t,keywords:e},l={begin:"{",end:"}",contains:[{className:"attr",begin:/"/,end:/"/,contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(a,{begin:/:/})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(a)],illegal:"\\S"};return t.push(l,s),i.forEach((function(n){t.push(n)})),{name:"JSON",contains:t,keywords:e,illegal:"\\S"}}}()); +hljs.registerLanguage("kotlin",function(){"use strict";return function(e){var n={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual trait volatile transient native default",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},a={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@"},i={className:"subst",begin:"\\${",end:"}",contains:[e.C_NUMBER_MODE]},s={className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},t={className:"string",variants:[{begin:'"""',end:'"""(?=[^"])',contains:[s,i]},{begin:"'",end:"'",illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/,contains:[e.BACKSLASH_ESCAPE,s,i]}]};i.contains.push(t);var r={className:"meta",begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?"},l={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/,end:/\)/,contains:[e.inherit(t,{className:"meta-string"})]}]},c=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),o={variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/,contains:[]}]},d=o;return d.variants[1].contains=[o],o.variants[1].contains=[d],{name:"Kotlin",aliases:["kt"],keywords:n,contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,c,{className:"keyword",begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol",begin:/@\w+/}]}},a,r,l,{className:"function",beginKeywords:"fun",end:"[(]|$",returnBegin:!0,excludeEnd:!0,keywords:n,illegal:/fun\s+(<.*>)?[^\s\(]+(\s+[^\s\(]+)\s*=/,relevance:5,contains:[{begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://,keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:n,relevance:0,contains:[{begin:/:/,end:/[=,\/]/,endsWithParent:!0,contains:[o,e.C_LINE_COMMENT_MODE,c],relevance:0},e.C_LINE_COMMENT_MODE,c,r,l,t,e.C_NUMBER_MODE]},c]},{className:"class",beginKeywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0,illegal:"extends implements",contains:[{beginKeywords:"public protected internal private constructor"},e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,]|$/,excludeBegin:!0,returnEnd:!0},r,l]},t,{className:"meta",begin:"^#!/usr/bin/env",end:"$",illegal:"\n"},{className:"number",begin:"\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",relevance:0}]}}}()); +hljs.registerLanguage("less",function(){"use strict";return function(e){var n="([\\w-]+|@{[\\w-]+})",a=[],s=[],t=function(e){return{className:"string",begin:"~?"+e+".*?"+e}},r=function(e,n,a){return{className:e,begin:n,relevance:a}},i={begin:"\\(",end:"\\)",contains:s,relevance:0};s.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,t("'"),t('"'),e.CSS_NUMBER_MODE,{begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]",excludeEnd:!0}},r("number","#[0-9A-Fa-f]+\\b"),i,r("variable","@@?[\\w-]+",10),r("variable","@{[\\w-]+}"),r("built_in","~?`[^`]*?`"),{className:"attribute",begin:"[\\w-]+\\s*:",end:":",returnBegin:!0,excludeEnd:!0},{className:"meta",begin:"!important"});var c=s.concat({begin:"{",end:"}",contains:a}),l={beginKeywords:"when",endsWithParent:!0,contains:[{beginKeywords:"and not"}].concat(s)},o={begin:n+"\\s*:",returnBegin:!0,end:"[;}]",relevance:0,contains:[{className:"attribute",begin:n,end:":",excludeEnd:!0,starts:{endsWithParent:!0,illegal:"[<=$]",relevance:0,contains:s}}]},g={className:"keyword",begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",starts:{end:"[;{}]",returnEnd:!0,contains:s,relevance:0}},d={className:"variable",variants:[{begin:"@[\\w-]+\\s*:",relevance:15},{begin:"@[\\w-]+"}],starts:{end:"[;}]",returnEnd:!0,contains:c}},b={variants:[{begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:n,end:"{"}],returnBegin:!0,returnEnd:!0,illegal:"[<='$\"]",relevance:0,contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,l,r("keyword","all\\b"),r("variable","@{[\\w-]+}"),r("selector-tag",n+"%?",0),r("selector-id","#"+n),r("selector-class","\\."+n,0),r("selector-tag","&",0),{className:"selector-attr",begin:"\\[",end:"\\]"},{className:"selector-pseudo",begin:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{begin:"\\(",end:"\\)",contains:c},{begin:"!important"}]};return a.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,g,d,o,b),{name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:a}}}()); +hljs.registerLanguage("lua",function(){"use strict";return function(e){var t={begin:"\\[=*\\[",end:"\\]=*\\]",contains:["self"]},a=[e.COMMENT("--(?!\\[=*\\[)","$"),e.COMMENT("--\\[=*\\[","\\]=*\\]",{contains:[t],relevance:10})];return{name:"Lua",keywords:{$pattern:e.UNDERSCORE_IDENT_RE,literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},contains:a.concat([{className:"function",beginKeywords:"function",end:"\\)",contains:[e.inherit(e.TITLE_MODE,{begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params",begin:"\\(",endsWithParent:!0,contains:a}].concat(a)},e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",begin:"\\[=*\\[",end:"\\]=*\\]",contains:[t],relevance:5}])}}}()); +hljs.registerLanguage("makefile",function(){"use strict";return function(e){var i={className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)",contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%`]+/}]}]}]};return{name:"HTML, XML",aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],case_insensitive:!0,contains:[{className:"meta",begin:"",relevance:10,contains:[a,i,t,s,{begin:"\\[",end:"\\]",contains:[{className:"meta",begin:"",contains:[a,s,i,t]}]}]},e.COMMENT("\x3c!--","--\x3e",{relevance:10}),{begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},n,{className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag",begin:")",end:">",keywords:{name:"style"},contains:[c],starts:{end:"",returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",begin:")",end:">",keywords:{name:"script"},contains:[c],starts:{end:"<\/script>",returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{className:"tag",begin:"",contains:[{className:"name",begin:/[^\/><\s]+/,relevance:0},c]}]}}}()); +hljs.registerLanguage("markdown",function(){"use strict";return function(n){const e={begin:"<",end:">",subLanguage:"xml",relevance:0},a={begin:"\\[.+?\\][\\(\\[].*?[\\)\\]]",returnBegin:!0,contains:[{className:"string",begin:"\\[",end:"\\]",excludeBegin:!0,returnEnd:!0,relevance:0},{className:"link",begin:"\\]\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0},{className:"symbol",begin:"\\]\\[",end:"\\]",excludeBegin:!0,excludeEnd:!0}],relevance:10},i={className:"strong",contains:[],variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},s={className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{begin:/_(?!_)/,end:/_/,relevance:0}]};i.contains.push(s),s.contains.push(i);var c=[e,a];return i.contains=i.contains.concat(c),s.contains=s.contains.concat(c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:c=c.concat(i,s)},{begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",contains:c}]}]},e,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",end:"\\s+",excludeEnd:!0},i,s,{className:"quote",begin:"^>\\s+",contains:c,end:"$"},{className:"code",variants:[{begin:"(`{3,})(.|\\n)*?\\1`*[ ]*"},{begin:"(~{3,})(.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{begin:"^[-\\*]{3,}",end:"$"},a,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}}()); +hljs.registerLanguage("nginx",function(){"use strict";return function(e){var n={className:"variable",variants:[{begin:/\$\d+/},{begin:/\$\{/,end:/}/},{begin:"[\\$\\@]"+e.UNDERSCORE_IDENT_RE}]},a={endsWithParent:!0,keywords:{$pattern:"[a-z/_]+",literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},relevance:0,illegal:"=>",contains:[e.HASH_COMMENT_MODE,{className:"string",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:/"/,end:/"/},{begin:/'/,end:/'/}]},{begin:"([a-z]+):/",end:"\\s",endsWithParent:!0,excludeEnd:!0,contains:[n]},{className:"regexp",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:"\\s\\^",end:"\\s|{|;",returnEnd:!0},{begin:"~\\*?\\s+",end:"\\s|{|;",returnEnd:!0},{begin:"\\*(\\.[a-z\\-]+)+"},{begin:"([a-z\\-]+\\.)+\\*"}]},{className:"number",begin:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{className:"number",begin:"\\b\\d+[kKmMgGdshdwy]*\\b",relevance:0},n]};return{name:"Nginx config",aliases:["nginxconf"],contains:[e.HASH_COMMENT_MODE,{begin:e.UNDERSCORE_IDENT_RE+"\\s+{",returnBegin:!0,end:"{",contains:[{className:"section",begin:e.UNDERSCORE_IDENT_RE}],relevance:0},{begin:e.UNDERSCORE_IDENT_RE+"\\s",end:";|{",returnBegin:!0,contains:[{className:"attribute",begin:e.UNDERSCORE_IDENT_RE,starts:a}],relevance:0}],illegal:"[^\\s\\}]"}}}()); +hljs.registerLanguage("objectivec",function(){"use strict";return function(e){var n=/[a-zA-Z@][a-zA-Z0-9_]*/,_={$pattern:n,keyword:"@interface @class @protocol @implementation"};return{name:"Objective-C",aliases:["mm","objc","obj-c"],keywords:{$pattern:n,keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once"},illegal:"/,end:/$/,illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"class",begin:"("+_.keyword.split(" ").join("|")+")\\b",end:"({|$)",excludeEnd:!0,keywords:_,contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"\\."+e.UNDERSCORE_IDENT_RE,relevance:0}]}}}()); +hljs.registerLanguage("perl",function(){"use strict";return function(e){var n={$pattern:/[\w.]+/,keyword:"getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qq fileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmget sub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedir ioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when"},t={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:n},s={begin:"->{",end:"}"},r={variants:[{begin:/\$\d/},{begin:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{begin:/[\$%@][^\s\w{]/,relevance:0}]},i=[e.BACKSLASH_ESCAPE,t,r],a=[r,e.HASH_COMMENT_MODE,e.COMMENT("^\\=\\w","\\=cut",{endsWithParent:!0}),s,{className:"string",contains:i,variants:[{begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*\\<",end:"\\>",relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",contains:[e.BACKSLASH_ESCAPE]},{begin:"{\\w+}",contains:[],relevance:0},{begin:"-?\\w+\\s*\\=\\>",contains:[],relevance:0}]},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[e.HASH_COMMENT_MODE,{className:"regexp",begin:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",relevance:10},{className:"regexp",begin:"(m|qr)?/",end:"/[a-z]*",contains:[e.BACKSLASH_ESCAPE],relevance:0}]},{className:"function",beginKeywords:"sub",end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]}];return t.contains=a,s.contains=a,{name:"Perl",aliases:["pl","pm"],keywords:n,contains:a}}}()); +hljs.registerLanguage("php",function(){"use strict";return function(e){var r={begin:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"},t={className:"meta",variants:[{begin:/<\?php/,relevance:10},{begin:/<\?[=]?/},{begin:/\?>/}]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:'b"',end:'"'},{begin:"b'",end:"'"},e.inherit(e.APOS_STRING_MODE,{illegal:null}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null})]},n={variants:[e.BINARY_NUMBER_MODE,e.C_NUMBER_MODE]},i={keyword:"__CLASS__ __DIR__ __FILE__ __FUNCTION__ __LINE__ __METHOD__ __NAMESPACE__ __TRAIT__ die echo exit include include_once print require require_once array abstract and as binary bool boolean break callable case catch class clone const continue declare default do double else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval extends final finally float for foreach from global goto if implements instanceof insteadof int integer interface isset iterable list new object or private protected public real return string switch throw trait try unset use var void while xor yield",literal:"false null true",built_in:"Error|0 AppendIterator ArgumentCountError ArithmeticError ArrayIterator ArrayObject AssertionError BadFunctionCallException BadMethodCallException CachingIterator CallbackFilterIterator CompileError Countable DirectoryIterator DivisionByZeroError DomainException EmptyIterator ErrorException Exception FilesystemIterator FilterIterator GlobIterator InfiniteIterator InvalidArgumentException IteratorIterator LengthException LimitIterator LogicException MultipleIterator NoRewindIterator OutOfBoundsException OutOfRangeException OuterIterator OverflowException ParentIterator ParseError RangeException RecursiveArrayIterator RecursiveCachingIterator RecursiveCallbackFilterIterator RecursiveDirectoryIterator RecursiveFilterIterator RecursiveIterator RecursiveIteratorIterator RecursiveRegexIterator RecursiveTreeIterator RegexIterator RuntimeException SeekableIterator SplDoublyLinkedList SplFileInfo SplFileObject SplFixedArray SplHeap SplMaxHeap SplMinHeap SplObjectStorage SplObserver SplObserver SplPriorityQueue SplQueue SplStack SplSubject SplSubject SplTempFileObject TypeError UnderflowException UnexpectedValueException ArrayAccess Closure Generator Iterator IteratorAggregate Serializable Throwable Traversable WeakReference Directory __PHP_Incomplete_Class parent php_user_filter self static stdClass"};return{aliases:["php","php3","php4","php5","php6","php7"],case_insensitive:!0,keywords:i,contains:[e.HASH_COMMENT_MODE,e.COMMENT("//","$",{contains:[t]}),e.COMMENT("/\\*","\\*/",{contains:[{className:"doctag",begin:"@[A-Za-z]+"}]}),e.COMMENT("__halt_compiler.+?;",!1,{endsWithParent:!0,keywords:"__halt_compiler"}),{className:"string",begin:/<<<['"]?\w+['"]?$/,end:/^\w+;?$/,contains:[e.BACKSLASH_ESCAPE,{className:"subst",variants:[{begin:/\$\w+/},{begin:/\{\$/,end:/\}/}]}]},t,{className:"keyword",begin:/\$this\b/},r,{begin:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{className:"function",beginKeywords:"fn function",end:/[;{]/,excludeEnd:!0,illegal:"[$%\\[]",contains:[e.UNDERSCORE_TITLE_MODE,{className:"params",begin:"\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0,keywords:i,contains:["self",r,e.C_BLOCK_COMMENT_MODE,a,n]}]},{className:"class",beginKeywords:"class interface",end:"{",excludeEnd:!0,illegal:/[:\(\$"]/,contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"namespace",end:";",illegal:/[\.']/,contains:[e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"use",end:";",contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"=>"},a,n]}}}()); +hljs.registerLanguage("php-template",function(){"use strict";return function(n){return{name:"PHP template",subLanguage:"xml",contains:[{begin:/<\?(php|=)?/,end:/\?>/,subLanguage:"php",contains:[{begin:"/\\*",end:"\\*/",skip:!0},{begin:'b"',end:'"',skip:!0},{begin:"b'",end:"'",skip:!0},n.inherit(n.APOS_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0}),n.inherit(n.QUOTE_STRING_MODE,{illegal:null,className:null,contains:null,skip:!0})]}]}}}()); +hljs.registerLanguage("plaintext",function(){"use strict";return function(t){return{name:"Plain text",aliases:["text","txt"],disableAutodetect:!0}}}()); +hljs.registerLanguage("properties",function(){"use strict";return function(e){var n="[ \\t\\f]*",t="("+n+"[:=]"+n+"|[ \\t\\f]+)",a="([^\\\\:= \\t\\f\\n]|\\\\.)+",s={end:t,relevance:0,starts:{className:"string",end:/$/,relevance:0,contains:[{begin:"\\\\\\n"}]}};return{name:".properties",case_insensitive:!0,illegal:/\S/,contains:[e.COMMENT("^\\s*[!#]","$"),{begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+"+t,returnBegin:!0,contains:[{className:"attr",begin:"([^\\\\\\W:= \\t\\f\\n]|\\\\.)+",endsParent:!0,relevance:0}],starts:s},{begin:a+t,returnBegin:!0,relevance:0,contains:[{className:"meta",begin:a,endsParent:!0,relevance:0}],starts:s},{className:"attr",relevance:0,begin:a+n+"$"}]}}}()); +hljs.registerLanguage("python",function(){"use strict";return function(e){var n={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10",built_in:"Ellipsis NotImplemented",literal:"False None True"},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/,end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},r={className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{begin:/(u|b)?r?'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(u|b)?r?"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{begin:/(fr|rf|f)'''/,end:/'''/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(fr|rf|f)"""/,end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/(u|r|ur)'/,end:/'/,relevance:10},{begin:/(u|r|ur)"/,end:/"/,relevance:10},{begin:/(b|br)'/,end:/'/},{begin:/(b|br)"/,end:/"/},{begin:/(fr|rf|f)'/,end:/'/,contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/(fr|rf|f)"/,end:/"/,contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},l={className:"number",relevance:0,variants:[{begin:e.BINARY_NUMBER_RE+"[lLjJ]?"},{begin:"\\b(0o[0-7]+)[lLjJ]?"},{begin:e.C_NUMBER_RE+"[lLjJ]?"}]},t={className:"params",variants:[{begin:/\(\s*\)/,skip:!0,className:null},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,contains:["self",a,l,r,e.HASH_COMMENT_MODE]}]};return i.contains=[r,l,a],{name:"Python",aliases:["py","gyp","ipython"],keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,l,{beginKeywords:"if",relevance:0},r,e.HASH_COMMENT_MODE,{variants:[{className:"function",beginKeywords:"def"},{className:"class",beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/,contains:[e.UNDERSCORE_TITLE_MODE,t,{begin:/->/,endsWithParent:!0,keywords:"None"}]},{className:"meta",begin:/^[\t ]*@/,end:/$/},{begin:/\b(print|exec)\(/}]}}}()); +hljs.registerLanguage("python-repl",function(){"use strict";return function(n){return{aliases:["pycon"],contains:[{className:"meta",starts:{end:/ |$/,starts:{end:"$",subLanguage:"python"}},variants:[{begin:/^>>>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}}}()); +hljs.registerLanguage("ruby",function(){"use strict";return function(e){var n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",a={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},s={className:"doctag",begin:"@[A-Za-z]+"},i={begin:"#<",end:">"},r=[e.COMMENT("#","$",{contains:[s]}),e.COMMENT("^\\=begin","^\\=end",{contains:[s],relevance:10}),e.COMMENT("^__END__","\\n$")],c={className:"subst",begin:"#\\{",end:"}",keywords:a},t={className:"string",contains:[e.BACKSLASH_ESCAPE,c],variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:"%[qQwWx]?\\(",end:"\\)"},{begin:"%[qQwWx]?\\[",end:"\\]"},{begin:"%[qQwWx]?{",end:"}"},{begin:"%[qQwWx]?<",end:">"},{begin:"%[qQwWx]?/",end:"/"},{begin:"%[qQwWx]?%",end:"%"},{begin:"%[qQwWx]?-",end:"-"},{begin:"%[qQwWx]?\\|",end:"\\|"},{begin:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{begin:/<<[-~]?'?(\w+)(?:.|\n)*?\n\s*\1\b/,returnBegin:!0,contains:[{begin:/<<[-~]?'?/},e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,contains:[e.BACKSLASH_ESCAPE,c]})]}]},b={className:"params",begin:"\\(",end:"\\)",endsParent:!0,keywords:a},d=[t,i,{className:"class",beginKeywords:"class module",end:"$|;",illegal:/=/,contains:[e.inherit(e.TITLE_MODE,{begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{begin:"<\\s*",contains:[{begin:"("+e.IDENT_RE+"::)?"+e.IDENT_RE}]}].concat(r)},{className:"function",beginKeywords:"def",end:"$|;",contains:[e.inherit(e.TITLE_MODE,{begin:n}),b].concat(r)},{begin:e.IDENT_RE+"::"},{className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"(\\!|\\?)?:",relevance:0},{className:"symbol",begin:":(?!\\s)",contains:[t,{begin:n}],relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{className:"params",begin:/\|/,end:/\|/,keywords:a},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[i,{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:"%r{",end:"}[a-z]*"},{begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}]}].concat(r),relevance:0}].concat(r);c.contains=d,b.contains=d;var g=[{begin:/^\s*=>/,starts:{end:"$",contains:d}},{className:"meta",begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>)",starts:{end:"$",contains:d}}];return{name:"Ruby",aliases:["rb","gemspec","podspec","thor","irb"],keywords:a,illegal:/\/\*/,contains:r.concat(g).concat(d)}}}()); +hljs.registerLanguage("rust",function(){"use strict";return function(e){var n="([ui](8|16|32|64|128|size)|f(32|64))?",t="drop i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 str char bool Box Option Result String Vec Copy Send Sized Sync Drop Fn FnMut FnOnce ToOwned Clone Debug PartialEq PartialOrd Eq Ord AsRef AsMut Into From Default Iterator Extend IntoIterator DoubleEndedIterator ExactSizeIterator SliceConcatExt ToString assert! assert_eq! bitflags! bytes! cfg! col! concat! concat_idents! debug_assert! debug_assert_eq! env! panic! file! format! format_args! include_bin! include_str! line! local_data_key! module_path! option_env! print! println! select! stringify! try! unimplemented! unreachable! vec! write! writeln! macro_rules! assert_ne! debug_assert_ne!";return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?",keyword:"abstract as async await become box break const continue crate do dyn else enum extern false final fn for if impl in let loop macro match mod move mut override priv pub ref return self Self static struct super trait true try type typeof unsafe unsized use virtual where while yield",literal:"true false Some None Ok Err",built_in:t},illegal:""}]}}}()); +hljs.registerLanguage("scss",function(){"use strict";return function(e){var t={className:"variable",begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b"},i={className:"number",begin:"#[0-9A-Fa-f]+"};return e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,e.C_BLOCK_COMMENT_MODE,{name:"SCSS",case_insensitive:!0,illegal:"[=/|']",contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"selector-id",begin:"\\#[A-Za-z0-9_-]+",relevance:0},{className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0},{className:"selector-attr",begin:"\\[",end:"\\]",illegal:"$"},{className:"selector-tag",begin:"\\b(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|command|datalist|dd|del|details|dfn|div|dl|dt|em|embed|fieldset|figcaption|figure|footer|form|frame|frameset|(h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|keygen|label|legend|li|link|map|mark|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|rp|rt|ruby|samp|script|section|select|small|span|strike|strong|style|sub|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)\\b",relevance:0},{className:"selector-pseudo",begin:":(visited|valid|root|right|required|read-write|read-only|out-range|optional|only-of-type|only-child|nth-of-type|nth-last-of-type|nth-last-child|nth-child|not|link|left|last-of-type|last-child|lang|invalid|indeterminate|in-range|hover|focus|first-of-type|first-line|first-letter|first-child|first|enabled|empty|disabled|default|checked|before|after|active)"},{className:"selector-pseudo",begin:"::(after|before|choices|first-letter|first-line|repeat-index|repeat-item|selection|value)"},t,{className:"attribute",begin:"\\b(src|z-index|word-wrap|word-spacing|word-break|width|widows|white-space|visibility|vertical-align|unicode-bidi|transition-timing-function|transition-property|transition-duration|transition-delay|transition|transform-style|transform-origin|transform|top|text-underline-position|text-transform|text-shadow|text-rendering|text-overflow|text-indent|text-decoration-style|text-decoration-line|text-decoration-color|text-decoration|text-align-last|text-align|tab-size|table-layout|right|resize|quotes|position|pointer-events|perspective-origin|perspective|page-break-inside|page-break-before|page-break-after|padding-top|padding-right|padding-left|padding-bottom|padding|overflow-y|overflow-x|overflow-wrap|overflow|outline-width|outline-style|outline-offset|outline-color|outline|orphans|order|opacity|object-position|object-fit|normal|none|nav-up|nav-right|nav-left|nav-index|nav-down|min-width|min-height|max-width|max-height|mask|marks|margin-top|margin-right|margin-left|margin-bottom|margin|list-style-type|list-style-position|list-style-image|list-style|line-height|letter-spacing|left|justify-content|initial|inherit|ime-mode|image-orientation|image-resolution|image-rendering|icon|hyphens|height|font-weight|font-variant-ligatures|font-variant|font-style|font-stretch|font-size-adjust|font-size|font-language-override|font-kerning|font-feature-settings|font-family|font|float|flex-wrap|flex-shrink|flex-grow|flex-flow|flex-direction|flex-basis|flex|filter|empty-cells|display|direction|cursor|counter-reset|counter-increment|content|column-width|column-span|column-rule-width|column-rule-style|column-rule-color|column-rule|column-gap|column-fill|column-count|columns|color|clip-path|clip|clear|caption-side|break-inside|break-before|break-after|box-sizing|box-shadow|box-decoration-break|bottom|border-width|border-top-width|border-top-style|border-top-right-radius|border-top-left-radius|border-top-color|border-top|border-style|border-spacing|border-right-width|border-right-style|border-right-color|border-right|border-radius|border-left-width|border-left-style|border-left-color|border-left|border-image-width|border-image-source|border-image-slice|border-image-repeat|border-image-outset|border-image|border-color|border-collapse|border-bottom-width|border-bottom-style|border-bottom-right-radius|border-bottom-left-radius|border-bottom-color|border-bottom|border|background-size|background-repeat|background-position|background-origin|background-image|background-color|background-clip|background-attachment|background-blend-mode|background|backface-visibility|auto|animation-timing-function|animation-play-state|animation-name|animation-iteration-count|animation-fill-mode|animation-duration|animation-direction|animation-delay|animation|align-self|align-items|align-content)\\b",illegal:"[^\\s]"},{begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"},{begin:":",end:";",contains:[t,i,e.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,{className:"meta",begin:"!important"}]},{begin:"@(page|font-face)",lexemes:"@[a-z-]+",keywords:"@page @font-face"},{begin:"@",end:"[{;]",returnBegin:!0,keywords:"and or not only",contains:[{begin:"@[a-z-]+",className:"keyword"},t,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,i,e.CSS_NUMBER_MODE]}]}}}()); +hljs.registerLanguage("shell",function(){"use strict";return function(s){return{name:"Shell Session",aliases:["console"],contains:[{className:"meta",begin:"^\\s{0,3}[/\\w\\d\\[\\]()@-]*[>%$#]",starts:{end:"$",subLanguage:"bash"}}]}}}()); +hljs.registerLanguage("sql",function(){"use strict";return function(e){var t=e.COMMENT("--","$");return{name:"SQL",case_insensitive:!0,illegal:/[<>{}*]/,contains:[{beginKeywords:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment values with",end:/;/,endsWithParent:!0,keywords:{$pattern:/[\w\.]+/,keyword:"as abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias all allocate allow alter always analyze ancillary and anti any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound bucket buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain explode export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force foreign form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour hours http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lateral lax lcase lead leading least leaves left len lenght length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minutes minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notnull notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second seconds section securefile security seed segment select self semi sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tablesample tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unnest unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace window with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null unknown",built_in:"array bigint binary bit blob bool boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text time timestamp tinyint varchar varchar2 varying void"},contains:[{className:"string",begin:"'",end:"'",contains:[{begin:"''"}]},{className:"string",begin:'"',end:'"',contains:[{begin:'""'}]},{className:"string",begin:"`",end:"`"},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]},e.C_BLOCK_COMMENT_MODE,t,e.HASH_COMMENT_MODE]}}}()); +hljs.registerLanguage("swift",function(){"use strict";return function(e){var i={keyword:"#available #colorLiteral #column #else #elseif #endif #file #fileLiteral #function #if #imageLiteral #line #selector #sourceLocation _ __COLUMN__ __FILE__ __FUNCTION__ __LINE__ Any as as! as? associatedtype associativity break case catch class continue convenience default defer deinit didSet do dynamic dynamicType else enum extension fallthrough false fileprivate final for func get guard if import in indirect infix init inout internal is lazy left let mutating nil none nonmutating open operator optional override postfix precedence prefix private protocol Protocol public repeat required rethrows return right self Self set static struct subscript super switch throw throws true try try! try? Type typealias unowned var weak where while willSet",literal:"true false nil",built_in:"abs advance alignof alignofValue anyGenerator assert assertionFailure bridgeFromObjectiveC bridgeFromObjectiveCUnconditional bridgeToObjectiveC bridgeToObjectiveCUnconditional c compactMap contains count countElements countLeadingZeros debugPrint debugPrintln distance dropFirst dropLast dump encodeBitsAsWords enumerate equal fatalError filter find getBridgedObjectiveCType getVaList indices insertionSort isBridgedToObjectiveC isBridgedVerbatimToObjectiveC isUniquelyReferenced isUniquelyReferencedNonObjC join lazy lexicographicalCompare map max maxElement min minElement numericCast overlaps partition posix precondition preconditionFailure print println quickSort readLine reduce reflect reinterpretCast reverse roundUpToAlignment sizeof sizeofValue sort split startsWith stride strideof strideofValue swap toString transcode underestimateCount unsafeAddressOf unsafeBitCast unsafeDowncast unsafeUnwrap unsafeReflect withExtendedLifetime withObjectAtPlusZero withUnsafePointer withUnsafePointerToObject withUnsafeMutablePointer withUnsafeMutablePointers withUnsafePointer withUnsafePointers withVaList zip"},n=e.COMMENT("/\\*","\\*/",{contains:["self"]}),t={className:"subst",begin:/\\\(/,end:"\\)",keywords:i,contains:[]},a={className:"string",contains:[e.BACKSLASH_ESCAPE,t],variants:[{begin:/"""/,end:/"""/},{begin:/"/,end:/"/}]},r={className:"number",begin:"\\b([\\d_]+(\\.[\\deE_]+)?|0x[a-fA-F0-9_]+(\\.[a-fA-F0-9p_]+)?|0b[01_]+|0o[0-7_]+)\\b",relevance:0};return t.contains=[r],{name:"Swift",keywords:i,contains:[a,e.C_LINE_COMMENT_MODE,n,{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*[!?]"},{className:"type",begin:"\\b[A-Z][\\wÀ-ʸ']*",relevance:0},r,{className:"function",beginKeywords:"func",end:"{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][0-9A-Za-z$_]*/}),{begin://},{className:"params",begin:/\(/,end:/\)/,endsParent:!0,keywords:i,contains:["self",r,a,e.C_BLOCK_COMMENT_MODE,{begin:":"}],illegal:/["']/}],illegal:/\[|%/},{className:"class",beginKeywords:"struct protocol class extension enum",keywords:i,end:"\\{",excludeEnd:!0,contains:[e.inherit(e.TITLE_MODE,{begin:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/})]},{className:"meta",begin:"(@discardableResult|@warn_unused_result|@exported|@lazy|@noescape|@NSCopying|@NSManaged|@objc|@objcMembers|@convention|@required|@noreturn|@IBAction|@IBDesignable|@IBInspectable|@IBOutlet|@infix|@prefix|@postfix|@autoclosure|@testable|@available|@nonobjc|@NSApplicationMain|@UIApplicationMain|@dynamicMemberLookup|@propertyWrapper)\\b"},{beginKeywords:"import",end:/$/,contains:[e.C_LINE_COMMENT_MODE,n]}]}}}()); +hljs.registerLanguage("typescript",function(){"use strict";const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]);return function(r){var t={$pattern:"[A-Za-z$_][0-9A-Za-z$_]*",keyword:e.concat(["type","namespace","typedef","interface","public","private","protected","implements","declare","abstract","readonly"]).join(" "),literal:n.join(" "),built_in:a.concat(["any","void","number","boolean","string","object","never","enum"]).join(" ")},s={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},i={className:"number",variants:[{begin:"\\b(0[bB][01]+)n?"},{begin:"\\b(0[oO][0-7]+)n?"},{begin:r.C_NUMBER_RE+"n?"}],relevance:0},o={className:"subst",begin:"\\$\\{",end:"\\}",keywords:t,contains:[]},c={begin:"html`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"xml"}},l={begin:"css`",end:"",starts:{end:"`",returnEnd:!1,contains:[r.BACKSLASH_ESCAPE,o],subLanguage:"css"}},E={className:"string",begin:"`",end:"`",contains:[r.BACKSLASH_ESCAPE,o]};o.contains=[r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,i,r.REGEXP_MODE];var d={begin:"\\(",end:/\)/,keywords:t,contains:["self",r.QUOTE_STRING_MODE,r.APOS_STRING_MODE,r.NUMBER_MODE]},u={className:"params",begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,s,d]};return{name:"TypeScript",aliases:["ts"],keywords:t,contains:[r.SHEBANG(),{className:"meta",begin:/^\s*['"]use strict['"]/},r.APOS_STRING_MODE,r.QUOTE_STRING_MODE,c,l,E,r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,i,{begin:"("+r.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[r.C_LINE_COMMENT_MODE,r.C_BLOCK_COMMENT_MODE,r.REGEXP_MODE,{className:"function",begin:"(\\([^(]*(\\([^(]*(\\([^(]*\\))?\\))?\\)|"+r.UNDERSCORE_IDENT_RE+")\\s*=>",returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{begin:r.UNDERSCORE_IDENT_RE},{className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:t,contains:d.contains}]}]}],relevance:0},{className:"function",beginKeywords:"function",end:/[\{;]/,excludeEnd:!0,keywords:t,contains:["self",r.inherit(r.TITLE_MODE,{begin:"[A-Za-z$_][0-9A-Za-z$_]*"}),u],illegal:/%/,relevance:0},{beginKeywords:"constructor",end:/[\{;]/,excludeEnd:!0,contains:["self",u]},{begin:/module\./,keywords:{built_in:"module"},relevance:0},{beginKeywords:"module",end:/\{/,excludeEnd:!0},{beginKeywords:"interface",end:/\{/,excludeEnd:!0,keywords:"interface extends"},{begin:/\$[(.]/},{begin:"\\."+r.IDENT_RE,relevance:0},s,d]}}}()); +hljs.registerLanguage("yaml",function(){"use strict";return function(e){var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*\\'()[\\]]+",s={className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",variants:[{begin:"{{",end:"}}"},{begin:"%{",end:"}"}]}]},i=e.inherit(s,{variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={end:",",endsWithParent:!0,excludeEnd:!0,contains:[],keywords:n,relevance:0},t={begin:"{",end:"}",contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]",contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---s*$",relevance:10},{className:"string",begin:"[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*"},{begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type",begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"\\-(?=[ ]|$)",relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{className:"number",begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"},{className:"number",begin:e.C_NUMBER_RE+"\\b"},t,g,s],c=[...b];return c.pop(),c.push(i),l.contains=c,{name:"YAML",case_insensitive:!0,aliases:["yml","YAML"],contains:b}}}()); \ No newline at end of file diff --git a/presto-ui/src/static/vendor/highlightjs/10.1.2/styles/solarized-dark.min.css b/presto-ui/src/static/vendor/highlightjs/10.1.2/styles/solarized-dark.min.css new file mode 100644 index 0000000000000..9e8e2b9131890 --- /dev/null +++ b/presto-ui/src/static/vendor/highlightjs/10.1.2/styles/solarized-dark.min.css @@ -0,0 +1,72 @@ +.hljs { + display: block; + overflow-x: auto; + padding: .5em; + background: #002b36; + color: #839496 +} + +.hljs-comment, +.hljs-quote { + color: #586e75 +} + +.hljs-addition, +.hljs-keyword, +.hljs-selector-tag { + color: #859900 +} + +.hljs-doctag, +.hljs-literal, +.hljs-meta .hljs-meta-string, +.hljs-number, +.hljs-regexp, +.hljs-string { + color: #2aa198 +} + +.hljs-name, +.hljs-section, +.hljs-selector-class, +.hljs-selector-id, +.hljs-title { + color: #268bd2 +} + +.hljs-attr, +.hljs-attribute, +.hljs-class .hljs-title, +.hljs-template-variable, +.hljs-type, +.hljs-variable { + color: #b58900 +} + +.hljs-bullet, +.hljs-link, +.hljs-meta, +.hljs-meta .hljs-keyword, +.hljs-selector-attr, +.hljs-selector-pseudo, +.hljs-subst, +.hljs-symbol { + color: #cb4b16 +} + +.hljs-built_in, +.hljs-deletion { + color: #dc322f +} + +.hljs-formula { + background: #073642 +} + +.hljs-emphasis { + font-style: italic +} + +.hljs-strong { + font-weight: 700 +} \ No newline at end of file diff --git a/presto-ui/src/templates/query_viewer.html b/presto-ui/src/templates/query_viewer.html index 20a55a1cc79ad..9b43341c2e0e6 100644 --- a/presto-ui/src/templates/query_viewer.html +++ b/presto-ui/src/templates/query_viewer.html @@ -43,23 +43,23 @@