diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000000..cb828eabaed39 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:24.04 + +# Install necessary tools +RUN apt-get update && apt-get install -y \ + # Intellij IDEA dev container prerequisites + curl \ + git \ + unzip \ + # Java 8 and 17 jdk + openjdk-8-jdk \ + openjdk-17-jdk \ + # for documentation + python3 \ + python3-pip \ + python3.12-venv \ + fonts-freefont-otf \ + xindy + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000000..12ef523071cec --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,6 @@ +{ + "name": "PrestoDB Dev Container", + "build": { + "dockerfile": "Dockerfile" + } +} diff --git a/.github/actions/maven-owasp-scan/action.yml b/.github/actions/maven-owasp-scan/action.yml new file mode 100644 index 0000000000000..9681434fad2d6 --- /dev/null +++ b/.github/actions/maven-owasp-scan/action.yml @@ -0,0 +1,35 @@ +name: 'Maven OWASP Dependency Check Scan' +description: 'Runs OWASP dependency check Maven scan with consistent settings' +inputs: + working-directory: + description: 'Working directory for Maven command' + required: false + default: '.' + owasp-version: + description: 'OWASP dependency check plugin version' + required: false + default: '12.1.3' + data-directory: + description: 'OWASP data directory path' + required: false + default: '$HOME/.owasp/dependency-check-data' +runs: + using: 'composite' + steps: + - name: Run OWASP dependency check + working-directory: ${{ inputs.working-directory }} + shell: bash + run: | + mvn org.owasp:dependency-check-maven:${{ inputs.owasp-version }}:aggregate \ + -DskipTests \ + -Dformat=JSON \ + -DprettyPrint=true \ + -DfailOnError=false \ + -DossindexAnalyzerEnabled=true \ + -DnvdApiAnalyzerEnabled=false \ + -DnodeAnalyzerEnabled=false \ + -DassemblyAnalyzerEnabled=false \ + -DcentralAnalyzerEnabled=false \ + -DnuspecAnalyzerEnabled=false \ + -DnvdValidForHours=168 \ + -DdataDirectory=${{ inputs.data-directory }} \ No newline at end of file diff --git a/.github/workflows/conventional-commit-check.yml b/.github/workflows/conventional-commit-check.yml new file mode 100644 index 0000000000000..c7f2b9e15741f --- /dev/null +++ b/.github/workflows/conventional-commit-check.yml @@ -0,0 +1,41 @@ +name: Semantic Commit Check + +on: + pull_request: + types: + - opened + - reopened + - edited + - synchronize + +permissions: + contents: read + +jobs: + semantic-pr: + concurrency: + group: ${{ github.workflow }}-semantic-commit-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + name: Validate PR Title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@2d952a1bf90a6a7ab8f0293dc86f5fdf9acb1915 # v5.5.3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + types: | + feat + fix + docs + refactor + perf + test + build + ci + chore + revert + misc + requireScope: false + subjectPattern: ^[A-Z].*[^.]$ + subjectPatternError: | + The PR title subject must start with a capital letter and not end with a period. diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index acfcc41efdbc3..2b5f35eefe920 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -23,10 +23,10 @@ jobs: cancel-in-progress: true steps: - name: Free Disk Space - run: | - df -h - sudo apt-get clean - df -h + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + swap-storage: false - uses: actions/checkout@v4 with: show-progress: false diff --git a/.github/workflows/owasp-dependency-check.yml b/.github/workflows/owasp-dependency-check.yml new file mode 100644 index 0000000000000..c2834b0a31d5c --- /dev/null +++ b/.github/workflows/owasp-dependency-check.yml @@ -0,0 +1,152 @@ +name: Maven OWASP Dependency Check +permissions: + contents: read +on: + pull_request: + workflow_dispatch: + inputs: + cvss-threshold: + description: 'CVSS score threshold for failing (7.0 = high/critical)' + required: false + default: '7.0' + type: string + +jobs: + dependency-check: + runs-on: ubuntu-latest + env: + CVSS_THRESHOLD: ${{ github.event.inputs.cvss-threshold || '7.0' }} + OWASP_VERSION: '12.1.3' + steps: + # Checkout PR branch first to get access to the composite action + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + cache: 'maven' + + - name: Get date for cache key + id: get-date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Restore OWASP database cache + uses: actions/cache/restore@v4 + id: cache-owasp-restore + with: + path: ~/.owasp/dependency-check-data + key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }} + restore-keys: | + owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}- + owasp-cache-${{ runner.os }}- + + - name: Run OWASP check on base branch + uses: ./.github/actions/maven-owasp-scan + with: + working-directory: base + owasp-version: ${{ env.OWASP_VERSION }} + data-directory: $HOME/.owasp/dependency-check-data + + - name: Save OWASP cache after base scan + if: steps.cache-owasp-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ~/.owasp/dependency-check-data + key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }}-partial + + - name: Run OWASP check on PR branch + uses: ./.github/actions/maven-owasp-scan + with: + working-directory: . + owasp-version: ${{ env.OWASP_VERSION }} + data-directory: $HOME/.owasp/dependency-check-data + + - name: Compare and fail on new CVEs above threshold + run: | + # Extract CVEs above threshold from both branches (CVSS >= $CVSS_THRESHOLD) + threshold="${{ env.CVSS_THRESHOLD }}" + + # Validate report files exist + if [ ! -f base/target/dependency-check-report.json ]; then + echo "❌ Missing base report: base/target/dependency-check-report.json" + exit 1 + fi + if [ ! -f target/dependency-check-report.json ]; then + echo "❌ Missing PR report: target/dependency-check-report.json" + exit 1 + fi + + # Validate report files are valid JSON + if ! jq empty base/target/dependency-check-report.json >/dev/null 2>&1; then + echo "❌ Malformed JSON in base/target/dependency-check-report.json" + exit 1 + fi + if ! jq empty target/dependency-check-report.json >/dev/null 2>&1; then + echo "❌ Malformed JSON in target/dependency-check-report.json" + exit 1 + fi + + base_cves=$(jq -r ".dependencies[].vulnerabilities[]? | select((.cvssv2.score // 0) >= $threshold or (.cvssv3.baseScore // 0) >= $threshold) | .name" base/target/dependency-check-report.json | grep -E '^CVE-[0-9]{4}-[0-9]+$' | sort -u) + pr_cves=$(jq -r ".dependencies[].vulnerabilities[]? | select((.cvssv2.score // 0) >= $threshold or (.cvssv3.baseScore // 0) >= $threshold) | .name" target/dependency-check-report.json | grep -E '^CVE-[0-9]{4}-[0-9]+$' | sort -u) + + # Find new CVEs introduced in PR + new_cves=$(comm -13 <(echo "$base_cves") <(echo "$pr_cves")) + + if [ -n "$new_cves" ]; then + echo "❌ New vulnerabilities with CVSS >= $threshold introduced in PR:" + echo "$new_cves" + echo "" + + for cve in $new_cves; do + echo "==================================================" + echo "CVE: $cve" + echo "==================================================" + + # Find which dependencies have this CVE + jq -r ' + .dependencies[] + | select(.vulnerabilities[]?.name == "'"$cve"'") + | "Module: " + (.projectReferences // ["root"])[0] + + "\nDependency: " + .fileName + + "\nPackage: " + (if .packages and .packages[0] then .packages[0].id else "N/A" end) + + "\nDescription: " + ( + [.vulnerabilities[] | select(.name == "'"$cve"'") | .description] + | unique + | join("\nDescription: ") + ) + ' target/dependency-check-report.json + + echo "" + done + + exit 1 + else + echo "✅ No new vulnerabilities introduced" + fi + + - name: Save OWASP database cache + if: always() + uses: actions/cache/save@v4 + with: + path: ~/.owasp/dependency-check-data + key: owasp-cache-${{ runner.os }}-v${{ env.OWASP_VERSION }}-${{ steps.get-date.outputs.date }} + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: owasp-reports + path: | + base/target/dependency-check-report.json + target/dependency-check-report.json diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml index 67a2744b7d140..554daeee95254 100644 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -79,6 +79,8 @@ jobs: github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' run: | source /opt/rh/gcc-toolset-12/enable + cp -r presto-native-tests/presto_cpp/tests presto-native-execution/presto_cpp/main/functions/dynamic_registry + echo "add_subdirectory(tests)" >> presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt cd presto-native-execution cmake \ -B _build/release \ @@ -126,6 +128,7 @@ jobs: path: | presto-native-execution/_build/release/presto_cpp/main/presto_server presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + presto-native-execution/_build/release/presto_cpp/main/functions/dynamic_registry/tests/ prestocpp-linux-presto-e2e-tests: needs: [changes, prestocpp-linux-build-for-test] @@ -328,13 +331,19 @@ jobs: MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" steps: - uses: actions/checkout@v4 + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' - name: Fix git permissions + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' # Usually actions/checkout does this but as we run in a container # it doesn't work run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Download artifacts + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/download-artifact@v4 with: name: presto-native-build @@ -342,6 +351,8 @@ jobs: # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - name: Restore execute permissions and library path + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' run: | chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main @@ -349,15 +360,21 @@ jobs: ldconfig /usr/local/lib - name: Install OpenJDK17 + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17.0.15 cache: 'maven' - name: Download nodejs to maven cache + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs - name: Maven install + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" @@ -365,6 +382,8 @@ jobs: for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) - name: Run presto-on-spark native tests + if: | + github.event_name == 'schedule' || needs.changes.outputs.codechange == 'true' run: | export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'` diff --git a/.github/workflows/product-tests-specific-environment.yml b/.github/workflows/product-tests-specific-environment.yml index 8b5259f00bb32..c40a506f739aa 100644 --- a/.github/workflows/product-tests-specific-environment.yml +++ b/.github/workflows/product-tests-specific-environment.yml @@ -42,11 +42,9 @@ jobs: steps: - name: Free Disk Space if: needs.changes.outputs.codechange == 'true' - run: | - df -h - sudo apt-get clean - rm -rf /opt/hostedtoolcache - df -h + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + docker-images: false - uses: actions/checkout@v4 if: needs.changes.outputs.codechange == 'true' with: @@ -108,11 +106,9 @@ jobs: steps: - name: Free Disk Space if: needs.changes.outputs.codechange == 'true' - run: | - df -h - sudo apt-get clean - rm -rf /opt/hostedtoolcache - df -h + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + docker-images: false - uses: actions/checkout@v4 if: needs.changes.outputs.codechange == 'true' with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1bdc07fca4f75..f0c4f4e57cc47 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: - ":presto-main-base" - ":presto-main" - ":presto-mongodb -P ci-full-tests" - - ":presto-redis -P test-redis-integration-smoke-test" + - ":presto-redis -P ci-full-tests" - ":presto-elasticsearch" - ":presto-orc" - ":presto-thrift-connector" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32a61b22ccef3..b3712f244a293 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,17 @@ To commit code, you should: * Add or modify existing tests related to code changes being submitted * Run and ensure that local tests pass before submitting a merge request +## Release Process and Version Support + +* **Release Cadence**: ~2 months (volunteer dependent) +* **Version Support**: Latest release plus N-1 through N-4 receive critical fixes; N-5+ unsupported +* **Trunk**: Not stable - never use in production +* **Testing**: Extended RC periods with community testing + +Details: +* [Release Process Documentation](presto-docs/src/main/sphinx/develop/release-process.rst) - For developers +* [Version Support Guide](presto-docs/src/main/sphinx/admin/version-support.rst) - For administrators + ## Designing Your Code * Consider your code through 3 axes 1. Code Quality and Maintainability, for example: @@ -299,42 +310,99 @@ We recommend you use IntelliJ as your IDE. The code style template for the proje ## Commit Standards -* ### Commit Size - * Recommended lines of code should be no more than +1000 lines, and should focus on one single major topic.\ - If your commit is more than 1000 lines, consider breaking it down into multiple commits, each handling a specific small topic. -* ### Commit Message Style - * **Separate subject from body with a blank line** - * **Subject** - * Limit the subject line to 10 words or 50 characters - * If you cannot make the subject short, you may be committing too many changes at once - * Capitalize the subject line - * Do not end the subject line with a period - * Use the imperative mood in the subject line - * **Body** - * Wrap the body at 72 characters - * Use the body to explain what and why versus how - * Use the indicative mood in the body\ - For example, “If applied, this commit will ___________” - * Communicate only context (why) for the commit in the subject line - * Use the body for What and Why - * If your commit is complex or dense, share some of the How context - * Assume someone may need to revert your change during an emergency - * **Content** - * **Aim for smaller commits for easier review and simpler code maintenance** - * All bug fixes and new features must have associated tests - * Commits should pass tests on their own, not be dependent on other commits - * The following is recommended: - * Describe why a change is being made. - * How does it address the issue? - * What effects does the patch have? - * Do not assume the reviewer understands what the original problem was. - * Do not assume the code is self-evident or self-documenting. - * Read the commit message to see if it hints at improved code structure. - * The first commit line is the most important. - * Describe any limitations of the current code. - * Do not include patch set-specific comments. - -Details for each point and good commit message examples can be found on https://wiki.openstack.org/wiki/GitCommitMessages#Information_in_commit_messages + +### Conventional Commits +We follow the [Conventional Commits](https://www.conventionalcommits.org/) specification for our commit messages and PR titles. + +**PR Title Format:** +``` +[(scope)]: +``` + +**Types:** Defined in [.github/workflows/conventional-commit-check.yml](.github/workflows/conventional-commit-check.yml): +* **feat**: New feature or functionality +* **fix**: Bug fix +* **docs**: Documentation only changes +* **refactor**: Code refactoring without changing functionality +* **perf**: Performance improvements +* **test**: Adding or modifying tests +* **build**: Build system or dependency changes +* **ci**: CI/CD configuration changes +* **chore**: Maintenance tasks +* **revert**: Reverting a previous commit +* **misc**: Miscellaneous changes + +Note: Each PR/commit should have a single primary type. If your changes span multiple categories, choose the most significant one or consider splitting into separate PRs. + +**Scope (optional):** The area of code affected. Common scopes include: + +* `parser` - SQL parser and grammar +* `analyzer` - Query analysis and validation +* `planner` - Query planning, optimization, and rules (including CBO) +* `spi` - Service Provider Interface changes +* `scheduler` - Task scheduling and execution +* `protocol` - Wire protocol and serialization +* `connector` - Changes to connector functionality +* `resource` - Resource management (memory manager, resource groups) +* `security` - Authentication and authorization +* `function` - Built-in functions and operators +* `type` - Type system and type coercion +* `expression` - Expression evaluation +* `operator` - Query operators (join, aggregation, etc.) +* `client` - Client libraries and protocols +* `server` - Server configuration and management +* `native` - Native execution engine +* `testing` - Test framework and utilities +* `docs` - Documentation +* `build` - Build system and dependencies + +Additionally, any connector name (e.g., `hive`, `iceberg`, `delta`, `kafka`) or plugin name (e.g., `session-property-manager`, `access-control`, `event-listener`) can be used as a scope. + +**Description:** +* Must start with a capital letter +* Must not end with a period +* Use imperative mood ("Add feature" not "Added feature") +* Be concise but descriptive + +**Breaking Changes:** +* Use `!` after the type/scope (e.g., `feat!: Remove deprecated API`) +* AND include `BREAKING CHANGE:` in the commit description footer with a detailed explanation of the change +* Use to indicate any change that is not backward compatible as defined in the [Backward Compatibility Guidelines](presto-docs/src/main/sphinx/develop/release-process.rst#backward-compatibility-guidelines) + +**Examples:** +* `feat(connector): Add support for dynamic catalog registration` +* `fix: Resolve memory leak in query executor` +* `docs(api): Update REST API documentation` +* `feat!: Remove deprecated configuration options` (breaking change) + +### Single Commit PRs +* **All PRs must be merged as a single commit** using GitHub's "Squash and merge" feature +* The PR title will become the commit message, so it must follow the conventional commit format +* Multiple commits within a PR are allowed during development for easier review, but they will be squashed on merge +* If you need to reference other commits or PRs, include them in the PR description or commit body, not as separate commits + +### Commit Message Guidelines +* **PR Title/First Line** + * Must follow conventional commit format + * Limit to 50-72 characters when possible + * If you cannot make it concise, you may be changing too much at once + +* **PR Description/Commit Body** + * Separate from title with a blank line + * Wrap at 72 characters + * Explain what and why, not how + * Include: + * Why the change is being made + * What issue it addresses + * Any side effects or limitations + * Breaking changes or migration notes if applicable + * Assume someone may need to revert your change during an emergency + +* **Content Requirements** + * All bug fixes and new features must have associated tests + * Changes should be focused on a single topic + * Code should pass all tests independently + * Include documentation updates with code changes * **Metadata** * If the commit was to solve a Github issue, refer to it at the end of a commit message in a rfc822 header line format.\ @@ -449,20 +517,13 @@ We use the [Fork and Pull model](https://docs.github.com/en/pull-requests/collab - Make sure your code follows the [code style guidelines](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#code-style), [development guidelines](https://github.com/prestodb/presto/wiki/Presto-Development-Guidelines#development) and [formatting guidelines](https://github.com/prestodb/presto/wiki/Presto-Development-Guidelines#formatting) -- Make sure you follow the [review and commit guidelines](https://github.com/prestodb/presto/wiki/Review-and-Commit-guidelines), in particular: - - - Ensure that each commit is correct independently. Each commit should compile and pass tests. - - When possible, reduce the size of the commit for ease of review. Consider breaking a large PR into multiple commits, with each one addressing a particular issue. For example, if you are introducing a new feature that requires certain refactor, making a separate refactor commit before the real change helps the reviewer to isolate the changes. - - Do not send commits like addressing comments or fixing tests for previous commits in the same PR. Squash such commits to its corresponding base commit before the PR is rebased and merged. - - Make sure commit messages [follow these guidelines](https://chris.beams.io/posts/git-commit/). In particular (from the guidelines): +- Make sure you follow the [Commit Standards](#commit-standards) section above, which uses Conventional Commits format: - * Separate subject from body with a blank line - * Limit the subject line to 50 characters - * Capitalize the subject line - * Do not end the subject line with a period - * Use the imperative mood in the subject line - * Wrap the body at 72 characters - * Use the body to explain what and why vs. how + - PR titles must follow the conventional commit format (e.g., `feat: Add new feature`, `fix: Resolve bug`) + - All PRs will be squashed into a single commit on merge, so the PR title becomes the commit message + - While developing, you can have multiple commits in your PR for easier review + - Ensure each commit in your PR compiles and passes tests independently + - The PR description should explain what and why, not how. Keep lines wrapped at 72 characters for better readability. Include context about why the change is needed, what issue it addresses, any side effects or breaking changes, and enough detail that someone could understand whether to revert it during an emergency. - Ensure all code is peer reviewed within your own organization or peers before submitting - Implement and address existing feedback before requesting further review - Make a good faith effort to locate example or referential code before requesting someone else direct you towards it diff --git a/pom.xml b/pom.xml index 49eaf516e1042..045a68efee7f1 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT pom presto-root @@ -1472,7 +1472,7 @@ org.jetbrains annotations - 19.0.0 + 26.0.2 diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 5a3ff65981923..0b21198df50bf 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-accumulo @@ -354,7 +354,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-analyzer/pom.xml b/presto-analyzer/pom.xml index e1c2ff5bf062c..907dea53fea99 100644 --- a/presto-analyzer/pom.xml +++ b/presto-analyzer/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-analyzer diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java index 7da15ef222470..0d7c27769823b 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Analysis.java @@ -49,6 +49,7 @@ import com.facebook.presto.sql.tree.Offset; import com.facebook.presto.sql.tree.OrderBy; import com.facebook.presto.sql.tree.Parameter; +import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.QuantifiedComparisonExpression; import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.QuerySpecification; @@ -214,7 +215,11 @@ public class Analysis private Optional> updatedSourceColumns = Optional.empty(); + // names of tables and aliased relations. All names are resolved case-insensitive. + private final Map, QualifiedName> relationNames = new LinkedHashMap<>(); private final Map, TableFunctionInvocationAnalysis> tableFunctionAnalyses = new LinkedHashMap<>(); + private final Set> aliasedRelations = new LinkedHashSet<>(); + private final Set> polymorphicTableFunctions = new LinkedHashSet<>(); public Analysis(@Nullable Statement root, Map, Expression> parameters, boolean isDescribe) { @@ -1120,6 +1125,36 @@ public TableFunctionInvocationAnalysis getTableFunctionAnalysis(TableFunctionInv return tableFunctionAnalyses.get(NodeRef.of(node)); } + public void setRelationName(Relation relation, QualifiedName name) + { + relationNames.put(NodeRef.of(relation), name); + } + + public QualifiedName getRelationName(Relation relation) + { + return relationNames.get(NodeRef.of(relation)); + } + + public void addAliased(Relation relation) + { + aliasedRelations.add(NodeRef.of(relation)); + } + + public boolean isAliased(Relation relation) + { + return aliasedRelations.contains(NodeRef.of(relation)); + } + + public void addPolymorphicTableFunction(TableFunctionInvocation invocation) + { + polymorphicTableFunctions.add(NodeRef.of(invocation)); + } + + public boolean isPolymorphicTableFunction(TableFunctionInvocation invocation) + { + return polymorphicTableFunctions.contains(NodeRef.of(invocation)); + } + @Immutable public static final class Insert { @@ -1371,16 +1406,218 @@ public int hashCode() } } + public static class TableArgumentAnalysis + { + private final String argumentName; + private final Optional name; + private final Relation relation; + private final Optional> partitionBy; // it is allowed to partition by empty list + private final Optional orderBy; + private final boolean pruneWhenEmpty; + private final boolean rowSemantics; + private final boolean passThroughColumns; + + private TableArgumentAnalysis( + String argumentName, + Optional name, + Relation relation, + Optional> partitionBy, + Optional orderBy, + boolean pruneWhenEmpty, + boolean rowSemantics, + boolean passThroughColumns) + { + this.argumentName = requireNonNull(argumentName, "argumentName is null"); + this.name = requireNonNull(name, "name is null"); + this.relation = requireNonNull(relation, "relation is null"); + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null").map(ImmutableList::copyOf); + this.orderBy = requireNonNull(orderBy, "orderBy is null"); + this.pruneWhenEmpty = pruneWhenEmpty; + this.rowSemantics = rowSemantics; + this.passThroughColumns = passThroughColumns; + } + + public String getArgumentName() + { + return argumentName; + } + + public Optional getName() + { + return name; + } + + public Relation getRelation() + { + return relation; + } + + public Optional> getPartitionBy() + { + return partitionBy; + } + + public Optional getOrderBy() + { + return orderBy; + } + + public boolean isPruneWhenEmpty() + { + return pruneWhenEmpty; + } + + public boolean isRowSemantics() + { + return rowSemantics; + } + + public boolean isPassThroughColumns() + { + return passThroughColumns; + } + + public static Builder builder() + { + return new Builder(); + } + + public static final class Builder + { + private String argumentName; + private Optional name = Optional.empty(); + private Relation relation; + private Optional> partitionBy = Optional.empty(); + private Optional orderBy = Optional.empty(); + private boolean pruneWhenEmpty; + private boolean rowSemantics; + private boolean passThroughColumns; + + private Builder() {} + + public Builder withArgumentName(String argumentName) + { + this.argumentName = argumentName; + return this; + } + + public Builder withName(QualifiedName name) + { + this.name = Optional.of(name); + return this; + } + + public Builder withRelation(Relation relation) + { + this.relation = relation; + return this; + } + + public Builder withPartitionBy(List partitionBy) + { + this.partitionBy = Optional.of(partitionBy); + return this; + } + + public Builder withOrderBy(OrderBy orderBy) + { + this.orderBy = Optional.of(orderBy); + return this; + } + + public Builder withPruneWhenEmpty(boolean pruneWhenEmpty) + { + this.pruneWhenEmpty = pruneWhenEmpty; + return this; + } + + public Builder withRowSemantics(boolean rowSemantics) + { + this.rowSemantics = rowSemantics; + return this; + } + + public Builder withPassThroughColumns(boolean passThroughColumns) + { + this.passThroughColumns = passThroughColumns; + return this; + } + + public TableArgumentAnalysis build() + { + return new TableArgumentAnalysis(argumentName, name, relation, partitionBy, orderBy, pruneWhenEmpty, rowSemantics, passThroughColumns); + } + } + } + /** * Encapsulates the result of analyzing a table function invocation. * Includes the connector ID, function name, argument bindings, and the * connector-specific table function handle needed for planning and execution. + * + * Example of a TableFunctionInvocationAnalysis for a table function + * with two table arguments, required columns, and co-partitioning + * implemented by {@link TestingTableFunctions.TwoTableArgumentsFunction} + * + * SQL: + * SELECT * FROM TABLE(system.two_table_arguments_function( + * input1 => TABLE(t1) PARTITION BY (a, b), + * input2 => TABLE(SELECT 1, 2) t1(x, y) PARTITION BY (x, y) + * COPARTITION(t1, s1.t1))) + * + * Table Function: + * super( + * SCHEMA_NAME, + * "two_table_arguments_function", + * ImmutableList.of( + * TableArgumentSpecification.builder() + * .name("INPUT1") + * .build(), + * TableArgumentSpecification.builder() + * .name("INPUT2") + * .build()), + * GENERIC_TABLE); + * + * analyze: + * return TableFunctionAnalysis.builder() + * .handle(HANDLE) + * .returnedType(new Descriptor(ImmutableList.of(new Descriptor.Field(COLUMN_NAME, Optional.of(BOOLEAN))))) + * .requiredColumns("INPUT1", ImmutableList.of(0)) + * .requiredColumns("INPUT2", ImmutableList.of(0)) + * .build(); + * + * Example values: + * connectorId = "tpch" + * functionName = "two_table_arguments_function" + * arguments = { + * "input1" -> row(a bigint,b bigint,c bigint,d bigint), + * "input2" -> row(x integer,y integer) + * } + * tableArgumentAnalyses = [ + * { argumentName="INPUT1", relation=Table{t1}, partitionBy=[a, b], orderBy=[], pruneWhenEmpty=false }, + * { argumentName="INPUT2", relation=AliasedRelation{SELECT 1, 2 AS x, y}, partitionBy=[x, y], orderBy=[], pruneWhenEmpty=false } + * ] + * requiredColumns = { + * "INPUT1" -> [0], + * "INPUT2" -> [0] + * } + * copartitioningLists = [ + * ["INPUT2", "INPUT1"] + * ] + * properColumnsCount = 1 + * connectorTableFunctionHandle = TestingTableFunctionPushdownHandle + * transactionHandle = AbstractAnalyzerTest$1$1 + * */ public static class TableFunctionInvocationAnalysis { private final ConnectorId connectorId; private final String functionName; private final Map arguments; + private final List tableArgumentAnalyses; + private final Map> requiredColumns; + private final List> copartitioningLists; + private final int properColumnsCount; private final ConnectorTableFunctionHandle connectorTableFunctionHandle; private final ConnectorTransactionHandle transactionHandle; @@ -1388,6 +1625,10 @@ public TableFunctionInvocationAnalysis( ConnectorId connectorId, String functionName, Map arguments, + List tableArgumentAnalyses, + Map> requiredColumns, + List> copartitioningLists, + int properColumnsCount, ConnectorTableFunctionHandle connectorTableFunctionHandle, ConnectorTransactionHandle transactionHandle) { @@ -1396,6 +1637,11 @@ public TableFunctionInvocationAnalysis( this.arguments = ImmutableMap.copyOf(arguments); this.connectorTableFunctionHandle = requireNonNull(connectorTableFunctionHandle, "connectorTableFunctionHandle is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); + this.tableArgumentAnalyses = ImmutableList.copyOf(tableArgumentAnalyses); + this.requiredColumns = requiredColumns.entrySet().stream() + .collect(toImmutableMap(Map.Entry::getKey, entry -> ImmutableList.copyOf(entry.getValue()))); + this.copartitioningLists = ImmutableList.copyOf(copartitioningLists); + this.properColumnsCount = properColumnsCount; } public ConnectorId getConnectorId() @@ -1413,6 +1659,31 @@ public Map getArguments() return arguments; } + public List getTableArgumentAnalyses() + { + return tableArgumentAnalyses; + } + + public Map> getRequiredColumns() + { + return requiredColumns; + } + + public List> getCopartitioningLists() + { + return copartitioningLists; + } + + /** + * Proper columns are the columns produced by the table function, as opposed to pass-through columns from input tables. + * Proper columns should be considered the actual result of the table function. + * @return the number of table function's proper columns + */ + public int getProperColumnsCount() + { + return properColumnsCount; + } + public ConnectorTableFunctionHandle getConnectorTableFunctionHandle() { return connectorTableFunctionHandle; diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java index 5c9be03540f67..80eef0f465959 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/SemanticErrorCode.java @@ -57,14 +57,6 @@ public enum SemanticErrorCode INVALID_FUNCTION_NAME, DUPLICATE_PARAMETER_NAME, EXCEPTIONS_WHEN_RESOLVING_FUNCTIONS, - - MISSING_RETURN_TYPE, - AMBIGUOUS_RETURN_TYPE, - MISSING_ARGUMENT, - INVALID_FUNCTION_ARGUMENT, - INVALID_ARGUMENTS, - NOT_IMPLEMENTED, - ORDER_BY_MUST_BE_IN_SELECT, ORDER_BY_MUST_BE_IN_AGGREGATE, REFERENCE_TO_OUTPUT_ATTRIBUTE_WITHIN_ORDER_BY_GROUPING, @@ -119,4 +111,16 @@ public enum SemanticErrorCode TOO_MANY_GROUPING_SETS, INVALID_OFFSET_ROW_COUNT, + + TABLE_FUNCTION_MISSING_RETURN_TYPE, + TABLE_FUNCTION_AMBIGUOUS_RETURN_TYPE, + TABLE_FUNCTION_MISSING_ARGUMENT, + TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + TABLE_FUNCTION_INVALID_ARGUMENTS, + TABLE_FUNCTION_IMPLEMENTATION_ERROR, + TABLE_FUNCTION_INVALID_COPARTITIONING, + TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, + TABLE_FUNCTION_DUPLICATE_RANGE_VARIABLE, + TABLE_FUNCTION_INVALID_COLUMN_REFERENCE, + TABLE_FUNCTION_COLUMN_NOT_FOUND } diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java index fa84d900eef57..9f29effad10ec 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java @@ -45,6 +45,7 @@ import com.facebook.presto.sql.tree.Grant; import com.facebook.presto.sql.tree.GrantRoles; import com.facebook.presto.sql.tree.Insert; +import com.facebook.presto.sql.tree.Merge; import com.facebook.presto.sql.tree.Prepare; import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.RefreshMaterializedView; @@ -105,6 +106,7 @@ private StatementUtils() {} builder.put(Delete.class, QueryType.DELETE); builder.put(Update.class, QueryType.UPDATE); + builder.put(Merge.class, QueryType.MERGE); builder.put(ShowCatalogs.class, QueryType.DESCRIBE); builder.put(ShowCreate.class, QueryType.DESCRIBE); diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index d686e4b8d5db6..87a817f07919a 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-atop diff --git a/presto-base-arrow-flight/pom.xml b/presto-base-arrow-flight/pom.xml index 9f38ba2ec0ed1..9457fb5006e91 100644 --- a/presto-base-arrow-flight/pom.xml +++ b/presto-base-arrow-flight/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-base-arrow-flight diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 79a951187f9ee..0cefe58d78065 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-base-jdbc diff --git a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java index 7f7bb084b60e2..63f2f15e1ed7d 100644 --- a/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java +++ b/presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/BaseJdbcClient.java @@ -370,7 +370,7 @@ protected JdbcOutputTableHandle createTable(ConnectorTableMetadata tableMetadata boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers(); String remoteSchema = toRemoteSchemaName(session, identity, connection, schemaTableName.getSchemaName()); String remoteTable = toRemoteTableName(session, identity, connection, remoteSchema, schemaTableName.getTableName()); - if (uppercase) { + if (uppercase && !caseSensitiveNameMatchingEnabled) { tableName = tableName.toUpperCase(ENGLISH); } String catalog = connection.getCatalog(); @@ -380,7 +380,7 @@ protected JdbcOutputTableHandle createTable(ConnectorTableMetadata tableMetadata ImmutableList.Builder columnList = ImmutableList.builder(); for (ColumnMetadata column : tableMetadata.getColumns()) { String columnName = column.getName(); - if (uppercase) { + if (uppercase && !caseSensitiveNameMatchingEnabled) { columnName = columnName.toUpperCase(ENGLISH); } columnNames.add(columnName); @@ -447,7 +447,7 @@ protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTabl String tableName = oldTable.getTableName(); String newSchemaName = newTable.getSchemaName(); String newTableName = newTable.getTableName(); - if (metadata.storesUpperCaseIdentifiers()) { + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { schemaName = schemaName.toUpperCase(ENGLISH); tableName = tableName.toUpperCase(ENGLISH); newSchemaName = newSchemaName.toUpperCase(ENGLISH); @@ -495,7 +495,7 @@ public void addColumn(ConnectorSession session, JdbcIdentity identity, JdbcTable String table = handle.getTableName(); String columnName = column.getName(); DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers()) { + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { schema = schema != null ? schema.toUpperCase(ENGLISH) : null; table = table.toUpperCase(ENGLISH); columnName = columnName.toUpperCase(ENGLISH); @@ -516,14 +516,20 @@ public void renameColumn(ConnectorSession session, JdbcIdentity identity, JdbcTa { try (Connection connection = connectionFactory.openConnection(identity)) { DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers()) { + String schema = handle.getSchemaName(); + String table = handle.getTableName(); + String columnName = jdbcColumn.getColumnName(); + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { + schema = schema != null ? schema.toUpperCase(ENGLISH) : null; + table = table.toUpperCase(ENGLISH); + columnName = columnName.toUpperCase(ENGLISH); newColumnName = newColumnName.toUpperCase(ENGLISH); } String sql = format( "ALTER TABLE %s RENAME COLUMN %s TO %s", - quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()), - jdbcColumn.getColumnName(), - newColumnName); + quoted(handle.getCatalogName(), schema, table), + quoted(columnName), + quoted(newColumnName)); execute(connection, sql); } catch (SQLException e) { @@ -535,10 +541,19 @@ public void renameColumn(ConnectorSession session, JdbcIdentity identity, JdbcTa public void dropColumn(ConnectorSession session, JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column) { try (Connection connection = connectionFactory.openConnection(identity)) { + DatabaseMetaData metadata = connection.getMetaData(); + String schema = handle.getSchemaName(); + String table = handle.getTableName(); + String columnName = column.getColumnName(); + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { + schema = schema != null ? schema.toUpperCase(ENGLISH) : null; + table = table.toUpperCase(ENGLISH); + columnName = columnName.toUpperCase(ENGLISH); + } String sql = format( "ALTER TABLE %s DROP COLUMN %s", - quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()), - column.getColumnName()); + quoted(handle.getCatalogName(), schema, table), + quoted(columnName)); execute(connection, sql); } catch (SQLException e) { @@ -666,7 +681,7 @@ protected String toRemoteSchemaName(ConnectorSession session, JdbcIdentity ident try { DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers()) { + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { return schemaName.toUpperCase(ENGLISH); } return schemaName; @@ -714,7 +729,7 @@ protected String toRemoteTableName(ConnectorSession session, JdbcIdentity identi try { DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers()) { + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { return tableName.toUpperCase(ENGLISH); } return tableName; diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 76725407eb1d5..9656e918bea0d 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-benchmark-driver diff --git a/presto-benchmark-runner/pom.xml b/presto-benchmark-runner/pom.xml index 75c24687af740..0e87ab2b95583 100644 --- a/presto-benchmark-runner/pom.xml +++ b/presto-benchmark-runner/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-benchmark-runner diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 50a69466cde0f..0c109bfbf3434 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-benchmark diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml index d6e02bf909005..1a63839b316b4 100644 --- a/presto-benchto-benchmarks/pom.xml +++ b/presto-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-benchto-benchmarks diff --git a/presto-bigquery/pom.xml b/presto-bigquery/pom.xml index 611d8703d3226..69c17b606a911 100644 --- a/presto-bigquery/pom.xml +++ b/presto-bigquery/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-bigquery @@ -15,9 +15,9 @@ ${project.parent.basedir} - 1.33.1 + 1.39.1 1.11.0 - 1.46.3 + 2.0.0 true @@ -26,7 +26,7 @@ com.google.cloud libraries-bom - 16.3.0 + 26.68.0 pom import @@ -46,7 +46,7 @@ com.google.api-client google-api-client - 1.31.1 + 2.8.0 diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index cddba23bf3c87..9e37e40d2aa13 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-blackhole diff --git a/presto-built-in-worker-function-tools/pom.xml b/presto-built-in-worker-function-tools/pom.xml index 8159c62c45b5d..b774b0f203503 100644 --- a/presto-built-in-worker-function-tools/pom.xml +++ b/presto-built-in-worker-function-tools/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-bytecode/pom.xml b/presto-bytecode/pom.xml index 62b53fab7226d..9af4a4fbfbda6 100644 --- a/presto-bytecode/pom.xml +++ b/presto-bytecode/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-bytecode diff --git a/presto-cache/pom.xml b/presto-cache/pom.xml index b0cf8597044cb..e1986359dc7f1 100644 --- a/presto-cache/pom.xml +++ b/presto-cache/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-cache diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 3415ac90db6d0..71f1a0b50b0c5 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index c2298b07486c2..5ae530bc147fb 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-cli diff --git a/presto-clickhouse/pom.xml b/presto-clickhouse/pom.xml index 82bc433f1607a..8f66eda027896 100644 --- a/presto-clickhouse/pom.xml +++ b/presto-clickhouse/pom.xml @@ -4,7 +4,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-clickhouse diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseClient.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseClient.java index ba7b932169f78..1592dff1fde01 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseClient.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseClient.java @@ -62,12 +62,7 @@ import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.RealType.REAL; import static com.facebook.presto.common.type.SmallintType.SMALLINT; -import static com.facebook.presto.common.type.TimeType.TIME; -import static com.facebook.presto.common.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE; -import static com.facebook.presto.common.type.TimestampType.TIMESTAMP; -import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.TinyintType.TINYINT; -import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.plugin.clickhouse.ClickHouseEngineType.MERGETREE; import static com.facebook.presto.plugin.clickhouse.ClickHouseErrorCode.JDBC_ERROR; import static com.facebook.presto.plugin.clickhouse.ClickhouseDXLKeyWords.ORDER_BY_PROPERTY; @@ -94,21 +89,6 @@ public class ClickHouseClient { private static final Logger log = Logger.get(ClickHouseClient.class); - private static final Map SQL_TYPES = ImmutableMap.builder() - .put(BOOLEAN, "boolean") - .put(BIGINT, "bigint") - .put(INTEGER, "integer") - .put(SMALLINT, "smallint") - .put(TINYINT, "tinyint") - .put(DOUBLE, "double precision") - .put(REAL, "real") - .put(VARBINARY, "varbinary") - .put(DATE, "Date") - .put(TIME, "time") - .put(TIME_WITH_TIME_ZONE, "time with timezone") - .put(TIMESTAMP, "timestamp") - .put(TIMESTAMP_WITH_TIME_ZONE, "timestamp with timezone") - .build(); private static final String tempTableNamePrefix = "tmp_presto_"; protected static final String identifierQuote = "\""; protected final String connectorId; @@ -185,7 +165,7 @@ public final Set getSchemaNames(ClickHouseIdentity identity) } } - public ConnectorSplitSource getSplits(ClickHouseIdentity identity, ClickHouseTableLayoutHandle layoutHandle) + public ConnectorSplitSource getSplits(ClickHouseTableLayoutHandle layoutHandle) { ClickHouseTableHandle tableHandle = layoutHandle.getTable(); ClickHouseSplit clickHouseSplit = new ClickHouseSplit( @@ -213,7 +193,7 @@ public List getColumns(ConnectorSession session, ClickHo resultSet.getInt("DECIMAL_DIGITS"), Optional.empty(), Optional.empty()); - Optional columnMapping = toPrestoType(session, typeHandle); + Optional columnMapping = toPrestoType(typeHandle); // skip unsupported column types if (columnMapping.isPresent()) { String columnName = resultSet.getString("COLUMN_NAME"); @@ -235,7 +215,7 @@ public List getColumns(ConnectorSession session, ClickHo } } - public Optional toPrestoType(ConnectorSession session, ClickHouseTypeHandle typeHandle) + public Optional toPrestoType(ClickHouseTypeHandle typeHandle) { return jdbcTypeToPrestoType(typeHandle, mapStringAsVarchar); } @@ -293,7 +273,7 @@ public String buildInsertSql(ClickHouseOutputTableHandle handle) String columns = Joiner.on(',').join(nCopies(handle.getColumnNames().size(), "?")); return new StringBuilder() .append("INSERT INTO ") - .append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName())) + .append(quoted(handle.getSchemaName(), handle.getTemporaryTableName())) .append(" VALUES (").append(columns).append(")") .toString(); } @@ -389,7 +369,7 @@ private static String escapeNamePattern(String name, String escape) return name; } - protected String quoted(@Nullable String catalog, @Nullable String schema, String table) + protected String quoted(@Nullable String schema, String table) { StringBuilder builder = new StringBuilder(); if (!isNullOrEmpty(schema)) { @@ -406,16 +386,10 @@ public void addColumn(ClickHouseIdentity identity, ClickHouseTableHandle handle, String columnName = column.getName(); String sql = format( "ALTER TABLE %s ADD COLUMN %s", - quoted(handle.getCatalogName(), schema, table), + quoted(schema, table), getColumnDefinitionSql(column, columnName)); try (Connection connection = connectionFactory.openConnection(identity)) { - DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { - schema = schema != null ? schema.toUpperCase(ENGLISH) : null; - table = table.toUpperCase(ENGLISH); - columnName = columnName.toUpperCase(ENGLISH); - } execute(connection, sql); } catch (SQLException e) { @@ -448,14 +422,9 @@ private ClickHouseOutputTableHandle beginWriteTable(ConnectorSession session, Co public void dropColumn(ClickHouseIdentity identity, ClickHouseTableHandle handle, ClickHouseColumnHandle column) { try (Connection connection = connectionFactory.openConnection(identity)) { - DatabaseMetaData metadata = connection.getMetaData(); - String columnName = column.getColumnName(); - if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { - columnName = columnName.toUpperCase(ENGLISH); - } String sql = format( "ALTER TABLE %s DROP COLUMN %s", - quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()), + quoted(handle.getSchemaName(), handle.getTableName()), quoted(column.getColumnName())); execute(connection, sql); } @@ -466,8 +435,8 @@ public void dropColumn(ClickHouseIdentity identity, ClickHouseTableHandle handle public void finishInsertTable(ClickHouseIdentity identity, ClickHouseOutputTableHandle handle) { - String temporaryTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()); - String targetTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()); + String temporaryTable = quoted(handle.getSchemaName(), handle.getTemporaryTableName()); + String targetTable = quoted(handle.getSchemaName(), handle.getTableName()); String insertSql = format("INSERT INTO %s SELECT * FROM %s", targetTable, temporaryTable); String cleanupSql = "DROP TABLE " + temporaryTable; @@ -538,15 +507,11 @@ public void renameColumn(ClickHouseIdentity identity, ClickHouseTableHandle hand { String sql = format( "ALTER TABLE %s RENAME COLUMN %s TO %s", - quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()), + quoted(handle.getSchemaName(), handle.getTableName()), quoted(clickHouseColumn.getColumnName()), quoted(newColumnName)); try (Connection connection = connectionFactory.openConnection(identity)) { - DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { - newColumnName = newColumnName.toUpperCase(ENGLISH); - } execute(connection, sql); } catch (SQLException e) { @@ -717,7 +682,7 @@ public void dropTable(ClickHouseIdentity identity, ClickHouseTableHandle handle) { StringBuilder sql = new StringBuilder() .append("DROP TABLE ") - .append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName())); + .append(quoted(handle.getSchemaName(), handle.getTableName())); try (Connection connection = connectionFactory.openConnection(identity)) { execute(connection, sql.toString()); @@ -744,7 +709,7 @@ public void renameTable(ClickHouseIdentity identity, ClickHouseTableHandle handl renameTable(identity, handle.getCatalogName(), handle.getSchemaTableName(), newTable); } - public void createSchema(ClickHouseIdentity identity, String schemaName, Map properties) + public void createSchema(ClickHouseIdentity identity, String schemaName) { try (Connection connection = connectionFactory.openConnection(identity)) { execute(connection, "CREATE DATABASE " + quoted(schemaName)); @@ -770,20 +735,11 @@ protected void renameTable(ClickHouseIdentity identity, String catalogName, Sche String tableName = oldTable.getTableName(); String newSchemaName = newTable.getSchemaName(); String newTableName = newTable.getTableName(); - String sql = format("RENAME TABLE %s.%s TO %s.%s", - quoted(schemaName), - quoted(tableName), - quoted(newTable.getSchemaName()), - quoted(newTable.getTableName())); + String sql = format("RENAME TABLE %s TO %s", + quoted(schemaName, tableName), + quoted(newSchemaName, newTableName)); try (Connection connection = connectionFactory.openConnection(identity)) { - DatabaseMetaData metadata = connection.getMetaData(); - if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { - schemaName = schemaName.toUpperCase(ENGLISH); - tableName = tableName.toUpperCase(ENGLISH); - newSchemaName = newSchemaName.toUpperCase(ENGLISH); - newTableName = newTableName.toUpperCase(ENGLISH); - } execute(connection, sql); } catch (SQLException e) { @@ -901,8 +857,8 @@ protected void copyTableSchema(ClickHouseIdentity identity, String catalogName, String newCreateTableName = newTableName.getTableName(); String sql = format( "CREATE TABLE %s AS %s ", - quoted(null, schemaName, newCreateTableName), - quoted(null, schemaName, oldCreateTableName)); + quoted(schemaName, newCreateTableName), + quoted(schemaName, oldCreateTableName)); try (Connection connection = connectionFactory.openConnection(identity)) { execute(connection, sql); @@ -917,7 +873,6 @@ protected void copyTableSchema(ClickHouseIdentity identity, String catalogName, private String quoted(RemoteTableName remoteTableName) { return quoted( - remoteTableName.getCatalogName().orElse(null), remoteTableName.getSchemaName().orElse(null), remoteTableName.getTableName()); } diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseMetadata.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseMetadata.java index 8dbd357f08035..0b4649115056d 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseMetadata.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseMetadata.java @@ -273,7 +273,7 @@ public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTab @Override public void createSchema(ConnectorSession session, String schemaName, Map properties) { - clickHouseClient.createSchema(ClickHouseIdentity.from(session), schemaName, properties); + clickHouseClient.createSchema(ClickHouseIdentity.from(session), schemaName); } @Override diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseRecordCursor.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseRecordCursor.java index b194e28190253..b2e8d48f700ad 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseRecordCursor.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseRecordCursor.java @@ -63,7 +63,7 @@ public ClickHouseRecordCursor(ClickHouseClient clickHouseClient, ConnectorSessio sliceReadFunctions = new SliceReadFunction[columnHandles.size()]; for (int i = 0; i < this.columnHandles.length; i++) { - ReadMapping readMapping = clickHouseClient.toPrestoType(session, columnHandles.get(i).getClickHouseTypeHandle()) + ReadMapping readMapping = clickHouseClient.toPrestoType(columnHandles.get(i).getClickHouseTypeHandle()) .orElseThrow(() -> new VerifyException("Unsupported column type")); Class javaType = readMapping.getType().getJavaType(); ReadFunction readFunction = readMapping.getReadFunction(); diff --git a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseSplitManager.java b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseSplitManager.java index c043a87225ee2..4300e2287d0e9 100755 --- a/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseSplitManager.java +++ b/presto-clickhouse/src/main/java/com/facebook/presto/plugin/clickhouse/ClickHouseSplitManager.java @@ -41,6 +41,6 @@ public ConnectorSplitSource getSplits( SplitSchedulingContext splitSchedulingContext) { ClickHouseTableLayoutHandle layoutHandle = (ClickHouseTableLayoutHandle) layout; - return clickHouseClient.getSplits(ClickHouseIdentity.from(session), layoutHandle); + return clickHouseClient.getSplits(layoutHandle); } } diff --git a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestClickHouseDistributedQueries.java b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestClickHouseDistributedQueries.java index aa58f81150100..3a44b568a6a23 100755 --- a/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestClickHouseDistributedQueries.java +++ b/presto-clickhouse/src/test/java/com/facebook/presto/plugin/clickhouse/TestClickHouseDistributedQueries.java @@ -359,6 +359,22 @@ public void testAddColumn() assertFalse(getQueryRunner().tableExists(getSession(), tableName)); } + @Override + public void testRenameTable() + { + String tableName = "test_rename_table_" + randomTableSuffix(); + String newTableName = "test_rename_table_new_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + " (id int NOT NULL, x VARCHAR) WITH (engine = 'MergeTree', order_by = ARRAY['id'])"); + assertUpdate("INSERT INTO " + tableName + " (id, x) VALUES(1, 'first')", 1); + + assertUpdate("ALTER TABLE " + tableName + " RENAME TO " + newTableName); + assertFalse(getQueryRunner().tableExists(getSession(), tableName)); + assertTrue(getQueryRunner().tableExists(getSession(), newTableName)); + assertUpdate("DROP TABLE " + newTableName); + + assertFalse(getQueryRunner().tableExists(getSession(), newTableName)); + } + @Test public void testShowCreateTable() { diff --git a/presto-client/pom.xml b/presto-client/pom.xml index 6435a2cda2551..717ef775f59f1 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-client diff --git a/presto-cluster-ttl-providers/pom.xml b/presto-cluster-ttl-providers/pom.xml index 9c0d2063eb275..0c1c9b1e20d1b 100644 --- a/presto-cluster-ttl-providers/pom.xml +++ b/presto-cluster-ttl-providers/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-common/pom.xml b/presto-common/pom.xml index 5a708694a5ab1..bafa40b87277d 100644 --- a/presto-common/pom.xml +++ b/presto-common/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-common diff --git a/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java b/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java index 129e24a3e75da..7e982a98489b5 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java +++ b/presto-common/src/main/java/com/facebook/presto/common/RuntimeMetricName.java @@ -69,6 +69,9 @@ private RuntimeMetricName() // Blocked time for the operators due to waiting for inputs. public static final String TASK_BLOCKED_TIME_NANOS = "taskBlockedTimeNanos"; public static final String TASK_UPDATE_DELIVERED_WALL_TIME_NANOS = "taskUpdateDeliveredWallTimeNanos"; + public static final String TASK_START_WAIT_FOR_EVENT_LOOP = "taskStartWaitForEventLoop"; + public static final String TASK_UPDATE_DELIVERED_UPDATES = "taskUpdateDeliveredUpdates"; + public static final String TASK_UPDATE_ROUND_TRIP_TIME = "taskUpdateRoundTripTime"; public static final String TASK_UPDATE_SERIALIZED_CPU_TIME_NANOS = "taskUpdateSerializedCpuNanos"; public static final String TASK_PLAN_SERIALIZED_CPU_TIME_NANOS = "taskPlanSerializedCpuNanos"; // Time for event loop to execute a method diff --git a/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java b/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java index 3a732de316454..f2900acbaaf45 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java @@ -27,7 +27,8 @@ public enum QueryType INSERT(6), SELECT(7), CONTROL(8), - UPDATE(9) + UPDATE(9), + MERGE(10) /**/; private final int value; diff --git a/presto-db-session-property-manager/pom.xml b/presto-db-session-property-manager/pom.xml index d0ef4ca921014..834a30895ad44 100644 --- a/presto-db-session-property-manager/pom.xml +++ b/presto-db-session-property-manager/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-db-session-property-manager @@ -54,13 +54,18 @@ - javax.annotation - javax.annotation-api + javax.inject + javax.inject - javax.inject - javax.inject + jakarta.annotation + jakarta.annotation-api + + + + jakarta.inject + jakarta.inject-api diff --git a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java index 528d51c395c6e..c0b7a059c88e4 100644 --- a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java +++ b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java @@ -17,8 +17,7 @@ import com.facebook.presto.session.SessionMatchSpec; import com.facebook.presto.spi.session.SessionConfigurationContext; import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; - -import javax.inject.Inject; +import jakarta.inject.Inject; import java.util.List; diff --git a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java index 4a94ac0d9e61a..55759dc278db8 100644 --- a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java +++ b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java @@ -23,6 +23,7 @@ public class DbSessionPropertyManagerConfig { private String configDbUrl; + private String jdbcDriverName = "com.mysql.jdbc.Driver"; private Duration specsRefreshPeriod = new Duration(10, SECONDS); @NotNull @@ -38,6 +39,19 @@ public DbSessionPropertyManagerConfig setConfigDbUrl(String configDbUrl) return this; } + @NotNull + public String getJdbcDriverName() + { + return jdbcDriverName; + } + + @Config("session-property-manager.db.driver-name") + public DbSessionPropertyManagerConfig setJdbcDriverName(String jdbcDriverName) + { + this.jdbcDriverName = jdbcDriverName; + return this; + } + @NotNull @MinDuration("1ms") public Duration getSpecsRefreshPeriod() diff --git a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java index 2694fd5e54e9e..7c44a813cc13e 100644 --- a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java +++ b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java @@ -17,10 +17,9 @@ import com.facebook.presto.session.SessionMatchSpec; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.inject.Inject; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.inject.Inject; import java.util.List; import java.util.concurrent.ScheduledExecutorService; diff --git a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java index 06617a1885396..a567cf5338014 100644 --- a/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java +++ b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java @@ -13,10 +13,10 @@ */ package com.facebook.presto.session.db; +import jakarta.inject.Inject; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.sqlobject.SqlObjectPlugin; -import javax.inject.Inject; import javax.inject.Provider; import java.sql.DriverManager; @@ -33,6 +33,14 @@ public SessionPropertiesDaoProvider(DbSessionPropertyManagerConfig config) { requireNonNull(config, "config is null"); requireNonNull(config.getConfigDbUrl(), "db url is null"); + + try { + Class.forName(config.getJdbcDriverName()); + } + catch (ClassNotFoundException e) { + throw new RuntimeException("JDBC driver class not found: " + config.getJdbcDriverName(), e); + } + this.dao = Jdbi.create(() -> DriverManager.getConnection(config.getConfigDbUrl())) .installPlugin(new SqlObjectPlugin()) .onDemand(SessionPropertiesDao.class); diff --git a/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java index d976eb722390c..f22548acf1fc7 100644 --- a/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java +++ b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java @@ -32,6 +32,7 @@ public void testDefaults() { assertRecordedDefaults(recordDefaults(DbSessionPropertyManagerConfig.class) .setConfigDbUrl(null) + .setJdbcDriverName("com.mysql.jdbc.Driver") .setSpecsRefreshPeriod(new Duration(10, SECONDS))); } @@ -40,11 +41,13 @@ public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() .put("session-property-manager.db.url", "foo") + .put("session-property-manager.db.driver-name", "org.mariadb.jdbc.Driver") .put("session-property-manager.db.refresh-period", "50s") .build(); DbSessionPropertyManagerConfig expected = new DbSessionPropertyManagerConfig() .setConfigDbUrl("foo") + .setJdbcDriverName("org.mariadb.jdbc.Driver") .setSpecsRefreshPeriod(new Duration(50, TimeUnit.SECONDS)); assertFullMapping(properties, expected); diff --git a/presto-delta/pom.xml b/presto-delta/pom.xml index 39fde11fe84e3..2aa4e4070b34c 100644 --- a/presto-delta/pom.xml +++ b/presto-delta/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-delta diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 03388aa190447..b4d71d2f9b914 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-docs diff --git a/presto-docs/src/main/sphinx/admin.rst b/presto-docs/src/main/sphinx/admin.rst index da9997630ed5d..90cca88110398 100644 --- a/presto-docs/src/main/sphinx/admin.rst +++ b/presto-docs/src/main/sphinx/admin.rst @@ -20,3 +20,4 @@ Administration admin/spark admin/verifier admin/grafana-cloud + admin/version-support diff --git a/presto-docs/src/main/sphinx/admin/properties.rst b/presto-docs/src/main/sphinx/admin/properties.rst index 7a0a8de99348f..b1aea2b875fb1 100644 --- a/presto-docs/src/main/sphinx/admin/properties.rst +++ b/presto-docs/src/main/sphinx/admin/properties.rst @@ -1029,9 +1029,9 @@ The corresponding session property is :ref:`admin/properties-session:\`\`optimiz * **Default Value:** ``false`` Enables the optimizer to use histograms when available to perform cost estimate calculations -during query optimization. When set to ``false``, this parameter does not prevent histograms -from being collected by ``ANALYZE``, but prevents them from being used during query -optimization. This behavior can be controlled on a per-query basis using the +during query optimization. When set to ``false``, this parameter prevents histograms from +being collected by ``ANALYZE``, and also prevents the existing histograms from being used +during query optimization. This behavior can be controlled on a per-query basis using the ``optimizer_use_histograms`` session property. Planner Properties diff --git a/presto-docs/src/main/sphinx/admin/resource-groups.rst b/presto-docs/src/main/sphinx/admin/resource-groups.rst index cf73d25d53830..54b48dc14601a 100644 --- a/presto-docs/src/main/sphinx/admin/resource-groups.rst +++ b/presto-docs/src/main/sphinx/admin/resource-groups.rst @@ -277,6 +277,8 @@ Here are the key components of selector rules in PrestoDB: - ``ANALYZE``: ``ANALYZE`` queries. - ``DATA_DEFINITION``: Queries that alter/create/drop the metadata of schemas/tables/views, and that manage prepared statements, privileges, sessions, and transactions. + - ``CONTROL``: Transaction control queries like ``COMMIT``, ``ROLLBACK`` and session control queries like + ``USE``, ``SET SESSION``. * ``clientTags`` (optional): List of tags. To match, every tag in this list must be in the list of client-provided tags associated with the query. @@ -352,13 +354,13 @@ There are four selectors, that define which queries run in which resource group: dynamically-created per-user pipeline group under the ``global.pipeline`` group. * The fourth selector matches queries that come from BI tools which have a source matching the regular - expression ``jdbc#(?.*)``, and have client provided tags that are a superset of ``hi-pri``. - These are placed in a dynamically-created sub-group under the ``global.pipeline.tools`` group. The dynamic - sub-group is created based on the named variable ``toolname``, which is extracted from the - regular expression for source. + expression ``jdbc#(?.*)``, and have client provided tags that are a superset of ``hipri``. + These are placed in a dynamically created sub-group under the ``global.adhoc`` group. The dynamic sub-groups + are created based on the values of named variables ``toolname`` and ``USER``. The values are derived from the + source regular expression and the query user respectively. Consider a query with a source ``jdbc#powerfulbi``, user ``kayla``, and - client tags ``hipri`` and ``fast``. This query is routed to the ``global.pipeline.bi-powerfulbi.kayla`` + client tags ``hipri`` and ``fast``. This query is routed to the ``global.adhoc.bi-powerfulbi.kayla`` resource group. * The last selector is a catch-all, which places all queries that have not yet been matched into a per-user @@ -373,9 +375,9 @@ For the remaining users: * No more than 100 total queries may run concurrently. -* Up to 5 concurrent DDL queries with a source ``pipeline`` can run. Queries are run in FIFO order. +* Up to 5 concurrent DDL queries with a source that includes ``pipeline`` in its name can run. Queries are run in FIFO order. -* Non-DDL queries will run under the ``global.pipeline`` group, with a total concurrency of 45, and a per-user +* Non-DDL queries with a source that includes ``pipeline`` in its name run under the ``global.pipeline`` group, with a total concurrency of 45, and a per-user concurrency of 5. Queries are run in FIFO order. * For BI tools, each tool can run up to 10 concurrent queries, and each user can run up to 3. If the total demand diff --git a/presto-docs/src/main/sphinx/admin/version-support.rst b/presto-docs/src/main/sphinx/admin/version-support.rst new file mode 100644 index 0000000000000..883efecd129de --- /dev/null +++ b/presto-docs/src/main/sphinx/admin/version-support.rst @@ -0,0 +1,230 @@ +=============== +Version Support +=============== + +Overview +-------- + +Presto is maintained by volunteers. This document describes which versions receive support and what level of support to expect. + +Support Philosophy +------------------ + +* Data correctness issues are taken extremely seriously and typically fixed quickly +* Runtime bugs and security vulnerabilities are prioritized and addressed promptly +* Support depends on volunteer availability - no formal SLAs +* Users are encouraged to contribute fixes for issues affecting them + +.. _current-version-support: + +Current Version Support +----------------------- + +**Latest Release** + * Primary focus for bug fixes + * Recommended for new deployments after testing + +**Past 4 Releases (N-1 through N-4)** + * Critical fixes only when: + + - Data correctness issues are found + - Volunteers are available to backport + + * Patch releases for severe issues only + * Support decreases with age + +**Older Releases (N-5 and earlier)** + * Not supported + * Exceptions only when: + + - Volunteer provides the backport + - Fix applies cleanly + - Testing is available + + * Upgrade required + +**Trunk/Master Branch** + * Development branch + * **Never use in production** + * Contains experimental features and bugs + * For testing upcoming changes only + +**Edge Releases** + * Weekly builds from master + * **Never use in production** + * **Not supported** - no fixes provided + * For testing upcoming features + +Support Lifecycle +----------------- + +Timeframes are approximate and depend on volunteer availability. + +A typical release follows this lifecycle: + +1. **Release Candidates** (2-4 weeks) + + - One RC version per release + - Active bug fixing with fixes verified in the existing RC + - High community engagement + +2. **Current Release** (approximately 2 months) + + - Primary focus for bug fixes + - Active monitoring for issues + - Most community attention + +3. **Supported Releases** (N-1 through N-4, approximately 8 months) + + - Critical fixes only + - Progressively reduced community focus + - Patch releases for severe issues become less likely with age + +4. **Archived** (N-5 and older) + + - No active support + - Users strongly encouraged to upgrade + - See :ref:`current-version-support` for details + +Types of Support +---------------- + +**Bug Fixes** + Highest priority (typically fixed very quickly): + + * Data correctness issues - taken extremely seriously + + High priority: + + * Runtime bugs and crashes + * Severe performance regressions + + Lower priority: + + * Minor performance issues + * UI/cosmetic problems + * Feature enhancements + +**Security Vulnerabilities** + * Upgrade to latest release (default recommendation) + * Patches for N-1 through N-4 available upon request + * Backport availability depends on volunteers and severity + * Plan to upgrade rather than rely on backports + +**Documentation** + * Release notes and full documentation for all versions remain available + * Migration guides for major changes + * Community-contributed upgrade experiences + +Getting Support +--------------- + +**Community Channels** + +* `Presto Slack `_ - Real-time community discussion +* `GitHub Issues `_ - Bug reports and feature requests +* `Mailing List `_ - Development discussions + +**Self-Support Resources** + +* Release notes and documentation +* Community Slack search history +* GitHub issues and pull requests +* Stack Overflow questions tagged 'presto' + +Recommendations for Production Use +---------------------------------- + +**Version Selection** + +1. **For new deployments**: Use the latest stable release after thorough testing +2. **For existing deployments**: Stay within 4 versions of the latest release +3. **For conservative environments**: Wait for at least one patch release (if any) before upgrading +4. **Never use trunk/master or edge** in production + +**Upgrade Strategy** + +* Plan regular upgrades (every 2-4 months) +* Test thoroughly in staging environments +* Monitor community channels for known issues +* Maintain ability to rollback if needed +* Consider skipping releases if stable (but don't fall too far behind) + +**Risk Mitigation** + +* Maintain test environments matching production +* Participate in release candidate testing +* Monitor community discussions for your version +* Contribute test cases for critical workflows + +Contributing to Support +----------------------- + +Ways to contribute: + +**Report Issues** + * File detailed bug reports with reproduction steps on `GitHub Issues `_ + * Test fixes and provide feedback + * Share workarounds with the community + +**Contribute Fixes** + * Submit `pull requests `_ for bugs affecting you + * Help review and test others' fixes + * Backport critical fixes to versions you use + +**Share Knowledge** + * Document upgrade experiences + * Answer questions in `Presto Slack `_ + * Write blog posts about solutions + * Contribute to `documentation `_ + +**Sponsor Development** + * Allocate engineering resources to the project + * Fund specific feature development + * Support maintainers and release shepherds + +Special Considerations +---------------------- + +**Long-Term Support (LTS)** + * Not available + * Volunteer model incompatible with LTS commitments + +**End-of-Life Announcements** + * No formal EOL process + * Versions become unsupported as community moves forward + * Check release announcements for migration guidance + +**Compatibility** + * Breaking changes documented in release notes + * Migration guides provided for major changes + * Test when upgrading across multiple versions + +Support Expectations +-------------------- + +**Available:** + +* Typically quick response to data correctness and runtime bugs +* Priority focus on critical issues +* Active community troubleshooting help +* Transparency about known issues +* Documentation for old versions + +**Not Available:** + +* Guaranteed response times +* Fixes for all issues +* Support for old versions +* Feature backports +* 24/7 support + +Summary +------- + +Running Presto in production requires: + +* Regular upgrades (every 2-4 months) +* Thorough testing before deploying +* Understanding that support is volunteer-based +* Contributing fixes for issues you encounter \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/connector/iceberg.rst b/presto-docs/src/main/sphinx/connector/iceberg.rst index 7b9adc3c0933a..480ee57bb7978 100644 --- a/presto-docs/src/main/sphinx/connector/iceberg.rst +++ b/presto-docs/src/main/sphinx/connector/iceberg.rst @@ -1491,10 +1491,16 @@ Use ``ARRAY[...]`` instead of a string to specify multiple partition transforms ALTER TABLE iceberg.web.page_views ADD COLUMN dt date WITH (partitioning = ARRAY['year', 'bucket(16)', 'identity']); -Table properties can be modified for an Iceberg table using an ALTER TABLE SET PROPERTIES statement. Only `commit_retries` can be modified at present. -For example, to set `commit_retries` to 6 for the table `iceberg.web.page_views_v2`, use:: +Some Iceberg table properties can be modified using an ALTER TABLE SET PROPERTIES statement. The modifiable table properties are +``commit.retry.num-retries``, ``read.split.target-size``, ``write.metadata.delete-after-commit.enabled``, and ``write.metadata.previous-versions-max``. - ALTER TABLE iceberg.web.page_views_v2 SET PROPERTIES (commit_retries = 6); +For example, to set ``commit.retry.num-retries`` to 6 for the table ``iceberg.web.page_views_v2``, use:: + + ALTER TABLE iceberg.web.page_views_v2 SET PROPERTIES ("commit.retry.num-retries" = 6); + +To set ``write.metadata.delete-after-commit.enabled`` to true and set ``write.metadata.previous-versions-max`` to 5, use:: + + ALTER TABLE iceberg.web.page_views_v2 SET PROPERTIES ("write.metadata.delete-after-commit.enabled" = true, "write.metadata.previous-versions-max" = 5); ALTER VIEW ^^^^^^^^^^ diff --git a/presto-docs/src/main/sphinx/connector/redis.rst b/presto-docs/src/main/sphinx/connector/redis.rst index 133ef3a49cf1f..ab91303a31d9f 100644 --- a/presto-docs/src/main/sphinx/connector/redis.rst +++ b/presto-docs/src/main/sphinx/connector/redis.rst @@ -54,6 +54,7 @@ Property Name Description ``redis.user`` Redis server username ``redis.tls.enabled`` Whether TLS security is enabled (defaults to ``false``) ``redis.tls.truststore-path`` Path to the TLS certificate file +``case-sensitive-name-matching`` Enable case sensitive identifier support for schema, table, and column names for the connector. When disabled, names are matched case-insensitively using lowercase normalization. Defaults to ``false``. ================================= ============================================================== ``redis.table-names`` diff --git a/presto-docs/src/main/sphinx/develop.rst b/presto-docs/src/main/sphinx/develop.rst index 3f65e72865df2..67e033cee8984 100644 --- a/presto-docs/src/main/sphinx/develop.rst +++ b/presto-docs/src/main/sphinx/develop.rst @@ -22,3 +22,4 @@ This guide is intended for Presto contributors and plugin developers. develop/presto-console develop/presto-authenticator develop/client-request-filter + develop/release-process diff --git a/presto-docs/src/main/sphinx/develop/functions.rst b/presto-docs/src/main/sphinx/develop/functions.rst index b30d3cfc71113..2de3e550d149f 100644 --- a/presto-docs/src/main/sphinx/develop/functions.rst +++ b/presto-docs/src/main/sphinx/develop/functions.rst @@ -113,7 +113,7 @@ a wrapper around ``byte[]``, rather than ``String`` for its native container typ ``@SqlNullable`` if it can return ``NULL`` when the arguments are non-null. Parametric Scalar Functions ---------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Scalar functions that have type parameters have some additional complexity. To make our previous example work with any type we need the following: @@ -187,7 +187,7 @@ To make our previous example work with any type we need the following: } Another Scalar Function Example -------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``lowercaser`` function takes a single ``VARCHAR`` argument and returns a ``VARCHAR``, which is the argument converted to lower case: @@ -214,7 +214,7 @@ has no ``@SqlNullable`` annotations, meaning that if the argument is ``NULL``, the result will automatically be ``NULL`` (the function will not be called). Codegen Scalar Function Implementation --------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Scalar functions can also be implemented in bytecode, allowing us to specialize and optimize functions according to the ``@TypeParameter`` diff --git a/presto-docs/src/main/sphinx/develop/release-process.rst b/presto-docs/src/main/sphinx/develop/release-process.rst new file mode 100644 index 0000000000000..1dbbcf6fdce85 --- /dev/null +++ b/presto-docs/src/main/sphinx/develop/release-process.rst @@ -0,0 +1,280 @@ +=============== +Release Process +=============== + +Overview +======== + +Presto releases are managed by volunteer committers. Releases occur approximately every 2 months, with extended testing periods to ensure stability. + +Release Cadence +=============== + +Releases target a 2-month cycle. Actual timing depends on: + +* Release shepherd availability +* Testing feedback +* Critical issues requiring delay +* Contributing organization resources + +Schedules adjust based on volunteer availability. + +Release Quality Model +===================== + +**Note:** Trunk is not stable. Do not use master branch in production. + +**Extended Release Candidate Period** + * 2-4 week RC period for testing + * Issues fixed before final release + * Fixes verified in existing RC (no new RCs) + +**Community Testing** + * Organizations test RCs in their environments + * Weekly edge releases from master for early testing + * Join #releases in `Presto Slack `_ to participate + +**Automated Testing** + * Unit tests on all commits and RCs + * Product tests with full cluster deployment + * Shared connector tests with `AbstractTestQueries `_ + * CI for basic stability checks + +**Testing Contributions Needed** + * Additional `product test scenarios `_ + * Performance regression testing (shadow traffic) + * Test coverage for new features + +Version Numbering +================= + +* Format: ``0.XXX`` (for example, 0.293, 0.294) +* Major version fixed at 0 +* Minor version increments each release +* Patch releases (for example, 0.293.1) for critical fixes only + +Not semantic versioning. + +Release Types +============= + +**Regular Releases** + * Every ~2 months + * New features, improvements, bug fixes + * Extended RC testing period + +**Patch Releases** + * Critical issues only: + + - Security vulnerabilities (upon request - default: upgrade) + - Data correctness + - Performance regressions + - Stability issues + + * Based on previous stable release + * Minimal changes + +**Release Candidates** + * 2-4 week testing period + * One RC per release + * Fixes verified in existing RC + +**Edge Releases** + * Weekly from master + * Early access to features + * Testing only - not for production + +Release Shepherd Responsibilities +================================= + +Requirements: + +* Must be a committer +* Must understand codebase for judgment calls and rewrites + +Responsibilities: + +* Complete release process +* Ensure release notes follow guidelines +* Fix release note issues +* Cut and deploy release +* Coordinate with community +* Make go/no-go decisions + +See `Release Shepherding `_ for schedule and details. + +Contributing to Release Quality +=============================== + +**Testing** + * Run RCs in test environments + * Report issues with reproduction steps + * Verify cherry-picked fixes + +**Test Development** + * Add product tests for uncovered scenarios + * Unit tests for bug fixes + * Performance benchmarks (`pbench `_) + +**Documentation** + * Document behavior changes + * Write clear release notes + * Note compatibility issues + +**Code Review** + * Review PRs for correctness + * Identify compatibility issues + * Suggest test coverage + +Backward Compatibility Guidelines +================================= + +**Must Maintain Compatibility:** + +* **Client Libraries**: Evolve slowly. New server features must work with older clients. +* **SQL Syntax**: Keep stable. Deprecate with warnings before removal. +* **SPI**: Stable for connectors/plugins. Use ``@Deprecated`` for at least two releases before removal. + When adding new SPI methods, provide reasonable defaults to minimize connector updates. + Documented SPI interfaces must remain stable even without public implementations. + Exception: Undocumented AND unused SPI aspects. +* **Configuration**: Session and config properties need deprecation paths. Provide aliases for renames. + +**Can Change:** + +* Internal APIs (not SPI) +* Performance characteristics +* Query plans +* Default config values (document changes) + +**Developer Requirements:** + +* Document breaking changes in release notes +* Provide migration paths +* Revert inadvertent breaking changes to client protocol, SQL, SPI, or config + +Revert Guidelines +================= + +When to Revert +^^^^^^^^^^^^^^ + +Data Correctness Issues or Critical Bugs +---------------------------------------- + +Any change that introduces data correctness issues, major crashes, or severe stability problems +must be reverted immediately if a fix is not quick, especially near the RC finalization window. + +**Must revert**: + +- Wrong query results, data corruption, frequent crashes +- Memory leaks or resource exhaustion in common code paths + +**Should revert**: + +- Performance regressions of more than 50% in common queries + +Backwards Incompatible Client Changes +------------------------------------- + +Client libraries evolve slowly and many users cannot easily upgrade clients. Breaking changes to +the client protocol, SQL syntax, or session/config properties without proper migration paths must +be reverted if they cannot be fixed quickly, particularly near RC finalization: + +**Must revert**: + +- Breaking client-server protocol compatibility +- Removing SQL syntax without deprecation warnings + +**Should revert**: + +- Changing session/config property behavior without aliases + +Backwards Incompatible SPI Changes Without Migration Path +--------------------------------------------------------- + +If a backwards incompatible change to the SPI is discovered that lacks the required migration path +(for example, no deprecation period, no reasonable defaults for new methods), the change should be reverted +if a proper migration path cannot be added quickly, especially near RC finalization. Use these criteria: + +**Must revert**: + +- Breaking documented SPI interfaces or core connectors (Hive, Iceberg, Delta, Kafka) +- Breaking maintained connectors with active usage in the repository + +**Should revert**: + +- Breaking experimental or rarely-used connectors (weigh maintenance burden) + +Consider both documented interfaces and public implementations in the Presto repository. +Create a GitHub issue marked as "release blocker" to alert the release shepherd. + +When NOT to Revert +^^^^^^^^^^^^^^^^^^ + +If the fix is simpler than the revert and can be completed quickly (especially before RC finalization), prefer fixing forward. + +**Fix forward**: + +- Typos +- Logging issues +- Minor UI problems +- Test failures that don't affect production code +- Documentation errors or missing documentation + +Performance Trade-offs +---------------------- + +Performance changes with mixed impact require case-by-case evaluation based on community feedback: + +- **Evaluate carefully**: What's rare for one user may be critical for another +- **Consider configuration**: Can the optimization be made optional by using session properties? +- **Gather data**: Solicit feedback from multiple organizations during RC testing + +If multiple users report significant regressions, consider reverting or adding a feature flag. +Always document performance changes and workarounds in release notes. + +Proprietary or Hidden Infrastructure Dependencies +------------------------------------------------- + +Changes cannot be reverted based on impacts to proprietary infrastructure, private forks, or +non-public connectors or plugins. All revert decisions must be justifiable using only publicly +available code, documentation, and usage patterns visible in the open source project. + +Feature Additions With Minor Issues +----------------------------------- + +New features that don't affect existing functionality should be fixed. +Consider adding feature flags if stability is a concern. + +How to Revert +------------- + +- Create a GitHub issue that describes the problem and label it "release blocker" immediately +- Raise a PR to revert the problematic change and link to the issue + +Release Communication +===================== + +* `Presto Slack `_ #releases channel +* `GitHub Releases `_ +* `Mailing List `_ +* `Release notes in docs `_ + +Best Practices for Developers +============================= + +* Avoid risky merges near release cuts +* Create as many automated tests as possible, and for large changes, consider product tests and manual testing +* Always consider how new features are enabled, whether they're enabled by default, and if not opt-in through SPI or SQL, gate them with a session property +* Document breaking changes in release notes +* Monitor #releases channel during release cycles +* Fix release blockers promptly + +Getting Involved +================ + +* Join #releases in `Presto Slack `_ +* Test release candidates +* Volunteer as a `release shepherd `_ (committers only) +* Contribute tests +* Share production experiences \ No newline at end of file diff --git a/presto-docs/src/main/sphinx/installation/deployment.rst b/presto-docs/src/main/sphinx/installation/deployment.rst index d3437a38793b4..cfb5532d75943 100644 --- a/presto-docs/src/main/sphinx/installation/deployment.rst +++ b/presto-docs/src/main/sphinx/installation/deployment.rst @@ -459,7 +459,7 @@ in the Hive connector catalog file are set to the following: .. code-block:: none - connector.name=hive + connector.name=hive-hadoop2 hive.metastore=file hive.metastore.catalog.dir=file:///data/hive_data/ diff --git a/presto-docs/src/main/sphinx/optimizer/statistics.rst b/presto-docs/src/main/sphinx/optimizer/statistics.rst index eeb763575c82a..6be080b501f0f 100644 --- a/presto-docs/src/main/sphinx/optimizer/statistics.rst +++ b/presto-docs/src/main/sphinx/optimizer/statistics.rst @@ -46,8 +46,9 @@ The following statistics are available in Presto: The set of statistics available for a particular query depends on the connector being used and can also vary by table or even by table layout. For example, the -Hive connector does not currently provide statistics on data size. +Hive connector does not currently provide statistics on data size or histograms, +while the Iceberg connector provides both. -Table statistics can be can be fetched using the :doc:`/sql/show-stats` query. +Table statistics can be fetched using the :doc:`/sql/show-stats` query. For the Hive connector, refer to the :ref:`Hive connector ` documentation to learn how to update table statistics. diff --git a/presto-druid/pom.xml b/presto-druid/pom.xml index b723b5e31a60f..9a6d1f2de1044 100644 --- a/presto-druid/pom.xml +++ b/presto-druid/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-druid diff --git a/presto-druid/src/main/java/com/facebook/presto/druid/DruidBrokerPageSource.java b/presto-druid/src/main/java/com/facebook/presto/druid/DruidBrokerPageSource.java index 96010fe22c17e..97058911341fb 100644 --- a/presto-druid/src/main/java/com/facebook/presto/druid/DruidBrokerPageSource.java +++ b/presto-druid/src/main/java/com/facebook/presto/druid/DruidBrokerPageSource.java @@ -130,7 +130,7 @@ public Page getNextPage() Type type = columnTypes.get(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i); JsonNode value = rootNode.get(((DruidColumnHandle) columnHandles.get(i)).getColumnName()); - if (value == null) { + if (value == null || value.isNull()) { blockBuilder.appendNull(); continue; } diff --git a/presto-druid/src/test/java/com/facebook/presto/druid/TestDruidPageSourceNullHandling.java b/presto-druid/src/test/java/com/facebook/presto/druid/TestDruidPageSourceNullHandling.java new file mode 100644 index 0000000000000..8be69063be38a --- /dev/null +++ b/presto-druid/src/test/java/com/facebook/presto/druid/TestDruidPageSourceNullHandling.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.druid; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpStatus; +import com.facebook.airlift.http.client.testing.TestingHttpClient; +import com.facebook.airlift.http.client.testing.TestingResponse; +import com.facebook.presto.common.Page; +import com.facebook.presto.common.block.Block; +import com.facebook.presto.spi.ColumnHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ListMultimap; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static org.testng.Assert.assertTrue; + +public class TestDruidPageSourceNullHandling +{ + @Test + public void testNullAndMissingColumns() + { + String jsonRows = + "{\"region.Id\":1,\"city\":\"Boston\",\"fare\":10.0}\n" + + "{\"region.Id\":2,\"city\":null,\"fare\":20.0}\n" + // city column is having null value + "{\"region.Id\":3,\"fare\":30.0}\n" + // missing city column + "\n"; + + ListMultimap headers = ImmutableListMultimap.of( + "Content-Type", "application/json"); + TestingResponse response = new TestingResponse( + HttpStatus.OK, + headers, + jsonRows.getBytes(StandardCharsets.UTF_8)); + HttpClient httpClient = new TestingHttpClient(request -> response); + + DruidConfig druidConfig = new DruidConfig() + .setDruidSchema("default") + .setDruidCoordinatorUrl("http://localhost:8081") + .setDruidBrokerUrl("http://localhost:8082"); + + ImmutableList columnHandles = ImmutableList.of(new DruidColumnHandle("region.Id", BIGINT), + new DruidColumnHandle("city", VARCHAR), + new DruidColumnHandle("fare", DOUBLE)); + + DruidBrokerPageSource pageSource = new DruidBrokerPageSource( + new DruidQueryGenerator.GeneratedDql("testTable", "SELECT region.Id, city, fare FROM test", true), + columnHandles, + new DruidClient(druidConfig, httpClient)); + + Page page; + boolean foundNull = false; + boolean foundMissing = false; + + while ((page = pageSource.getNextPage()) != null) { + Block cityBlock = page.getBlock(1); + for (int i = 0; i < cityBlock.getPositionCount(); i++) { + if (cityBlock.isNull(i)) { + if (i == 1) { + foundNull = true; // row with "city":null + } + if (i == 2) { + foundMissing = true; // row missing "city" + } + } + } + } + + assertTrue(foundNull, "Expected null value in column 'city'"); + assertTrue(foundMissing, "Expected missing column to be treated as null"); + } +} diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index 45d26e475dab5..c7c51d65ce5cf 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-elasticsearch presto-elasticsearch @@ -225,7 +225,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index d437639c2f18f..1247cccb6f0a8 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-example-http diff --git a/presto-expressions/pom.xml b/presto-expressions/pom.xml index 13fd54fc8110a..356d23b465575 100644 --- a/presto-expressions/pom.xml +++ b/presto-expressions/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-expressions diff --git a/presto-file-session-property-manager/pom.xml b/presto-file-session-property-manager/pom.xml index 5c3a4ceb2dddd..c8dd0f752b4bb 100644 --- a/presto-file-session-property-manager/pom.xml +++ b/presto-file-session-property-manager/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-file-session-property-manager diff --git a/presto-function-namespace-managers-common/pom.xml b/presto-function-namespace-managers-common/pom.xml index 6df0ee742cd3f..3b8ee30f137b8 100644 --- a/presto-function-namespace-managers-common/pom.xml +++ b/presto-function-namespace-managers-common/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT diff --git a/presto-function-namespace-managers/pom.xml b/presto-function-namespace-managers/pom.xml index a882b50e35df6..e47c912efa007 100644 --- a/presto-function-namespace-managers/pom.xml +++ b/presto-function-namespace-managers/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-function-server/pom.xml b/presto-function-server/pom.xml index 0aeb9796c61d5..4ab7dfc3bde8e 100644 --- a/presto-function-server/pom.xml +++ b/presto-function-server/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-function-server diff --git a/presto-geospatial-toolkit/pom.xml b/presto-geospatial-toolkit/pom.xml index fa63ba0c4ec90..223b67ff5a880 100644 --- a/presto-geospatial-toolkit/pom.xml +++ b/presto-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-geospatial-toolkit diff --git a/presto-google-sheets/pom.xml b/presto-google-sheets/pom.xml index 4c66baecd571b..ee96d33f01248 100644 --- a/presto-google-sheets/pom.xml +++ b/presto-google-sheets/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-google-sheets @@ -16,6 +16,7 @@ ${project.parent.basedir} true + 2.0.0 @@ -34,7 +35,7 @@ com.google.apis google-api-services-sheets - v4-rev516-1.23.0 + v4-rev20250616-2.0.0 com.google.guava @@ -89,7 +90,7 @@ com.google.oauth-client google-oauth-client - 1.33.3 + 1.39.0 com.google.http-client @@ -116,7 +117,7 @@ com.google.http-client google-http-client - 1.27.0 + ${dep.http-client.version} commons-logging @@ -128,7 +129,7 @@ com.google.http-client google-http-client-jackson2 - 1.27.0 + ${dep.http-client.version} @@ -145,7 +146,7 @@ com.google.api-client google-api-client - 1.27.0 + 2.8.0 com.google.guava diff --git a/presto-grpc-api/pom.xml b/presto-grpc-api/pom.xml index e515d699280ac..044d6565ecd3b 100644 --- a/presto-grpc-api/pom.xml +++ b/presto-grpc-api/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-grpc-testing-udf-server/pom.xml b/presto-grpc-testing-udf-server/pom.xml index dfabcd14dedc4..289a5017bb3be 100644 --- a/presto-grpc-testing-udf-server/pom.xml +++ b/presto-grpc-testing-udf-server/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-hana/pom.xml b/presto-hana/pom.xml index 5a5e082758353..0cd6d3ee5ac03 100644 --- a/presto-hana/pom.xml +++ b/presto-hana/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hana @@ -161,7 +161,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-hdfs-core/pom.xml b/presto-hdfs-core/pom.xml index d53ca803d71f7..5e54d8d488672 100644 --- a/presto-hdfs-core/pom.xml +++ b/presto-hdfs-core/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT diff --git a/presto-hive-common/pom.xml b/presto-hive-common/pom.xml index fce199c6f625c..af7ae4b18e1bd 100644 --- a/presto-hive-common/pom.xml +++ b/presto-hive-common/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT diff --git a/presto-hive-function-namespace/pom.xml b/presto-hive-function-namespace/pom.xml index c732fd58bd707..50a5ea7bcee5c 100644 --- a/presto-hive-function-namespace/pom.xml +++ b/presto-hive-function-namespace/pom.xml @@ -4,7 +4,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hive-function-namespace diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index ded8bef090bf5..4e61b35406c1a 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hive-hadoop2 diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 6294f16eb09fc..f372891e5851d 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hive-metastore diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index c7fdbef130e1b..ff115fdceddc4 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hive @@ -572,6 +572,9 @@ **/TestParquetDistributedQueries.java **/TestHive2InsertOverwrite.java **/TestHive3InsertOverwrite.java + **/TestHiveSslWithKeyStore.java + **/TestHiveSslWithTrustStore.java + **/TestHiveSslWithTrustStoreKeyStore.java diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractHiveSslTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractHiveSslTest.java index 77c9c5b81fa8d..84db4a79b163f 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractHiveSslTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractHiveSslTest.java @@ -26,6 +26,7 @@ import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException; import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE3_IMAGE; +import static com.facebook.presto.tests.SslKeystoreManager.initializeKeystoreAndTruststore; import static com.facebook.presto.tests.sql.TestTable.randomTableSuffix; import static java.lang.String.format; @@ -40,6 +41,7 @@ public abstract class AbstractHiveSslTest AbstractHiveSslTest(Map sslConfig) { + initializeKeystoreAndTruststore(); this.sslConfig = sslConfig; } @@ -75,8 +77,7 @@ public void close() closeAllRuntimeException(dockerizedS3DataLake); } - // TODO: these tests are disabled because they rely on an expired certificate - @Test(enabled = false) + @Test public void testInsertTable() { String testTable = getTestTableName(); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithKeyStore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithKeyStore.java index ca34cb665282f..3b01b14525b18 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithKeyStore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithKeyStore.java @@ -16,21 +16,20 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; -import java.net.URISyntaxException; -import java.nio.file.Paths; +import static com.facebook.presto.tests.SslKeystoreManager.SSL_STORE_PASSWORD; +import static com.facebook.presto.tests.SslKeystoreManager.getKeystorePath; -// TODO: these tests are disabled because they rely on an expired certificate -@Test(enabled = false) +@Test public class TestHiveSslWithKeyStore extends AbstractHiveSslTest { - TestHiveSslWithKeyStore() throws URISyntaxException + TestHiveSslWithKeyStore() { super(ImmutableMap.builder() // This is required when connecting to ssl enabled hms .put("hive.metastore.thrift.client.tls.enabled", "true") - .put("hive.metastore.thrift.client.tls.keystore-path", Paths.get((TestHiveSslWithKeyStore.class.getResource("/hive_ssl_enable/hive-metastore.jks")).toURI()).toFile().toString()) - .put("hive.metastore.thrift.client.tls.keystore-password", "123456") + .put("hive.metastore.thrift.client.tls.keystore-path", getKeystorePath()) + .put("hive.metastore.thrift.client.tls.keystore-password", SSL_STORE_PASSWORD) .build()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStore.java index ed7f097226205..c392283d93bfe 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStore.java @@ -16,21 +16,20 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; -import java.net.URISyntaxException; -import java.nio.file.Paths; +import static com.facebook.presto.tests.SslKeystoreManager.SSL_STORE_PASSWORD; +import static com.facebook.presto.tests.SslKeystoreManager.getTruststorePath; -// TODO: these tests are disabled because they rely on an expired certificate -@Test(enabled = false) +@Test public class TestHiveSslWithTrustStore extends AbstractHiveSslTest { - TestHiveSslWithTrustStore() throws URISyntaxException + TestHiveSslWithTrustStore() { super(ImmutableMap.builder() // This is required when connecting to ssl enabled hms .put("hive.metastore.thrift.client.tls.enabled", "true") - .put("hive.metastore.thrift.client.tls.truststore-path", Paths.get((TestHiveSslWithTrustStore.class.getResource("/hive_ssl_enable/hive-metastore-truststore.jks")).toURI()).toFile().toString()) - .put("hive.metastore.thrift.client.tls.truststore-password", "123456") + .put("hive.metastore.thrift.client.tls.truststore-path", getTruststorePath()) + .put("hive.metastore.thrift.client.tls.truststore-password", SSL_STORE_PASSWORD) .build()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStoreKeyStore.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStoreKeyStore.java index 4f28957150886..3bc978a711872 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStoreKeyStore.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSslWithTrustStoreKeyStore.java @@ -16,23 +16,23 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; -import java.net.URISyntaxException; -import java.nio.file.Paths; +import static com.facebook.presto.tests.SslKeystoreManager.SSL_STORE_PASSWORD; +import static com.facebook.presto.tests.SslKeystoreManager.getKeystorePath; +import static com.facebook.presto.tests.SslKeystoreManager.getTruststorePath; -// TODO: these tests are disabled because they rely on an expired certificate -@Test(enabled = false) +@Test public class TestHiveSslWithTrustStoreKeyStore extends AbstractHiveSslTest { - TestHiveSslWithTrustStoreKeyStore() throws URISyntaxException + TestHiveSslWithTrustStoreKeyStore() { super(ImmutableMap.builder() // This is required when connecting to ssl enabled hms .put("hive.metastore.thrift.client.tls.enabled", "true") - .put("hive.metastore.thrift.client.tls.keystore-path", Paths.get((TestHiveSslWithTrustStoreKeyStore.class.getResource("/hive_ssl_enable/hive-metastore.jks")).toURI()).toFile().toString()) - .put("hive.metastore.thrift.client.tls.keystore-password", "123456") - .put("hive.metastore.thrift.client.tls.truststore-path", Paths.get((TestHiveSslWithTrustStoreKeyStore.class.getResource("/hive_ssl_enable/hive-metastore-truststore.jks")).toURI()).toFile().toString()) - .put("hive.metastore.thrift.client.tls.truststore-password", "123456") + .put("hive.metastore.thrift.client.tls.keystore-path", getKeystorePath()) + .put("hive.metastore.thrift.client.tls.keystore-password", SSL_STORE_PASSWORD) + .put("hive.metastore.thrift.client.tls.truststore-path", getTruststorePath()) + .put("hive.metastore.thrift.client.tls.truststore-password", SSL_STORE_PASSWORD) .build()); } } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java index 417847f078071..0a0e5bf7a67f9 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java @@ -25,11 +25,18 @@ import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE3_IMAGE; +import static com.facebook.presto.tests.SslKeystoreManager.getKeystorePath; +import static com.facebook.presto.tests.SslKeystoreManager.getTruststorePath; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.Network.newNetwork; @@ -38,6 +45,7 @@ public class HiveMinIODataLake { public static final String ACCESS_KEY = "accesskey"; public static final String SECRET_KEY = "secretkey"; + private static final Object SSL_LOCK = new Object(); private final String bucketName; private final MinIOContainer minIOContainer; @@ -75,9 +83,31 @@ public HiveMinIODataLake(String bucketName, Map hiveHadoopFilesT } filesToMount.put("hive_s3_insert_overwrite/hadoop-core-site.xml", hadoopCoreSitePath); if (isSslEnabledTest) { - filesToMount.put("hive_ssl_enable/hive-site.xml", "/opt/hive/conf/hive-site.xml"); - filesToMount.put("hive_ssl_enable/hive-metastore.jks", "/opt/hive/conf/hive-metastore.jks"); - filesToMount.put("hive_ssl_enable/hive-metastore-truststore.jks", "/opt/hive/conf/hive-metastore-truststore.jks"); + try { + // Copy dynamically generated keystore files into target/test-classes so that + // Testcontainers can resolve them. + // Without this step, the files would only exist on the filesystem and not + // on the test runtime classpath, causing classpath lookups to fail. + Path targetDir = Paths.get("target", "test-classes", "ssl_enable"); + Files.createDirectories(targetDir); + + Path keyStoreTarget = targetDir.resolve("keystore.jks"); + Path trustStoreTarget = targetDir.resolve("truststore.jks"); + + synchronized (SSL_LOCK) { + // Copy freshly generated keystores, replacing if they exist + Files.copy(Paths.get(getKeystorePath()), keyStoreTarget, REPLACE_EXISTING); + Files.copy(Paths.get(getTruststorePath()), trustStoreTarget, REPLACE_EXISTING); + + filesToMount.put("ssl_enable/keystore.jks", "/opt/hive/conf/hive-metastore.jks"); + filesToMount.put("ssl_enable/truststore.jks", "/opt/hive/conf/hive-metastore-truststore.jks"); + } + + filesToMount.put("hive_ssl_enable/hive-site.xml", "/opt/hive/conf/hive-site.xml"); + } + catch (IOException e) { + throw new UncheckedIOException("Failed to prepare keystore files for Testcontainers", e); + } } this.hiveHadoopContainer = closer.register( HiveHadoopContainer.builder() diff --git a/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore-truststore.jks b/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore-truststore.jks deleted file mode 100644 index 6a36087b18109..0000000000000 Binary files a/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore-truststore.jks and /dev/null differ diff --git a/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore.jks b/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore.jks deleted file mode 100644 index ddbd0546beefb..0000000000000 Binary files a/presto-hive/src/test/resources/hive_ssl_enable/hive-metastore.jks and /dev/null differ diff --git a/presto-hudi/pom.xml b/presto-hudi/pom.xml index 0682e3f4ae51d..400acfb14fb07 100644 --- a/presto-hudi/pom.xml +++ b/presto-hudi/pom.xml @@ -4,7 +4,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-hudi presto-hudi diff --git a/presto-i18n-functions/pom.xml b/presto-i18n-functions/pom.xml index fd6ad2420772d..a38dc951a3cec 100644 --- a/presto-i18n-functions/pom.xml +++ b/presto-i18n-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-i18n-functions diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index f63cf3fc293e7..e3fc939ba9180 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-iceberg presto-iceberg diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java index 29b85a11e4fc1..f9023ee9da7b2 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/HiveTableOperations.java @@ -38,20 +38,24 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Sets; import jakarta.annotation.Nullable; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileOutputFormat; -import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.LocationProviders; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadata.MetadataLogEntry; import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.util.Tasks; import java.io.FileNotFoundException; @@ -63,13 +67,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Supplier; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.DELETE; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.INSERT; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.SELECT; import static com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege.UPDATE; import static com.facebook.presto.hive.metastore.MetastoreUtil.TABLE_COMMENT; import static com.facebook.presto.hive.metastore.MetastoreUtil.isPrestoView; +import static com.facebook.presto.iceberg.HiveTableOperations.CommitStatus.FAILED; +import static com.facebook.presto.iceberg.HiveTableOperations.CommitStatus.SUCCESS; +import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_COMMIT_ERROR; import static com.facebook.presto.iceberg.IcebergErrorCode.ICEBERG_INVALID_METADATA; import static com.facebook.presto.iceberg.IcebergUtil.isIcebergTable; import static com.facebook.presto.iceberg.IcebergUtil.toHiveColumns; @@ -81,8 +90,17 @@ import static java.util.Objects.requireNonNull; import static java.util.UUID.randomUUID; import static org.apache.iceberg.BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE; +import static org.apache.iceberg.BaseMetastoreTableOperations.METADATA_LOCATION_PROP; import static org.apache.iceberg.BaseMetastoreTableOperations.TABLE_TYPE_PROP; import static org.apache.iceberg.TableMetadataParser.getFileExtension; +import static org.apache.iceberg.TableProperties.COMMIT_NUM_STATUS_CHECKS; +import static org.apache.iceberg.TableProperties.COMMIT_NUM_STATUS_CHECKS_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MAX_WAIT_MS; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MIN_WAIT_MS; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS; +import static org.apache.iceberg.TableProperties.COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT; import static org.apache.iceberg.TableProperties.METADATA_COMPRESSION; import static org.apache.iceberg.TableProperties.METADATA_COMPRESSION_DEFAULT; import static org.apache.iceberg.TableProperties.WRITE_METADATA_LOCATION; @@ -322,12 +340,62 @@ public void commit(@Nullable TableMetadata base, TableMetadata metadata) .put(table.getOwner(), new HivePrivilegeInfo(DELETE, true, owner, owner)) .build(), ImmutableMultimap.of()); - if (base == null) { - metastore.createTable(metastoreContext, table, privileges, emptyList()); + try { + if (base == null) { + metastore.createTable(metastoreContext, table, privileges, emptyList()); + } + else { + PartitionStatistics tableStats = metastore.getTableStatistics(metastoreContext, database, tableName); + metastore.persistTable(metastoreContext, database, tableName, table, privileges, () -> tableStats, useHMSLock ? ImmutableMap.of() : hmsEnvContext(base.metadataFileLocation())); + } + } + catch (AlreadyExistsException e) { + throw new PrestoException(HIVE_METASTORE_ERROR, format("Table already exists: %s.%s", database, tableName), e); + } + catch (CommitFailedException | CommitStateUnknownException e) { + throw e; } - else { - PartitionStatistics tableStats = metastore.getTableStatistics(metastoreContext, database, tableName); - metastore.persistTable(metastoreContext, database, tableName, table, privileges, () -> tableStats, useHMSLock ? ImmutableMap.of() : hmsEnvContext(base.metadataFileLocation())); + catch (Throwable e) { + if (e instanceof PrestoException && e.getCause() instanceof InvalidObjectException) { + throw new ValidationException(e, "Invalid Hive object for %s.%s", database, tableName); + } + if (e.getMessage() != null + && e.getMessage().contains("Table/View 'HIVE_LOCKS' does not exist")) { + throw new PrestoException(ICEBERG_COMMIT_ERROR, + "Failed to acquire locks from metastore because the underlying metastore " + + "table 'HIVE_LOCKS' does not exist. This can occur when using an embedded metastore which does not " + + "support transactions. To fix this use an alternative metastore.", + e); + } + CommitStatus commitStatus; + if (e.getMessage() != null + && e.getMessage() + .contains( + "The table has been modified. The parameter value for key '" + + METADATA_LOCATION_PROP + + "' is")) { + // It's possible the HMS client incorrectly retries a successful operation, due to network + // issue for example, and triggers this exception. So we need double-check to make sure + // this is really a concurrent modification. Hitting this exception means no pending + // requests, if any, can succeed later, so it's safe to check status in strict mode + commitStatus = checkCommitStatusStrict(newMetadataLocation, metadata); + if (commitStatus == FAILED) { + throw new CommitFailedException( + e, "The table %s.%s has been modified concurrently", database, tableName); + } + } + else { + // Cannot tell if commit to succeeded, attempting to reconnect and check. + commitStatus = checkCommitStatus(newMetadataLocation, metadata); + } + switch (commitStatus) { + case SUCCESS: + break; + case FAILED: + throw e; + case UNKNOWN: + throw new CommitStateUnknownException(e); + } } deleteRemovedMetadataFiles(base, metadata); } @@ -511,7 +579,7 @@ private Map hmsEnvContext(String metadataLocation) { return ImmutableMap.of( org.apache.iceberg.hive.HiveTableOperations.NO_LOCK_EXPECTED_KEY, - BaseMetastoreTableOperations.METADATA_LOCATION_PROP, + METADATA_LOCATION_PROP, org.apache.iceberg.hive.HiveTableOperations.NO_LOCK_EXPECTED_VALUE, metadataLocation); } @@ -521,4 +589,74 @@ public IcebergHiveTableOperationsConfig getConfig() { return config; } + + /** + * Validate if the new metadata location is the current metadata location or present within + * previous metadata files. + * + * @param newMetadataLocation newly written metadata location + * @return true if the new metadata location is the current metadata location or present within + * previous metadata files. + */ + private boolean checkCurrentMetadataLocation(String newMetadataLocation) + { + TableMetadata metadata = refresh(); + String currentMetadataFileLocation = metadata.metadataFileLocation(); + return currentMetadataFileLocation.equals(newMetadataLocation) + || metadata.previousFiles().stream() + .anyMatch(log -> log.file().equals(newMetadataLocation)); + } + + protected CommitStatus checkCommitStatus(String newMetadataLocation, TableMetadata config) + { + CommitStatus strictStatus = + checkCommitStatusStrict(newMetadataLocation, config); + if (strictStatus == FAILED) { + return CommitStatus.UNKNOWN; + } + return strictStatus; + } + + protected CommitStatus checkCommitStatusStrict(String newMetadataLocation, TableMetadata config) + { + Supplier commitStatusSupplier = () -> checkCurrentMetadataLocation(newMetadataLocation); + + int maxAttempts = + PropertyUtil.propertyAsInt( + config.properties(), COMMIT_NUM_STATUS_CHECKS, COMMIT_NUM_STATUS_CHECKS_DEFAULT); + long minWaitMs = + PropertyUtil.propertyAsLong( + config.properties(), COMMIT_STATUS_CHECKS_MIN_WAIT_MS, COMMIT_STATUS_CHECKS_MIN_WAIT_MS_DEFAULT); + long maxWaitMs = + PropertyUtil.propertyAsLong( + config.properties(), COMMIT_STATUS_CHECKS_MAX_WAIT_MS, COMMIT_STATUS_CHECKS_MAX_WAIT_MS_DEFAULT); + long totalRetryMs = + PropertyUtil.propertyAsLong( + config.properties(), + COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS, + COMMIT_STATUS_CHECKS_TOTAL_WAIT_MS_DEFAULT); + + AtomicReference status = new AtomicReference(CommitStatus.UNKNOWN); + + Tasks.foreach(newMetadataLocation) + .retry(maxAttempts) + .suppressFailureWhenFinished() + .exponentialBackoff(minWaitMs, maxWaitMs, totalRetryMs, 2.0) + .run( + location -> { + boolean commitSuccess = commitStatusSupplier.get(); + + if (commitSuccess) { + status.set(SUCCESS); + } + else { + status.set(FAILED); + } + }); + return status.get(); + } + + public enum CommitStatus { + SUCCESS, FAILED, UNKNOWN + } } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java index ca23e7d760903..70665ca138653 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java @@ -1064,7 +1064,7 @@ public ConnectorDeleteTableHandle beginDelete(ConnectorSession session, Connecto } @Override - public void finishDelete(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) + public Optional finishDeleteWithOutput(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) { IcebergTableHandle handle = (IcebergTableHandle) tableHandle; Table icebergTable = getIcebergTable(session, handle.getSchemaTableName()); @@ -1107,6 +1107,7 @@ public void finishDelete(ConnectorSession session, ConnectorDeleteTableHandle ta rowDelta.commit(); transaction.commitTransaction(); + return Optional.empty(); } @Override diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java index ab469be8e83bc..2a2460238edff 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java @@ -125,7 +125,6 @@ import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.REGULAR; -import static com.facebook.presto.hive.BaseHiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.hive.CacheQuota.NO_CACHE_CONSTRAINTS; import static com.facebook.presto.hive.HiveCommonSessionProperties.getOrcLazyReadSmallRanges; import static com.facebook.presto.hive.HiveCommonSessionProperties.getOrcMaxBufferSize; @@ -172,7 +171,6 @@ import static com.facebook.presto.parquet.predicate.PredicateUtils.buildPredicate; import static com.facebook.presto.parquet.predicate.PredicateUtils.predicateMatches; import static com.facebook.presto.parquet.reader.ColumnIndexFilterUtils.getColumnIndexStore; -import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Predicates.not; import static com.google.common.base.Suppliers.memoize; @@ -425,18 +423,6 @@ public static Optional getColumnType( return Optional.ofNullable(parquetIdToField.get(column.getId())); } - private static HiveColumnHandle.ColumnType getHiveColumnHandleColumnType(IcebergColumnHandle.ColumnType columnType) - { - switch (columnType) { - case REGULAR: - return REGULAR; - case SYNTHESIZED: - return SYNTHESIZED; - } - - throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unknown ColumnType: " + columnType); - } - private static TupleDomain getParquetTupleDomain(Map, RichColumnDescriptor> descriptorsByPath, TupleDomain effectivePredicate) { if (effectivePredicate.isNone()) { @@ -541,20 +527,13 @@ private static ConnectorPageSourceWithRowPositions createBatchOrcPageSource( List isRowPositionList = new ArrayList<>(); for (IcebergColumnHandle column : regularColumns) { IcebergOrcColumn icebergOrcColumn; - boolean isExcludeColumn = false; if (fileOrcColumnByIcebergId.isEmpty()) { + // This is a migrated table icebergOrcColumn = fileOrcColumnsByName.get(column.getName()); } else { icebergOrcColumn = fileOrcColumnByIcebergId.get(column.getId()); - if (icebergOrcColumn == null) { - // Cannot get orc column from 'fileOrcColumnByIcebergId', which means SchemaEvolution may have happened, so we get orc column by column name. - icebergOrcColumn = fileOrcColumnsByName.get(column.getName()); - if (icebergOrcColumn != null) { - isExcludeColumn = true; - } - } } if (icebergOrcColumn != null) { @@ -569,11 +548,8 @@ private static ConnectorPageSourceWithRowPositions createBatchOrcPageSource( Optional.empty()); physicalColumnHandles.add(columnHandle); - // Skip SchemaEvolution column - if (!isExcludeColumn) { - includedColumns.put(columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature())); - columnReferences.add(new TupleDomainOrcPredicate.ColumnReference<>(columnHandle, columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature()))); - } + includedColumns.put(columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature())); + columnReferences.add(new TupleDomainOrcPredicate.ColumnReference<>(columnHandle, columnHandle.getHiveColumnIndex(), typeManager.getType(columnHandle.getTypeSignature()))); } else { physicalColumnHandles.add(new HiveColumnHandle( @@ -581,7 +557,7 @@ private static ConnectorPageSourceWithRowPositions createBatchOrcPageSource( toHiveType(column.getType()), column.getType().getTypeSignature(), nextMissingColumnIndex++, - getHiveColumnHandleColumnType(column.getColumnType()), + column.getColumnType(), column.getComment(), column.getRequiredSubfields(), Optional.empty())); diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableProperties.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableProperties.java index a38cc12addc59..8a4473d94970d 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableProperties.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergTableProperties.java @@ -44,6 +44,8 @@ import static java.lang.String.format; import static java.util.Locale.ENGLISH; import static org.apache.iceberg.TableProperties.COMMIT_NUM_RETRIES; +import static org.apache.iceberg.TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED; +import static org.apache.iceberg.TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS; import static org.apache.iceberg.TableProperties.UPDATE_MODE; import static org.apache.iceberg.TableProperties.WRITE_DATA_LOCATION; @@ -99,14 +101,18 @@ public class IcebergTableProperties .put(COMMIT_RETRIES, TableProperties.COMMIT_NUM_RETRIES) .put(DELETE_MODE, TableProperties.DELETE_MODE) .put(METADATA_PREVIOUS_VERSIONS_MAX, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX) - .put(METADATA_DELETE_AFTER_COMMIT, TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED) - .put(METRICS_MAX_INFERRED_COLUMN, TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS) + .put(METADATA_DELETE_AFTER_COMMIT, METADATA_DELETE_AFTER_COMMIT_ENABLED) + .put(METRICS_MAX_INFERRED_COLUMN, METRICS_MAX_INFERRED_COLUMN_DEFAULTS) .build(); private static final Set UPDATABLE_PROPERTIES = ImmutableSet.builder() .add(COMMIT_RETRIES) .add(COMMIT_NUM_RETRIES) .add(TARGET_SPLIT_SIZE) + .add(METADATA_DELETE_AFTER_COMMIT) + .add(METADATA_DELETE_AFTER_COMMIT_ENABLED) + .add(METADATA_PREVIOUS_VERSIONS_MAX) + .add(TableProperties.METADATA_PREVIOUS_VERSIONS_MAX) .build(); private static final String DEFAULT_FORMAT_VERSION = "2"; @@ -183,12 +189,12 @@ public IcebergTableProperties(IcebergConfig icebergConfig) icebergConfig.getMetadataPreviousVersionsMax(), false)) .add(booleanProperty( - TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED, + METADATA_DELETE_AFTER_COMMIT_ENABLED, "Whether enables to delete the oldest metadata file after commit", icebergConfig.isMetadataDeleteAfterCommit(), false)) .add(integerProperty( - TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, + METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "The maximum number of columns for which metrics are collected", icebergConfig.getMetricsMaxInferredColumn(), false)) @@ -315,12 +321,12 @@ public Integer getMetadataPreviousVersionsMax(ConnectorSession session, Map tableProperties) { - return (Boolean) getTablePropertyWithDeprecationWarning(session, tableProperties, TableProperties.METADATA_DELETE_AFTER_COMMIT_ENABLED); + return (Boolean) getTablePropertyWithDeprecationWarning(session, tableProperties, METADATA_DELETE_AFTER_COMMIT_ENABLED); } public Integer getMetricsMaxInferredColumn(ConnectorSession session, Map tableProperties) { - return (Integer) getTablePropertyWithDeprecationWarning(session, tableProperties, TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS); + return (Integer) getTablePropertyWithDeprecationWarning(session, tableProperties, METRICS_MAX_INFERRED_COLUMN_DEFAULTS); } public RowLevelOperationMode getUpdateMode(Map tableProperties) diff --git a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java index 6c548619b17a1..5335a72caedc6 100644 --- a/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java +++ b/presto-iceberg/src/test/java/com/facebook/presto/iceberg/IcebergDistributedTestBase.java @@ -314,6 +314,26 @@ public void testRenameIdentityPartitionColumn() assertQuerySucceeds("DROP TABLE test_partitioned_table"); } + @Test(dataProvider = "fileFormat") + public void testQueryOnSchemaEvolution(String fileFormat) + { + String tableName = "test_query_on_schema_evolution_" + randomTableSuffix(); + assertUpdate("CREATE TABLE " + tableName + "(a int, b varchar) with (\"write.format.default\" = '" + fileFormat + "')"); + assertUpdate("INSERT INTO " + tableName + " VALUES(1, '1001'), (2, '1002')", 2); + assertUpdate("ALTER TABLE " + tableName + " RENAME COLUMN a to a2"); + assertQuery("SELECT * FROM " + tableName, "VALUES(1, '1001'), (2, '1002')"); + + assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN a varchar"); + assertQuery("SELECT * FROM " + tableName, "VALUES(1, '1001', NULL), (2, '1002', NULL)"); + + assertUpdate("ALTER TABLE " + tableName + " DROP COLUMN a"); + assertQuery("SELECT * FROM " + tableName, "VALUES(1, '1001'), (2, '1002')"); + assertUpdate("ALTER TABLE " + tableName + " ADD COLUMN a int"); + assertQuery("SELECT * FROM " + tableName, "VALUES(1, '1001', NULL), (2, '1002', NULL)"); + + assertUpdate("DROP TABLE " + tableName); + } + @DataProvider(name = "transforms") public String[][] transforms() { @@ -1927,6 +1947,52 @@ public void testMetadataVersionsMaintainingProperties() } } + @Test + public void testAlteringMetadataVersionsMaintainingProperties() + throws Exception + { + String alteringTableName = "test_table_with_altering_properties"; + try { + // Create a table with default table properties that maintain 100 previous metadata versions in current metadata, + // and do not automatically delete any metadata files + assertUpdate("CREATE TABLE " + alteringTableName + " (a INTEGER, b VARCHAR)"); + + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (1, '1001'), (2, '1002')", 2); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (3, '1003'), (4, '1004')", 2); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (5, '1005'), (6, '1006')", 2); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (7, '1007'), (8, '1008')", 2); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (9, '1009'), (10, '1010')", 2); + + Table targetTable = loadTable(alteringTableName); + TableMetadata currentTableMetadata = ((BaseTable) targetTable).operations().current(); + // Target table's current metadata record all 5 previous metadata files + assertEquals(currentTableMetadata.previousFiles().size(), 5); + + FileSystem fileSystem = getHdfsEnvironment().getFileSystem(new HdfsContext(SESSION), new Path(targetTable.location())); + // Target table's all existing metadata files count is 6 + FileStatus[] settingTableFiles = fileSystem.listStatus(new Path(targetTable.location(), "metadata"), name -> name.getName().contains(METADATA_FILE_EXTENSION)); + assertEquals(settingTableFiles.length, 6); + + // Alter the table to set properties that maintain only 1 previous metadata version in current metadata, + // and delete unuseful metadata files after each commit + assertUpdate("ALTER TABLE " + alteringTableName + " SET PROPERTIES(\"write.metadata.previous-versions-max\" = 1, \"write.metadata.delete-after-commit.enabled\" = true)"); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (11, '1011'), (12, '1012')", 2); + assertUpdate("INSERT INTO " + alteringTableName + " VALUES (13, '1013'), (14, '1014')", 2); + + targetTable = loadTable(alteringTableName); + currentTableMetadata = ((BaseTable) targetTable).operations().current(); + // Table `test_table_with_setting_properties`'s current metadata only record 1 previous metadata file + assertEquals(currentTableMetadata.previousFiles().size(), 1); + + // Target table's all existing metadata files count is 2 + FileStatus[] defaultTableFiles = fileSystem.listStatus(new Path(targetTable.location(), "metadata"), name -> name.getName().contains(METADATA_FILE_EXTENSION)); + assertEquals(defaultTableFiles.length, 2); + } + finally { + assertUpdate("DROP TABLE IF EXISTS " + alteringTableName); + } + } + @DataProvider(name = "batchReadEnabled") public Object[] batchReadEnabledReader() { diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index d5515e0d71b7c..6f201e1360873 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 06e8d3398d50d..b60b883e127a2 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 63d1c7ab81c77..a08db0d3cd1e8 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-kafka diff --git a/presto-kudu/pom.xml b/presto-kudu/pom.xml index 7fac357ab4982..369cfa7f3b12f 100644 --- a/presto-kudu/pom.xml +++ b/presto-kudu/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-kudu diff --git a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java index 603b82c56716d..b5a52a5898c06 100755 --- a/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java +++ b/presto-kudu/src/main/java/com/facebook/presto/kudu/KuduMetadata.java @@ -380,8 +380,9 @@ public ConnectorDeleteTableHandle beginDelete(ConnectorSession session, Connecto } @Override - public void finishDelete(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) + public Optional finishDeleteWithOutput(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) { + return Optional.empty(); } @Override diff --git a/presto-lark-sheets/pom.xml b/presto-lark-sheets/pom.xml index 3d3f48f22ff87..4ebbb57c9d8eb 100644 --- a/presto-lark-sheets/pom.xml +++ b/presto-lark-sheets/pom.xml @@ -3,7 +3,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index a3a7b172d3bb5..f4df44c84af06 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-local-file diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index 9c779dafd33c1..0a1d5ca8b2e68 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-main-base diff --git a/presto-main-base/src/main/java/com/facebook/presto/Session.java b/presto-main-base/src/main/java/com/facebook/presto/Session.java index 05e01e17e5355..0492f7d1bd300 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/Session.java +++ b/presto-main-base/src/main/java/com/facebook/presto/Session.java @@ -446,7 +446,8 @@ public Session beginTransactionId(TransactionId transactionId, boolean enableRol identity.getExtraCredentials(), identity.getExtraAuthenticators(), identity.getSelectedUser(), - identity.getReasonForSelect()), + identity.getReasonForSelect(), + identity.getCertificates()), source, catalog, schema, diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java index e81322959b865..509f7b5b7c668 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import jakarta.inject.Inject; @@ -61,6 +62,8 @@ private DDLDefinitionExecution( @Override protected ListenableFuture executeTask() { + accessControl.checkQueryIntegrity(stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, ImmutableMap.of(), ImmutableMap.of()); + return task.execute(statement, transactionManager, metadata, accessControl, stateMachine.getSession(), parameters, stateMachine.getWarningCollector(), query); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/SchedulerStatsTracker.java b/presto-main-base/src/main/java/com/facebook/presto/execution/SchedulerStatsTracker.java index 6a90b597fe44e..5abb0ef8aa36f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/SchedulerStatsTracker.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/SchedulerStatsTracker.java @@ -28,10 +28,25 @@ public void recordTaskPlanSerializedCpuTime(long nanos) {} @Override public void recordEventLoopMethodExecutionCpuTime(long nanos) {} + + @Override + public void recordDeliveredUpdates(int updates) {} + + @Override + public void recordRoundTripTime(long nanos) {} + + @Override + public void recordStartWaitForEventLoop(long nanos) {} }; void recordTaskUpdateDeliveredTime(long nanos); + void recordDeliveredUpdates(int updates); + + void recordRoundTripTime(long nanos); + + void recordStartWaitForEventLoop(long nanos); + void recordTaskUpdateSerializedCpuTime(long nanos); void recordTaskPlanSerializedCpuTime(long nanos); diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java b/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java index c84b6d4c2718b..faa589d46b42e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import jakarta.inject.Inject; @@ -61,6 +62,8 @@ private SessionDefinitionExecution( @Override protected ListenableFuture executeTask() { + accessControl.checkQueryIntegrity(stateMachine.getSession().getIdentity(), stateMachine.getSession().getAccessControlContext(), query, ImmutableMap.of(), ImmutableMap.of()); + return task.execute(statement, transactionManager, metadata, accessControl, stateMachine, parameters, query); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java b/presto-main-base/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java index 96fe14bf310c0..ff6497f1eac57 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java @@ -15,6 +15,7 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.stats.Distribution; +import com.facebook.presto.common.RuntimeMetricName; import com.facebook.presto.common.RuntimeStats; import com.facebook.presto.execution.StateMachine.StateChangeListener; import com.facebook.presto.execution.scheduler.ScheduleResult; @@ -46,9 +47,11 @@ import static com.facebook.presto.common.RuntimeMetricName.SCHEDULER_CPU_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.SCHEDULER_WALL_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.TASK_PLAN_SERIALIZED_CPU_TIME_NANOS; +import static com.facebook.presto.common.RuntimeMetricName.TASK_START_WAIT_FOR_EVENT_LOOP; import static com.facebook.presto.common.RuntimeMetricName.TASK_UPDATE_DELIVERED_WALL_TIME_NANOS; import static com.facebook.presto.common.RuntimeMetricName.TASK_UPDATE_SERIALIZED_CPU_TIME_NANOS; import static com.facebook.presto.common.RuntimeUnit.NANO; +import static com.facebook.presto.common.RuntimeUnit.NONE; import static com.facebook.presto.execution.StageExecutionState.ABORTED; import static com.facebook.presto.execution.StageExecutionState.CANCELED; import static com.facebook.presto.execution.StageExecutionState.FAILED; @@ -431,6 +434,22 @@ public void recordTaskUpdateDeliveredTime(long nanos) runtimeStats.addMetricValue(TASK_UPDATE_DELIVERED_WALL_TIME_NANOS, NANO, max(nanos, 0)); } + @Override + public void recordStartWaitForEventLoop(long nanos) + { + runtimeStats.addMetricValue(TASK_START_WAIT_FOR_EVENT_LOOP, NANO, max(nanos, 0)); + } + + public void recordDeliveredUpdates(int updates) + { + runtimeStats.addMetricValue(RuntimeMetricName.TASK_UPDATE_DELIVERED_UPDATES, NONE, max(updates, 0)); + } + + public void recordRoundTripTime(long nanos) + { + runtimeStats.addMetricValue(RuntimeMetricName.TASK_UPDATE_ROUND_TRIP_TIME, NANO, max(nanos, 0)); + } + @Override public void recordTaskUpdateSerializedCpuTime(long nanos) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java b/presto-main-base/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java index 4b9dd6097aa0e..5e906b430cb3c 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/executor/TaskExecutor.java @@ -67,7 +67,6 @@ import java.util.function.Predicate; import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; -import static com.facebook.airlift.concurrent.Threads.threadsNamed; import static com.facebook.presto.execution.executor.MultilevelSplitQueue.computeLevel; import static com.facebook.presto.util.MoreMath.min; import static com.google.common.base.MoreObjects.toStringHelper; @@ -266,7 +265,7 @@ public TaskExecutor( checkArgument(interruptSplitInterval.getValue(SECONDS) >= 1.0, "interruptSplitInterval must be at least 1 second"); // we manage thread pool size directly, so create an unlimited pool - this.executor = newCachedThreadPool(threadsNamed("task-processor-%s")); + this.executor = newCachedThreadPool(daemonThreadsNamed("task-processor-%s")); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executor); this.runnerThreads = runnerThreads; this.embedVersion = requireNonNull(embedVersion, "embedVersion is null"); diff --git a/presto-main-base/src/main/java/com/facebook/presto/execution/scheduler/nodeSelection/SimpleNodeSelector.java b/presto-main-base/src/main/java/com/facebook/presto/execution/scheduler/nodeSelection/SimpleNodeSelector.java index 3f9526b064d98..3345ce52c5be0 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/execution/scheduler/nodeSelection/SimpleNodeSelector.java +++ b/presto-main-base/src/main/java/com/facebook/presto/execution/scheduler/nodeSelection/SimpleNodeSelector.java @@ -256,13 +256,11 @@ private ToLongFunction createTaskLoadSplitWeightProvider(List fragments) + public Optional finishDeleteWithOutput(Session session, DeleteTableHandle tableHandle, Collection fragments) { - delegate.finishDelete(session, tableHandle, fragments); + return delegate.finishDeleteWithOutput(session, tableHandle, fragments); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java index 7547a8cbd26cd..e1527797bc129 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java @@ -61,6 +61,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; import com.facebook.presto.sql.analyzer.FunctionsConfig; +import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.sql.gen.CacheStatsMBean; import com.facebook.presto.sql.tree.QualifiedName; @@ -80,6 +81,7 @@ import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -100,6 +102,7 @@ import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.CastType.toOperatorType; import static com.facebook.presto.metadata.FunctionSignatureMatcher.constructFunctionNotFoundErrorMessage; +import static com.facebook.presto.metadata.FunctionSignatureMatcher.decideAndThrow; import static com.facebook.presto.metadata.SessionFunctionHandle.SESSION_NAMESPACE; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; import static com.facebook.presto.spi.StandardErrorCode.AMBIGUOUS_FUNCTION_CALL; @@ -1044,12 +1047,58 @@ private FunctionHandle getMatchingFunctionHandle( List parameterTypes, boolean coercionAllowed) { - Optional matchingDefaultFunctionSignature = - getMatchingFunction(functionNamespaceManager.getFunctions(transactionHandle, functionName), parameterTypes, coercionAllowed); - Optional matchingPluginFunctionSignature = - getMatchingFunction(builtInPluginFunctionNamespaceManager.getFunctions(transactionHandle, functionName), parameterTypes, coercionAllowed); - Optional matchingWorkerFunctionSignature = - getMatchingFunction(builtInWorkerFunctionNamespaceManager.getFunctions(transactionHandle, functionName), parameterTypes, coercionAllowed); + boolean foundMatch = false; + List exceptions = new ArrayList<>(); + List allCandidates = new ArrayList<>(); + Optional matchingDefaultFunctionSignature = Optional.empty(); + Optional matchingPluginFunctionSignature = Optional.empty(); + Optional matchingWorkerFunctionSignature = Optional.empty(); + + try { + Collection defaultCandidates = functionNamespaceManager.getFunctions(transactionHandle, functionName); + allCandidates.addAll(defaultCandidates); + matchingDefaultFunctionSignature = + getMatchingFunction(defaultCandidates, parameterTypes, coercionAllowed); + if (matchingDefaultFunctionSignature.isPresent()) { + foundMatch = true; + } + } + catch (SemanticException e) { + exceptions.add(e); + } + + try { + Collection pluginCandidates = builtInPluginFunctionNamespaceManager.getFunctions(transactionHandle, functionName); + allCandidates.addAll(pluginCandidates); + matchingPluginFunctionSignature = + getMatchingFunction(pluginCandidates, parameterTypes, coercionAllowed); + if (matchingPluginFunctionSignature.isPresent()) { + foundMatch = true; + } + } + catch (SemanticException e) { + exceptions.add(e); + } + + try { + Collection workerCandidates = builtInWorkerFunctionNamespaceManager.getFunctions(transactionHandle, functionName); + allCandidates.addAll(workerCandidates); + matchingWorkerFunctionSignature = + getMatchingFunction(workerCandidates, parameterTypes, coercionAllowed); + if (matchingWorkerFunctionSignature.isPresent()) { + foundMatch = true; + } + } + catch (SemanticException e) { + exceptions.add(e); + } + + if (!foundMatch && !exceptions.isEmpty()) { + decideAndThrow(exceptions, + allCandidates.stream().findFirst() + .map(function -> function.getSignature().getName().getObjectName()) + .orElse("")); + } if (matchingDefaultFunctionSignature.isPresent() && matchingPluginFunctionSignature.isPresent()) { throw new PrestoException(AMBIGUOUS_FUNCTION_CALL, format("Function '%s' has two matching signatures. Please specify parameter types. \n" + diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java index 46d34dc13d61b..bba1d4fb1e1b4 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java @@ -338,7 +338,7 @@ private static boolean returnsNullOnGivenInputTypes(ApplicableFunction applicabl * If there's only one SemanticException, it throws that SemanticException directly. * If there are multiple SemanticExceptions, it throws the SignatureMatchingException. */ - private static void decideAndThrow(List failedExceptions, String functionName) + public static void decideAndThrow(List failedExceptions, String functionName) throws SemanticException { if (failedExceptions.size() == 1) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java index f53d1b7022746..050702ff6ad38 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -339,7 +339,7 @@ public interface Metadata /** * Finish delete query */ - void finishDelete(Session session, DeleteTableHandle tableHandle, Collection fragments); + Optional finishDeleteWithOutput(Session session, DeleteTableHandle tableHandle, Collection fragments); /** * Begin update query diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java index 6e714c5013f3e..a1be6217dbb60 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -953,11 +953,11 @@ public DeleteTableHandle beginDelete(Session session, TableHandle tableHandle) } @Override - public void finishDelete(Session session, DeleteTableHandle tableHandle, Collection fragments) + public Optional finishDeleteWithOutput(Session session, DeleteTableHandle tableHandle, Collection fragments) { ConnectorId connectorId = tableHandle.getConnectorId(); ConnectorMetadata metadata = getMetadata(session, connectorId); - metadata.finishDelete(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); + return metadata.finishDeleteWithOutput(session.toConnectorSession(connectorId), tableHandle.getConnectorHandle(), fragments); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/TableFunctionRegistry.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/TableFunctionRegistry.java index fb8b5d7e72b0e..da7e23f9bbf2b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/TableFunctionRegistry.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/TableFunctionRegistry.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.function.SchemaFunctionName; import com.facebook.presto.spi.function.table.ArgumentSpecification; import com.facebook.presto.spi.function.table.ConnectorTableFunction; +import com.facebook.presto.spi.function.table.ReturnTypeSpecification.DescribedTable; import com.facebook.presto.spi.function.table.TableArgumentSpecification; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.tree.QualifiedName; @@ -152,5 +153,10 @@ private static void validateTableFunction(ConnectorTableFunction tableFunction) // The 'keep when empty' or 'prune when empty' property must not be explicitly specified for a table argument with row semantics. // Such a table argument is implicitly 'prune when empty'. The TableArgumentSpecification.Builder enforces the 'prune when empty' property // for a table argument with row semantics. + + if (tableFunction.getReturnTypeSpecification() instanceof DescribedTable) { + DescribedTable describedTable = (DescribedTable) tableFunction.getReturnTypeSpecification(); + checkArgument(describedTable.getDescriptor().isTyped(), "field types missing in returned type specification"); + } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java b/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java index f0fde936968db..0b87c68bf3c9f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/security/AccessControlManager.java @@ -56,7 +56,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import static com.facebook.presto.metadata.MetadataUtil.toSchemaTableName; import static com.facebook.presto.spi.StandardErrorCode.INVALID_COLUMN_MASK; @@ -80,13 +79,14 @@ public class AccessControlManager private final Map systemAccessControlFactories = new ConcurrentHashMap<>(); private final Map connectorAccessControl = new ConcurrentHashMap<>(); - private final AtomicReference systemAccessControl = new AtomicReference<>(new InitializingSystemAccessControl()); + private final StatsRecordingSystemAccessControl systemAccessControl = new StatsRecordingSystemAccessControl(new InitializingSystemAccessControl()); private final AtomicBoolean systemAccessControlLoading = new AtomicBoolean(); private final CounterStat authenticationSuccess = new CounterStat(); private final CounterStat authenticationFail = new CounterStat(); private final CounterStat authorizationSuccess = new CounterStat(); private final CounterStat authorizationFail = new CounterStat(); + private StatsRecordingSystemAccessControl.Stats detailedStats = new StatsRecordingSystemAccessControl.Stats(); @Inject public AccessControlManager(TransactionManager transactionManager) @@ -95,6 +95,7 @@ public AccessControlManager(TransactionManager transactionManager) addSystemAccessControlFactory(new AllowAllSystemAccessControl.Factory()); addSystemAccessControlFactory(new ReadOnlySystemAccessControl.Factory()); addSystemAccessControlFactory(new FileBasedSystemAccessControl.Factory()); + addSystemAccessControlFactory(new DenyQueryIntegrityCheckSystemAccessControl.Factory()); } public void addSystemAccessControlFactory(SystemAccessControlFactory accessControlFactory) @@ -158,8 +159,7 @@ protected void setSystemAccessControl(String name, Map propertie SystemAccessControlFactory systemAccessControlFactory = systemAccessControlFactories.get(name); checkState(systemAccessControlFactory != null, "Access control %s is not registered", name); - SystemAccessControl systemAccessControl = systemAccessControlFactory.create(ImmutableMap.copyOf(properties)); - this.systemAccessControl.set(systemAccessControl); + systemAccessControl.updateDelegate(systemAccessControlFactory.create(ImmutableMap.copyOf(properties))); log.info("-- Loaded system access control %s --", name); } @@ -170,7 +170,7 @@ public void checkCanSetUser(Identity identity, AccessControlContext context, Opt requireNonNull(principal, "principal is null"); requireNonNull(userName, "userName is null"); - authenticationCheck(() -> systemAccessControl.get().checkCanSetUser(identity, context, principal, userName)); + authenticationCheck(() -> systemAccessControl.checkCanSetUser(identity, context, principal, userName)); } @Override @@ -179,7 +179,7 @@ public AuthorizedIdentity selectAuthorizedIdentity(Identity identity, AccessCont requireNonNull(userName, "userName is null"); requireNonNull(certificates, "certificates is null"); - return systemAccessControl.get().selectAuthorizedIdentity(identity, context, userName, certificates); + return systemAccessControl.selectAuthorizedIdentity(identity, context, userName, certificates); } @Override @@ -188,7 +188,7 @@ public void checkQueryIntegrity(Identity identity, AccessControlContext context, requireNonNull(identity, "identity is null"); requireNonNull(query, "query is null"); - authenticationCheck(() -> systemAccessControl.get().checkQueryIntegrity(identity, context, query, viewDefinitions, materializedViewDefinitions)); + authenticationCheck(() -> systemAccessControl.checkQueryIntegrity(identity, context, query, viewDefinitions, materializedViewDefinitions)); } @Override @@ -197,7 +197,7 @@ public Set filterCatalogs(Identity identity, AccessControlContext contex requireNonNull(identity, "identity is null"); requireNonNull(catalogs, "catalogs is null"); - return systemAccessControl.get().filterCatalogs(identity, context, catalogs); + return systemAccessControl.filterCatalogs(identity, context, catalogs); } @Override @@ -206,7 +206,7 @@ public void checkCanAccessCatalog(Identity identity, AccessControlContext contex requireNonNull(identity, "identity is null"); requireNonNull(catalogName, "catalog is null"); - authenticationCheck(() -> systemAccessControl.get().checkCanAccessCatalog(identity, context, catalogName)); + authenticationCheck(() -> systemAccessControl.checkCanAccessCatalog(identity, context, catalogName)); } @Override @@ -217,7 +217,7 @@ public void checkCanCreateSchema(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateSchema(identity, context, schemaName)); + authorizationCheck(() -> systemAccessControl.checkCanCreateSchema(identity, context, schemaName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); if (entry != null) { @@ -233,7 +233,7 @@ public void checkCanDropSchema(TransactionId transactionId, Identity identity, A authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropSchema(identity, context, schemaName)); + authorizationCheck(() -> systemAccessControl.checkCanDropSchema(identity, context, schemaName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); if (entry != null) { @@ -249,7 +249,7 @@ public void checkCanRenameSchema(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, schemaName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameSchema(identity, context, schemaName, newSchemaName)); + authorizationCheck(() -> systemAccessControl.checkCanRenameSchema(identity, context, schemaName, newSchemaName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schemaName.getCatalogName()); if (entry != null) { @@ -265,7 +265,7 @@ public void checkCanShowSchemas(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - authorizationCheck(() -> systemAccessControl.get().checkCanShowSchemas(identity, context, catalogName)); + authorizationCheck(() -> systemAccessControl.checkCanShowSchemas(identity, context, catalogName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); if (entry != null) { @@ -284,7 +284,7 @@ public Set filterSchemas(TransactionId transactionId, Identity identity, return ImmutableSet.of(); } - schemaNames = systemAccessControl.get().filterSchemas(identity, context, catalogName, schemaNames); + schemaNames = systemAccessControl.filterSchemas(identity, context, catalogName, schemaNames); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); if (entry != null) { @@ -301,7 +301,7 @@ public void checkCanShowCreateTable(TransactionId transactionId, Identity identi authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanShowCreateTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanShowCreateTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -317,7 +317,7 @@ public void checkCanCreateTable(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanCreateTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -333,7 +333,7 @@ public void checkCanDropTable(TransactionId transactionId, Identity identity, Ac authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanDropTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -350,7 +350,7 @@ public void checkCanRenameTable(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameTable(identity, context, toCatalogSchemaTableName(tableName), toCatalogSchemaTableName(newTableName))); + authorizationCheck(() -> systemAccessControl.checkCanRenameTable(identity, context, toCatalogSchemaTableName(tableName), toCatalogSchemaTableName(newTableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -364,7 +364,7 @@ public void checkCanSetTableProperties(TransactionId transactionId, Identity ide requireNonNull(identity, "identity is null"); requireNonNull(tableName, "tableName is null"); authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanSetTableProperties(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanSetTableProperties(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { authorizationCheck(() -> entry.getAccessControl().checkCanSetTableProperties(entry.getTransactionHandle(transactionId), identity.toConnectorIdentity(tableName.getCatalogName()), context, toSchemaTableName(tableName), properties)); @@ -379,7 +379,7 @@ public void checkCanShowTablesMetadata(TransactionId transactionId, Identity ide authenticationCheck(() -> checkCanAccessCatalog(identity, context, schema.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanShowTablesMetadata(identity, context, schema)); + authorizationCheck(() -> systemAccessControl.checkCanShowTablesMetadata(identity, context, schema)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, schema.getCatalogName()); if (entry != null) { @@ -398,7 +398,7 @@ public Set filterTables(TransactionId transactionId, Identity i return ImmutableSet.of(); } - tableNames = systemAccessControl.get().filterTables(identity, context, catalogName, tableNames); + tableNames = systemAccessControl.filterTables(identity, context, catalogName, tableNames); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); if (entry != null) { @@ -417,7 +417,7 @@ public void checkCanShowColumnsMetadata(TransactionId transactionId, Identity id authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanShowColumnsMetadata(identity, context, catalogSchemaTableName)); + authorizationCheck(() -> systemAccessControl.checkCanShowColumnsMetadata(identity, context, catalogSchemaTableName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -438,7 +438,7 @@ public List filterColumns(TransactionId transactionId, Identity return ImmutableList.of(); } - columns = systemAccessControl.get().filterColumns(identity, context, toCatalogSchemaTableName(tableName), columns); + columns = systemAccessControl.filterColumns(identity, context, toCatalogSchemaTableName(tableName), columns); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -455,7 +455,7 @@ public void checkCanAddColumns(TransactionId transactionId, Identity identity, A authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanAddColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanAddColumn(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -471,7 +471,7 @@ public void checkCanDropColumn(TransactionId transactionId, Identity identity, A authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanDropColumn(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -487,7 +487,7 @@ public void checkCanRenameColumn(TransactionId transactionId, Identity identity, authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameColumn(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanRenameColumn(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -503,7 +503,7 @@ public void checkCanInsertIntoTable(TransactionId transactionId, Identity identi authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanInsertIntoTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanInsertIntoTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -519,7 +519,7 @@ public void checkCanDeleteFromTable(TransactionId transactionId, Identity identi authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDeleteFromTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanDeleteFromTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -535,7 +535,7 @@ public void checkCanTruncateTable(TransactionId transactionId, Identity identity authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanTruncateTable(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanTruncateTable(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -551,7 +551,7 @@ public void checkCanUpdateTableColumns(TransactionId transactionId, Identity ide authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanUpdateTableColumns(identity, context, toCatalogSchemaTableName(tableName), updatedColumnNames)); + authorizationCheck(() -> systemAccessControl.checkCanUpdateTableColumns(identity, context, toCatalogSchemaTableName(tableName), updatedColumnNames)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -567,7 +567,7 @@ public void checkCanCreateView(TransactionId transactionId, Identity identity, A authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateView(identity, context, toCatalogSchemaTableName(viewName))); + authorizationCheck(() -> systemAccessControl.checkCanCreateView(identity, context, toCatalogSchemaTableName(viewName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); if (entry != null) { @@ -584,7 +584,7 @@ public void checkCanRenameView(TransactionId transactionId, Identity identity, A authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRenameView(identity, context, toCatalogSchemaTableName(viewName), toCatalogSchemaTableName(newViewName))); + authorizationCheck(() -> systemAccessControl.checkCanRenameView(identity, context, toCatalogSchemaTableName(viewName), toCatalogSchemaTableName(newViewName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); if (entry != null) { @@ -600,7 +600,7 @@ public void checkCanDropView(TransactionId transactionId, Identity identity, Acc authenticationCheck(() -> checkCanAccessCatalog(identity, context, viewName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropView(identity, context, toCatalogSchemaTableName(viewName))); + authorizationCheck(() -> systemAccessControl.checkCanDropView(identity, context, toCatalogSchemaTableName(viewName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, viewName.getCatalogName()); if (entry != null) { @@ -616,7 +616,7 @@ public void checkCanCreateViewWithSelectFromColumns(TransactionId transactionId, authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanCreateViewWithSelectFromColumns(identity, context, toCatalogSchemaTableName(tableName), columnNames)); + authorizationCheck(() -> systemAccessControl.checkCanCreateViewWithSelectFromColumns(identity, context, toCatalogSchemaTableName(tableName), columnNames)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -633,7 +633,7 @@ public void checkCanGrantTablePrivilege(TransactionId transactionId, Identity id authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanGrantTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), grantee, withGrantOption)); + authorizationCheck(() -> systemAccessControl.checkCanGrantTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), grantee, withGrantOption)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -650,7 +650,7 @@ public void checkCanRevokeTablePrivilege(TransactionId transactionId, Identity i authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanRevokeTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), revokee, grantOptionFor)); + authorizationCheck(() -> systemAccessControl.checkCanRevokeTablePrivilege(identity, context, privilege, toCatalogSchemaTableName(tableName), revokee, grantOptionFor)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -664,7 +664,7 @@ public void checkCanSetSystemSessionProperty(Identity identity, AccessControlCon requireNonNull(identity, "identity is null"); requireNonNull(propertyName, "propertyName is null"); - authorizationCheck(() -> systemAccessControl.get().checkCanSetSystemSessionProperty(identity, context, propertyName)); + authorizationCheck(() -> systemAccessControl.checkCanSetSystemSessionProperty(identity, context, propertyName)); } @Override @@ -676,7 +676,7 @@ public void checkCanSetCatalogSessionProperty(TransactionId transactionId, Ident authenticationCheck(() -> checkCanAccessCatalog(identity, context, catalogName)); - authorizationCheck(() -> systemAccessControl.get().checkCanSetCatalogSessionProperty(identity, context, catalogName, propertyName)); + authorizationCheck(() -> systemAccessControl.checkCanSetCatalogSessionProperty(identity, context, catalogName, propertyName)); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, catalogName); if (entry != null) { @@ -693,7 +693,7 @@ public void checkCanSelectFromColumns(TransactionId transactionId, Identity iden authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanSelectFromColumns( + authorizationCheck(() -> systemAccessControl.checkCanSelectFromColumns( identity, context, toCatalogSchemaTableName(tableName), @@ -835,7 +835,7 @@ public void checkCanDropConstraint(TransactionId transactionId, Identity identit authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanDropConstraint(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanDropConstraint(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -851,7 +851,7 @@ public void checkCanAddConstraints(TransactionId transactionId, Identity identit authenticationCheck(() -> checkCanAccessCatalog(identity, context, tableName.getCatalogName())); - authorizationCheck(() -> systemAccessControl.get().checkCanAddConstraint(identity, context, toCatalogSchemaTableName(tableName))); + authorizationCheck(() -> systemAccessControl.checkCanAddConstraint(identity, context, toCatalogSchemaTableName(tableName))); CatalogAccessControlEntry entry = getConnectorAccessControl(transactionId, tableName.getCatalogName()); if (entry != null) { @@ -873,7 +873,7 @@ public List getRowFilters(TransactionId transactionId, Identity .forEach(filters::add); } - systemAccessControl.get().getRowFilters(identity, context, toCatalogSchemaTableName(tableName)) + systemAccessControl.getRowFilters(identity, context, toCatalogSchemaTableName(tableName)) .forEach(filters::add); return filters.build(); @@ -896,7 +896,7 @@ public Map getColumnMasks(TransactionId transact columnMasksBuilder.putAll(connectorMasks); } - Map systemMasks = systemAccessControl.get().getColumnMasks(identity, context, toCatalogSchemaTableName(tableName), columns); + Map systemMasks = systemAccessControl.getColumnMasks(identity, context, toCatalogSchemaTableName(tableName), columns); columnMasksBuilder.putAll(systemMasks); try { @@ -942,6 +942,13 @@ public CounterStat getAuthorizationFail() return authorizationFail; } + @Managed + @Nested + public StatsRecordingSystemAccessControl.Stats getDetailedStats() + { + return systemAccessControl.getStats(); + } + private void authenticationCheck(Runnable runnable) { try { diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java new file mode 100644 index 0000000000000..2c912ce326c37 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/security/DenyQueryIntegrityCheckSystemAccessControl.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.security; + +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.CatalogSchemaTableName; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.spi.security.SystemAccessControl; +import com.facebook.presto.spi.security.SystemAccessControlFactory; + +import java.security.Principal; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.spi.security.AccessDeniedException.denyQueryIntegrityCheck; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class DenyQueryIntegrityCheckSystemAccessControl + implements SystemAccessControl +{ + public static final String NAME = "deny-query-integrity-check"; + + private static final DenyQueryIntegrityCheckSystemAccessControl INSTANCE = new DenyQueryIntegrityCheckSystemAccessControl(); + + public static class Factory + implements SystemAccessControlFactory + { + @Override + public String getName() + { + return NAME; + } + + @Override + public SystemAccessControl create(Map config) + { + requireNonNull(config, "config is null"); + checkArgument(config.isEmpty(), "This access controller does not support any configuration properties"); + return INSTANCE; + } + } + + @Override + public void checkQueryIntegrity(Identity identity, AccessControlContext context, String query, Map viewDefinitions, Map materializedViewDefinitions) + { + denyQueryIntegrityCheck(); + } + + @Override + public void checkCanSetUser(Identity identity, AccessControlContext context, Optional principal, String userName) + { + } + + @Override + public void checkCanSetSystemSessionProperty(Identity identity, AccessControlContext context, String propertyName) + { + } + + @Override + public void checkCanAccessCatalog(Identity identity, AccessControlContext context, String catalogName) + { + } + + @Override + public Set filterCatalogs(Identity identity, AccessControlContext context, Set catalogs) + { + return catalogs; + } + + @Override + public Set filterSchemas(Identity identity, AccessControlContext context, String catalogName, Set schemaNames) + { + return schemaNames; + } + + @Override + public void checkCanCreateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + } + + @Override + public void checkCanShowCreateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + } + + @Override + public void checkCanSelectFromColumns(Identity identity, AccessControlContext context, CatalogSchemaTableName table, Set columns) + { + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java b/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java new file mode 100644 index 0000000000000..6d408afc4bec8 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/security/StatsRecordingSystemAccessControl.java @@ -0,0 +1,1082 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.security; + +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.RuntimeUnit; +import com.facebook.presto.spi.CatalogSchemaTableName; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.analyzer.ViewDefinition; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.AuthorizedIdentity; +import com.facebook.presto.spi.security.Identity; +import com.facebook.presto.spi.security.PrestoPrincipal; +import com.facebook.presto.spi.security.Privilege; +import com.facebook.presto.spi.security.SystemAccessControl; +import com.facebook.presto.spi.security.ViewExpression; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import java.security.Principal; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Objects.requireNonNull; + +public final class StatsRecordingSystemAccessControl + implements SystemAccessControl +{ + private final Stats stats = new Stats(); + private final AtomicReference delegate = new AtomicReference<>(); + + public StatsRecordingSystemAccessControl(SystemAccessControl delegate) + { + updateDelegate(delegate); + } + + public void updateDelegate(SystemAccessControl delegate) + { + this.delegate.set(requireNonNull(delegate, "delegate is null")); + } + + public Stats getStats() + { + return stats; + } + + @Override + public void checkCanSetUser(Identity identity, AccessControlContext context, Optional principal, String userName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanSetUser(identity, context, principal, userName); + } + catch (RuntimeException e) { + stats.checkCanSetUser.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanSetUser", RuntimeUnit.NANO, duration); + stats.checkCanSetUser.record(duration); + } + } + + @Override + public AuthorizedIdentity selectAuthorizedIdentity(Identity identity, AccessControlContext context, String userName, List certificates) + { + long start = System.nanoTime(); + try { + return delegate.get().selectAuthorizedIdentity(identity, context, userName, certificates); + } + catch (RuntimeException e) { + stats.selectAuthorizedIdentity.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.selectAuthorizedIdentity", RuntimeUnit.NANO, duration); + stats.selectAuthorizedIdentity.record(duration); + } + } + + @Override + public void checkQueryIntegrity(Identity identity, AccessControlContext context, String query, Map viewDefinitions, Map materializedViewDefinitions) + { + long start = System.nanoTime(); + try { + delegate.get().checkQueryIntegrity(identity, context, query, viewDefinitions, materializedViewDefinitions); + } + catch (RuntimeException e) { + stats.checkQueryIntegrity.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkQueryIntegrity", RuntimeUnit.NANO, duration); + stats.checkQueryIntegrity.record(duration); + } + } + + @Override + public void checkCanSetSystemSessionProperty(Identity identity, AccessControlContext context, String propertyName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanSetSystemSessionProperty(identity, context, propertyName); + } + catch (RuntimeException e) { + stats.checkCanSetSystemSessionProperty.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanSetSystemSessionProperty", RuntimeUnit.NANO, duration); + stats.checkCanSetSystemSessionProperty.record(duration); + } + } + + @Override + public void checkCanAccessCatalog(Identity identity, AccessControlContext context, String catalogName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanAccessCatalog(identity, context, catalogName); + } + catch (RuntimeException e) { + stats.checkCanAccessCatalog.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanAccessCatalog", RuntimeUnit.NANO, duration); + stats.checkCanAccessCatalog.record(duration); + } + } + + @Override + public Set filterCatalogs(Identity identity, AccessControlContext context, Set catalogs) + { + long start = System.nanoTime(); + try { + return delegate.get().filterCatalogs(identity, context, catalogs); + } + catch (RuntimeException e) { + stats.filterCatalogs.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.filterCatalogs", RuntimeUnit.NANO, duration); + stats.filterCatalogs.record(duration); + } + } + + @Override + public void checkCanCreateSchema(Identity identity, AccessControlContext context, CatalogSchemaName schema) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateSchema(identity, context, schema); + } + catch (RuntimeException e) { + stats.checkCanCreateSchema.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateSchema", RuntimeUnit.NANO, duration); + stats.checkCanCreateSchema.record(duration); + } + } + + @Override + public void checkCanDropSchema(Identity identity, AccessControlContext context, CatalogSchemaName schema) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDropSchema(identity, context, schema); + } + catch (RuntimeException e) { + stats.checkCanDropSchema.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDropSchema", RuntimeUnit.NANO, duration); + stats.checkCanDropSchema.record(duration); + } + } + + @Override + public void checkCanRenameSchema(Identity identity, AccessControlContext context, CatalogSchemaName schema, String newSchemaName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanRenameSchema(identity, context, schema, newSchemaName); + } + catch (RuntimeException e) { + stats.checkCanRenameSchema.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanRenameSchema", RuntimeUnit.NANO, duration); + stats.checkCanRenameSchema.record(duration); + } + } + + @Override + public void checkCanShowSchemas(Identity identity, AccessControlContext context, String catalogName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanShowSchemas(identity, context, catalogName); + } + catch (RuntimeException e) { + stats.checkCanShowSchemas.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanShowSchemas", RuntimeUnit.NANO, duration); + stats.checkCanShowSchemas.record(duration); + } + } + + @Override + public Set filterSchemas(Identity identity, AccessControlContext context, String catalogName, Set schemaNames) + { + long start = System.nanoTime(); + try { + return delegate.get().filterSchemas(identity, context, catalogName, schemaNames); + } + catch (RuntimeException e) { + stats.filterSchemas.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.filterSchemas", RuntimeUnit.NANO, duration); + stats.filterSchemas.record(duration); + } + } + + @Override + public void checkCanShowCreateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanShowCreateTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanShowCreateTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanShowCreateTable", RuntimeUnit.NANO, duration); + stats.checkCanShowCreateTable.record(duration); + } + } + + @Override + public void checkCanCreateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanCreateTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateTable", RuntimeUnit.NANO, duration); + stats.checkCanCreateTable.record(duration); + } + } + + @Override + public void checkCanSetTableProperties(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanSetTableProperties(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanSetTableProperties.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanSetTableProperties", RuntimeUnit.NANO, duration); + stats.checkCanSetTableProperties.record(duration); + } + } + + @Override + public void checkCanDropTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDropTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanDropTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDropTable", RuntimeUnit.NANO, duration); + stats.checkCanDropTable.record(duration); + } + } + + @Override + public void checkCanRenameTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table, CatalogSchemaTableName newTable) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanRenameTable(identity, context, table, newTable); + } + catch (RuntimeException e) { + stats.checkCanRenameTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanRenameTable", RuntimeUnit.NANO, duration); + stats.checkCanRenameTable.record(duration); + } + } + + @Override + public void checkCanShowTablesMetadata(Identity identity, AccessControlContext context, CatalogSchemaName schema) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanShowTablesMetadata(identity, context, schema); + } + catch (RuntimeException e) { + stats.checkCanShowTablesMetadata.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanShowTablesMetadata", RuntimeUnit.NANO, duration); + stats.checkCanShowTablesMetadata.record(duration); + } + } + + @Override + public Set filterTables(Identity identity, AccessControlContext context, String catalogName, Set tableNames) + { + long start = System.nanoTime(); + try { + return delegate.get().filterTables(identity, context, catalogName, tableNames); + } + catch (RuntimeException e) { + stats.filterTables.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.filterTables", RuntimeUnit.NANO, duration); + stats.filterTables.record(duration); + } + } + + @Override + public void checkCanShowColumnsMetadata(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanShowColumnsMetadata(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanShowColumnsMetadata.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanShowColumnsMetadata", RuntimeUnit.NANO, duration); + stats.checkCanShowColumnsMetadata.record(duration); + } + } + + @Override + public List filterColumns(Identity identity, AccessControlContext context, CatalogSchemaTableName table, List columns) + { + long start = System.nanoTime(); + try { + return delegate.get().filterColumns(identity, context, table, columns); + } + catch (RuntimeException e) { + stats.filterColumns.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.filterColumns", RuntimeUnit.NANO, duration); + stats.filterColumns.record(duration); + } + } + + @Override + public void checkCanAddColumn(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanAddColumn(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanAddColumn.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanAddColumn", RuntimeUnit.NANO, duration); + stats.checkCanAddColumn.record(duration); + } + } + + @Override + public void checkCanDropColumn(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDropColumn(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanDropColumn.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDropColumn", RuntimeUnit.NANO, duration); + stats.checkCanDropColumn.record(duration); + } + } + + @Override + public void checkCanRenameColumn(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanRenameColumn(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanRenameColumn.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanRenameColumn", RuntimeUnit.NANO, duration); + stats.checkCanRenameColumn.record(duration); + } + } + + @Override + public void checkCanSelectFromColumns(Identity identity, AccessControlContext context, CatalogSchemaTableName table, Set columns) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanSelectFromColumns(identity, context, table, columns); + } + catch (RuntimeException e) { + stats.checkCanSelectFromColumns.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanSelectFromColumns", RuntimeUnit.NANO, duration); + stats.checkCanSelectFromColumns.record(duration); + } + } + + @Override + public void checkCanInsertIntoTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanInsertIntoTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanInsertIntoTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanInsertIntoTable", RuntimeUnit.NANO, duration); + stats.checkCanInsertIntoTable.record(duration); + } + } + + @Override + public void checkCanDeleteFromTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDeleteFromTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanDeleteFromTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDeleteFromTable", RuntimeUnit.NANO, duration); + stats.checkCanDeleteFromTable.record(duration); + } + } + + @Override + public void checkCanTruncateTable(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanTruncateTable(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanTruncateTable.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanTruncateTable", RuntimeUnit.NANO, duration); + stats.checkCanTruncateTable.record(duration); + } + } + + @Override + public void checkCanUpdateTableColumns(Identity identity, AccessControlContext context, CatalogSchemaTableName table, Set updatedColumnNames) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanUpdateTableColumns(identity, context, table, updatedColumnNames); + } + catch (RuntimeException e) { + stats.checkCanUpdateTableColumns.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanUpdateTableColumns", RuntimeUnit.NANO, duration); + stats.checkCanUpdateTableColumns.record(duration); + } + } + + @Override + public void checkCanCreateView(Identity identity, AccessControlContext context, CatalogSchemaTableName view) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateView(identity, context, view); + } + catch (RuntimeException e) { + stats.checkCanCreateView.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateView", RuntimeUnit.NANO, duration); + stats.checkCanCreateView.record(duration); + } + } + + @Override + public void checkCanRenameView(Identity identity, AccessControlContext context, CatalogSchemaTableName view, CatalogSchemaTableName newView) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanRenameView(identity, context, view, newView); + } + catch (RuntimeException e) { + stats.checkCanRenameView.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanRenameView", RuntimeUnit.NANO, duration); + stats.checkCanRenameView.record(duration); + } + } + + @Override + public void checkCanDropView(Identity identity, AccessControlContext context, CatalogSchemaTableName view) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDropView(identity, context, view); + } + catch (RuntimeException e) { + stats.checkCanDropView.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDropView", RuntimeUnit.NANO, duration); + stats.checkCanDropView.record(duration); + } + } + + @Override + public void checkCanCreateViewWithSelectFromColumns(Identity identity, AccessControlContext context, CatalogSchemaTableName table, Set columns) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanCreateViewWithSelectFromColumns(identity, context, table, columns); + } + catch (RuntimeException e) { + stats.checkCanCreateViewWithSelectFromColumns.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanCreateViewWithSelectFromColumns", RuntimeUnit.NANO, duration); + stats.checkCanCreateViewWithSelectFromColumns.record(duration); + } + } + + @Override + public void checkCanSetCatalogSessionProperty(Identity identity, AccessControlContext context, String catalogName, String propertyName) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanSetCatalogSessionProperty(identity, context, catalogName, propertyName); + } + catch (RuntimeException e) { + stats.checkCanSetCatalogSessionProperty.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanSetCatalogSessionProperty", RuntimeUnit.NANO, duration); + stats.checkCanSetCatalogSessionProperty.record(duration); + } + } + + @Override + public void checkCanGrantTablePrivilege(Identity identity, AccessControlContext context, Privilege privilege, CatalogSchemaTableName table, PrestoPrincipal grantee, boolean withGrantOption) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanGrantTablePrivilege(identity, context, privilege, table, grantee, withGrantOption); + } + catch (RuntimeException e) { + stats.checkCanGrantTablePrivilege.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanGrantTablePrivilege", RuntimeUnit.NANO, duration); + stats.checkCanGrantTablePrivilege.record(duration); + } + } + + @Override + public void checkCanRevokeTablePrivilege(Identity identity, AccessControlContext context, Privilege privilege, CatalogSchemaTableName table, PrestoPrincipal revokee, boolean grantOptionFor) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanRevokeTablePrivilege(identity, context, privilege, table, revokee, grantOptionFor); + } + catch (RuntimeException e) { + stats.checkCanRevokeTablePrivilege.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanRevokeTablePrivilege", RuntimeUnit.NANO, duration); + stats.checkCanRevokeTablePrivilege.record(duration); + } + } + + @Override + public void checkCanDropConstraint(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanDropConstraint(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanDropConstraint.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanDropConstraint", RuntimeUnit.NANO, duration); + stats.checkCanDropConstraint.record(duration); + } + } + + @Override + public void checkCanAddConstraint(Identity identity, AccessControlContext context, CatalogSchemaTableName table) + { + long start = System.nanoTime(); + try { + delegate.get().checkCanAddConstraint(identity, context, table); + } + catch (RuntimeException e) { + stats.checkCanAddConstraint.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.checkCanAddConstraint", RuntimeUnit.NANO, duration); + stats.checkCanAddConstraint.record(duration); + } + } + + @Override + public List getRowFilters(Identity identity, AccessControlContext context, CatalogSchemaTableName tableName) + { + long start = System.nanoTime(); + try { + return delegate.get().getRowFilters(identity, context, tableName); + } + catch (RuntimeException e) { + stats.getRowFilters.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.getRowFilters", RuntimeUnit.NANO, duration); + stats.getRowFilters.record(duration); + } + } + + @Override + public Map getColumnMasks(Identity identity, AccessControlContext context, CatalogSchemaTableName tableName, List columns) + { + long start = System.nanoTime(); + try { + return delegate.get().getColumnMasks(identity, context, tableName, columns); + } + catch (RuntimeException e) { + stats.getColumnMasks.recordFailure(); + throw e; + } + finally { + long duration = System.nanoTime() - start; + context.getRuntimeStats().addMetricValue("systemAccessControl.getColumnMasks", RuntimeUnit.NANO, duration); + stats.getColumnMasks.record(duration); + } + } + + public static class Stats + { + final SystemAccessControlStats checkCanSetUser = new SystemAccessControlStats(); + final SystemAccessControlStats selectAuthorizedIdentity = new SystemAccessControlStats(); + final SystemAccessControlStats checkQueryIntegrity = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanSetSystemSessionProperty = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanAccessCatalog = new SystemAccessControlStats(); + final SystemAccessControlStats filterCatalogs = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateSchema = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDropSchema = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanRenameSchema = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanShowSchemas = new SystemAccessControlStats(); + final SystemAccessControlStats filterSchemas = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanShowCreateTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanSetTableProperties = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDropTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanRenameTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanShowTablesMetadata = new SystemAccessControlStats(); + final SystemAccessControlStats filterTables = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanShowColumnsMetadata = new SystemAccessControlStats(); + final SystemAccessControlStats filterColumns = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanAddColumn = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDropColumn = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanRenameColumn = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanSelectFromColumns = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanInsertIntoTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDeleteFromTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanTruncateTable = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanUpdateTableColumns = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateView = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanRenameView = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDropView = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanCreateViewWithSelectFromColumns = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanSetCatalogSessionProperty = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanGrantTablePrivilege = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanRevokeTablePrivilege = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanDropConstraint = new SystemAccessControlStats(); + final SystemAccessControlStats checkCanAddConstraint = new SystemAccessControlStats(); + final SystemAccessControlStats getRowFilters = new SystemAccessControlStats(); + final SystemAccessControlStats getColumnMasks = new SystemAccessControlStats(); + + @Managed + @Nested + public SystemAccessControlStats getCheckCanSetUser() + { + return checkCanSetUser; + } + + @Managed + @Nested + public SystemAccessControlStats getSelectAuthorizedIdentity() + { + return selectAuthorizedIdentity; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckQueryIntegrity() + { + return checkQueryIntegrity; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanSetSystemSessionProperty() + { + return checkCanSetSystemSessionProperty; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanAccessCatalog() + { + return checkCanAccessCatalog; + } + + @Managed + @Nested + public SystemAccessControlStats getFilterCatalogs() + { + return filterCatalogs; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanCreateSchema() + { + return checkCanCreateSchema; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDropSchema() + { + return checkCanDropSchema; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanRenameSchema() + { + return checkCanRenameSchema; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanShowSchemas() + { + return checkCanShowSchemas; + } + + @Managed + @Nested + public SystemAccessControlStats getFilterSchemas() + { + return filterSchemas; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanShowCreateTable() + { + return checkCanShowCreateTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanCreateTable() + { + return checkCanCreateTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanSetTableProperties() + { + return checkCanSetTableProperties; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDropTable() + { + return checkCanDropTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanRenameTable() + { + return checkCanRenameTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanShowTablesMetadata() + { + return checkCanShowTablesMetadata; + } + + @Managed + @Nested + public SystemAccessControlStats getFilterTables() + { + return filterTables; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanShowColumnsMetadata() + { + return checkCanShowColumnsMetadata; + } + + @Managed + @Nested + public SystemAccessControlStats getFilterColumns() + { + return filterColumns; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanAddColumn() + { + return checkCanAddColumn; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDropColumn() + { + return checkCanDropColumn; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanRenameColumn() + { + return checkCanRenameColumn; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanSelectFromColumns() + { + return checkCanSelectFromColumns; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanInsertIntoTable() + { + return checkCanInsertIntoTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDeleteFromTable() + { + return checkCanDeleteFromTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanTruncateTable() + { + return checkCanTruncateTable; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanUpdateTableColumns() + { + return checkCanUpdateTableColumns; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanCreateView() + { + return checkCanCreateView; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanRenameView() + { + return checkCanRenameView; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDropView() + { + return checkCanDropView; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanCreateViewWithSelectFromColumns() + { + return checkCanCreateViewWithSelectFromColumns; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanSetCatalogSessionProperty() + { + return checkCanSetCatalogSessionProperty; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanGrantTablePrivilege() + { + return checkCanGrantTablePrivilege; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanRevokeTablePrivilege() + { + return checkCanRevokeTablePrivilege; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanDropConstraint() + { + return checkCanDropConstraint; + } + + @Managed + @Nested + public SystemAccessControlStats getCheckCanAddConstraint() + { + return checkCanAddConstraint; + } + + @Managed + @Nested + public SystemAccessControlStats getGetRowFilters() + { + return getRowFilters; + } + + @Managed + @Nested + public SystemAccessControlStats getGetColumnMasks() + { + return getColumnMasks; + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/security/SystemAccessControlStats.java b/presto-main-base/src/main/java/com/facebook/presto/security/SystemAccessControlStats.java new file mode 100644 index 0000000000000..95babfbb9e80d --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/security/SystemAccessControlStats.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.security; + +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.TimeStat; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public class SystemAccessControlStats +{ + private final CounterStat failures = new CounterStat(); + private final TimeStat time = new TimeStat(MICROSECONDS); + + public void record(long nanos) + { + time.add(nanos, NANOSECONDS); + } + + public void recordFailure() + { + failures.update(1); + } + + @Managed + @Nested + public TimeStat getTime() + { + return time; + } + + @Managed + @Nested + public CounterStat getFailures() + { + return failures; + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java b/presto-main-base/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java index d79c6ce990881..1da085a50cef2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java +++ b/presto-main-base/src/main/java/com/facebook/presto/server/QuerySessionSupplier.java @@ -143,6 +143,7 @@ private Identity authenticateIdentity(QueryId queryId, SessionContext context) context.getIdentity().getExtraCredentials(), context.getIdentity().getExtraAuthenticators(), Optional.of(identity.getUserName()), - identity.getReasonForSelect())).orElseGet(context::getIdentity); + identity.getReasonForSelect(), + context.getCertificates())).orElseGet(context::getIdentity); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 300f2485b5e97..694a496d005fc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -54,6 +54,10 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_DEBUG_MEMORY_POOL_NAME_REGEX = "native_debug_memory_pool_name_regex"; public static final String NATIVE_DEBUG_MEMORY_POOL_WARN_THRESHOLD_BYTES = "native_debug_memory_pool_warn_threshold_bytes"; public static final String NATIVE_SELECTIVE_NIMBLE_READER_ENABLED = "native_selective_nimble_reader_enabled"; + public static final String NATIVE_ROW_SIZE_TRACKING_ENABLED = "row_size_tracking_enabled"; + public static final String NATIVE_PREFERRED_OUTPUT_BATCH_BYTES = "preferred_output_batch_bytes"; + public static final String NATIVE_PREFERRED_OUTPUT_BATCH_ROWS = "preferred_output_batch_rows"; + public static final String NATIVE_MAX_OUTPUT_BATCH_ROWS = "max_output_batch_rows"; public static final String NATIVE_MAX_PARTIAL_AGGREGATION_MEMORY = "native_max_partial_aggregation_memory"; public static final String NATIVE_MAX_EXTENDED_PARTIAL_AGGREGATION_MEMORY = "native_max_extended_partial_aggregation_memory"; public static final String NATIVE_MAX_SPILL_BYTES = "native_max_spill_bytes"; @@ -233,10 +237,28 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "reader is fully rolled out.", false, !nativeExecution), + booleanProperty( + NATIVE_ROW_SIZE_TRACKING_ENABLED, + "Flag to control whether row size tracking should be enabled as a fallback " + + "for reader row size estimates.", + true, + !nativeExecution), longProperty( - NATIVE_MAX_PARTIAL_AGGREGATION_MEMORY, - "The max partial aggregation memory when data reduction is not optimal.", - 1L << 24, + NATIVE_PREFERRED_OUTPUT_BATCH_BYTES, + "Prefered memory budget for operator output batches. " + + "Used in tandem with average row size estimates when available.", + 10L << 20, + !nativeExecution), + integerProperty( + NATIVE_PREFERRED_OUTPUT_BATCH_ROWS, + "Preferred row count per operator output batch. Used when average row size estimates are unknown.", + 1024, + !nativeExecution), + integerProperty( + NATIVE_MAX_OUTPUT_BATCH_ROWS, + "Upperbound for row count per output batch, used together with " + + "preferred_output_batch_bytes and average row size estimates.", + 10000, !nativeExecution), longProperty( NATIVE_MAX_EXTENDED_PARTIAL_AGGREGATION_MEMORY, diff --git a/presto-main-base/src/main/java/com/facebook/presto/spiller/AesSpillCipher.java b/presto-main-base/src/main/java/com/facebook/presto/spiller/AesSpillCipher.java index 7386bf8a68ede..253c59d7d281f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/spiller/AesSpillCipher.java +++ b/presto-main-base/src/main/java/com/facebook/presto/spiller/AesSpillCipher.java @@ -33,8 +33,8 @@ final class AesSpillCipher implements SpillCipher { - // 256-bit AES CBC mode - private static final String CIPHER_NAME = "AES/CBC/PKCS5Padding"; + // 256-bit AES CTR mode + private static final String CIPHER_NAME = "AES/CTR/NoPadding"; private static final int KEY_BITS = 256; private SecretKey key; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java index d4cb14ac2ac68..3ecc50bfa7fb1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.analyzer.AccessControlReferences; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.parser.SqlParser; @@ -43,7 +44,8 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.CANNOT_HAVE_AGGREGATIONS_WINDOWS_OR_GROUPING; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.UtilizedColumnsAnalyzer.analyzeForUtilizedColumns; -import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissions; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForColumns; +import static com.facebook.presto.util.AnalyzerUtil.checkAccessPermissionsForTable; import static java.util.Objects.requireNonNull; public class Analyzer @@ -107,7 +109,9 @@ public Analysis analyze(Statement statement) public Analysis analyze(Statement statement, boolean isDescribe) { Analysis analysis = analyzeSemantic(statement, isDescribe); - checkAccessPermissions(analysis.getAccessControlReferences(), query); + AccessControlReferences accessControlReferences = analysis.getAccessControlReferences(); + checkAccessPermissionsForTable(accessControlReferences); + checkAccessPermissionsForColumns(accessControlReferences); return analysis; } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index 086a929fdb4b3..2e7c907f34118 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -62,10 +62,12 @@ import com.facebook.presto.spi.function.table.ArgumentSpecification; import com.facebook.presto.spi.function.table.ConnectorTableFunction; import com.facebook.presto.spi.function.table.Descriptor; +import com.facebook.presto.spi.function.table.DescriptorArgument; import com.facebook.presto.spi.function.table.DescriptorArgumentSpecification; import com.facebook.presto.spi.function.table.ReturnTypeSpecification; import com.facebook.presto.spi.function.table.ScalarArgument; import com.facebook.presto.spi.function.table.ScalarArgumentSpecification; +import com.facebook.presto.spi.function.table.TableArgument; import com.facebook.presto.spi.function.table.TableArgumentSpecification; import com.facebook.presto.spi.function.table.TableFunctionAnalysis; import com.facebook.presto.spi.relation.DomainTranslator; @@ -75,9 +77,12 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.security.ViewAccessControl; import com.facebook.presto.spi.security.ViewExpression; +import com.facebook.presto.spi.type.UnknownTypeException; import com.facebook.presto.sql.ExpressionUtils; import com.facebook.presto.sql.MaterializedViewUtils; import com.facebook.presto.sql.SqlFormatterUtil; +import com.facebook.presto.sql.analyzer.Analysis.TableArgumentAnalysis; +import com.facebook.presto.sql.analyzer.Analysis.TableFunctionInvocationAnalysis; import com.facebook.presto.sql.parser.ParsingException; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.ExpressionInterpreter; @@ -112,6 +117,7 @@ import com.facebook.presto.sql.tree.DropSchema; import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; +import com.facebook.presto.sql.tree.EmptyTableTreatment; import com.facebook.presto.sql.tree.Except; import com.facebook.presto.sql.tree.Execute; import com.facebook.presto.sql.tree.Explain; @@ -139,6 +145,7 @@ import com.facebook.presto.sql.tree.Literal; import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.Merge; import com.facebook.presto.sql.tree.NaturalJoin; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NodeLocation; @@ -246,6 +253,8 @@ import static com.facebook.presto.spi.connector.ConnectorTableVersion.VersionType; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; +import static com.facebook.presto.spi.function.table.DescriptorArgument.NULL_DESCRIPTOR; +import static com.facebook.presto.spi.function.table.ReturnTypeSpecification.GenericTable.GENERIC_TABLE; import static com.facebook.presto.sql.MaterializedViewUtils.buildOwnerSession; import static com.facebook.presto.sql.MaterializedViewUtils.generateBaseTablePredicates; import static com.facebook.presto.sql.MaterializedViewUtils.generateFalsePredicates; @@ -268,7 +277,6 @@ import static com.facebook.presto.sql.analyzer.RefreshMaterializedViewPredicateAnalyzer.extractTablePredicates; import static com.facebook.presto.sql.analyzer.ScopeReferenceExtractor.hasReferencesToScope; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.AMBIGUOUS_ATTRIBUTE; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.AMBIGUOUS_RETURN_TYPE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.COLUMN_NAME_NOT_SPECIFIED; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.COLUMN_TYPE_UNKNOWN; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_COLUMN_NAME; @@ -276,8 +284,6 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_PROPERTY; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.DUPLICATE_RELATION; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.FUNCTION_NOT_FOUND; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_ARGUMENTS; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_FUNCTION_NAME; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_OFFSET_ROW_COUNT; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_ORDINAL; @@ -287,21 +293,29 @@ import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MATERIALIZED_VIEW_IS_RECURSIVE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISMATCHED_COLUMN_ALIASES; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISMATCHED_SET_COLUMN_TYPES; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_ARGUMENT; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_COLUMN; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_MATERIALIZED_VIEW; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_RETURN_TYPE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_SCHEMA; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MISSING_TABLE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.MUST_BE_WINDOW_FUNCTION; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NESTED_WINDOW; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NONDETERMINISTIC_ORDER_BY_EXPRESSION_WITH_SELECT_DISTINCT; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NON_NUMERIC_SAMPLE_PERCENTAGE; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_IMPLEMENTED; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.ORDER_BY_MUST_BE_IN_SELECT; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_ALREADY_EXISTS; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_AMBIGUOUS_RETURN_TYPE; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_COLUMN_NOT_FOUND; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_DUPLICATE_RANGE_VARIABLE; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_IMPLEMENTATION_ERROR; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_INVALID_ARGUMENTS; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_INVALID_COLUMN_REFERENCE; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_INVALID_COPARTITIONING; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_MISSING_ARGUMENT; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TABLE_FUNCTION_MISSING_RETURN_TYPE; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TOO_MANY_GROUPING_SETS; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH; import static com.facebook.presto.sql.analyzer.SemanticErrorCode.VIEW_ANALYSIS_ERROR; @@ -314,6 +328,7 @@ import static com.facebook.presto.sql.planner.ExpressionDeterminismEvaluator.isDeterministic; import static com.facebook.presto.sql.planner.ExpressionInterpreter.evaluateConstantExpression; import static com.facebook.presto.sql.planner.ExpressionInterpreter.expressionOptimizer; +import static com.facebook.presto.sql.tree.DereferenceExpression.getQualifiedName; import static com.facebook.presto.sql.tree.ExplainFormat.Type.JSON; import static com.facebook.presto.sql.tree.ExplainFormat.Type.TEXT; import static com.facebook.presto.sql.tree.ExplainType.Type.DISTRIBUTED; @@ -336,6 +351,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.Math.toIntExact; import static java.lang.String.format; import static java.util.Collections.emptyList; @@ -1346,9 +1362,7 @@ protected Scope visitTableFunctionInvocation(TableFunctionInvocation node, Optio ConnectorTableFunction function = tableFunctionMetadata.getFunction(); ConnectorId connectorId = tableFunctionMetadata.getConnectorId(); - QualifiedObjectName functionName = new QualifiedObjectName(connectorId.getCatalogName(), function.getSchema(), function.getName()); - - Map passedArguments = analyzeArguments(node, function.getArguments(), node.getArguments()); + ArgumentsAnalysis argumentsAnalysis = analyzeArguments(node, function.getArguments(), scope); TransactionManager transactionManager = metadata.getFunctionAndTypeManager().getTransactionManager(); CatalogMetadata registrationCatalogMetadata = transactionManager.getOptionalCatalogMetadata(session.getRequiredTransactionId(), connectorId.getCatalogName()).orElseThrow(() -> new IllegalStateException("Missing catalog metadata")); @@ -1356,86 +1370,180 @@ protected Scope visitTableFunctionInvocation(TableFunctionInvocation node, Optio ConnectorTransactionHandle transactionHandle = transactionManager.getConnectorTransaction( session.getRequiredTransactionId(), registrationCatalogMetadata.getConnectorId()); - TableFunctionAnalysis functionAnalysis = function.analyze(session.toConnectorSession(connectorId), transactionHandle, passedArguments); - analysis.setTableFunctionAnalysis(node, new Analysis.TableFunctionInvocationAnalysis(connectorId, functionName.toString(), passedArguments, functionAnalysis.getHandle(), transactionHandle)); + TableFunctionAnalysis functionAnalysis = function.analyze(session.toConnectorSession(connectorId), transactionHandle, argumentsAnalysis.getPassedArguments()); + List> copartitioningLists = analyzeCopartitioning(node.getCopartitioning(), argumentsAnalysis.getTableArgumentAnalyses()); - // TODO process the copartitioning: - // 1. validate input table references - // 2. the copartitioned tables in each set must be partitioned, and have the same number of partitioning columns - // 3. the corresponding columns must be comparable - // 4. within a set, determine and record coercions of the corresponding columns to a common supertype - // Note that if a table is part of multiple copartitioning sets, it might require a different coercion for a column - // per each set. Additionally, there might be another coercion required by the Table Function logic. Also, since - // all partitioning columns are passed-through, we also need an un-coerced copy. - // See ExpressionAnalyzer.sortKeyCoercionsForFrameBoundCalculation for multiple coercions on a column. - if (!node.getCopartitioning().isEmpty()) { - throw new SemanticException(NOT_IMPLEMENTED, node, "COPARTITION clause is not yet supported for table functions"); + // determine the result relation type per SQL standard ISO/IEC 9075-2, 4.33 SQL-invoked routines, p. 123, 413, 414 + ReturnTypeSpecification returnTypeSpecification = function.getReturnTypeSpecification(); + if (returnTypeSpecification == GENERIC_TABLE || !argumentsAnalysis.getTableArgumentAnalyses().isEmpty()) { + analysis.addPolymorphicTableFunction(node); } + Optional analyzedProperColumnsDescriptor = functionAnalysis.getReturnedType(); + Descriptor properColumnsDescriptor = verifyProperColumnsDescriptor(node, function, returnTypeSpecification, analyzedProperColumnsDescriptor); + + Map tableArgumentsByName = argumentsAnalysis.getTableArgumentAnalyses().stream() + .collect(toImmutableMap(TableArgumentAnalysis::getArgumentName, Function.identity())); + verifyRequiredColumns(node, functionAnalysis.getRequiredColumns(), tableArgumentsByName); - // determine the result relation type. // The result relation type of a table function consists of: - // 1. passed columns from input tables: + // 1. columns created by the table function, called the proper columns. + // 2. passed columns from input tables: // - for tables with the "pass through columns" option, these are all columns of the table, // - for tables without the "pass through columns" option, these are the partitioning columns of the table, if any. - // 2. columns created by the table function, called the proper columns. - ReturnTypeSpecification returnTypeSpecification = function.getReturnTypeSpecification(); - Optional analyzedProperColumnsDescriptor = functionAnalysis.getReturnedType(); - Descriptor properColumnsDescriptor; + ImmutableList.Builder fields = ImmutableList.builder(); + + // proper columns first + if (properColumnsDescriptor != null) { + properColumnsDescriptor.getFields().stream() + // per spec, field names are mandatory. We support anonymous fields. + .map(field -> Field.newUnqualified(Optional.empty(), field.getName(), field.getType().orElseThrow(() -> new IllegalStateException("missing returned type for proper field")))) + .forEach(fields::add); + } + + // next, columns derived from table arguments, in order of argument declarations + List tableArgumentNames = function.getArguments().stream() + .filter(argumentSpecification -> argumentSpecification instanceof TableArgumentSpecification) + .map(ArgumentSpecification::getName) + .collect(toImmutableList()); + + // table arguments in order of argument declarations + ImmutableList.Builder orderedTableArguments = ImmutableList.builder(); + + for (String name : tableArgumentNames) { + TableArgumentAnalysis argument = tableArgumentsByName.get(name); + orderedTableArguments.add(argument); + Scope argumentScope = analysis.getScope(argument.getRelation()); + if (argument.isPassThroughColumns()) { + argumentScope.getRelationType().getAllFields().stream() + .forEach(fields::add); + } + else if (argument.getPartitionBy().isPresent()) { + argument.getPartitionBy().get().stream() + .map(expression -> validateAndGetInputField(expression, argumentScope)) + .forEach(fields::add); + } + } + + analysis.setTableFunctionAnalysis(node, new TableFunctionInvocationAnalysis( + connectorId, + function.getName(), + argumentsAnalysis.getPassedArguments(), + orderedTableArguments.build(), + functionAnalysis.getRequiredColumns(), + copartitioningLists, + properColumnsDescriptor == null ? 0 : properColumnsDescriptor.getFields().size(), + functionAnalysis.getHandle(), + transactionHandle)); + + return createAndAssignScope(node, scope, fields.build()); + } + + private void verifyRequiredColumns(TableFunctionInvocation node, Map> requiredColumns, Map tableArgumentsByName) + { + Set allInputs = ImmutableSet.copyOf(tableArgumentsByName.keySet()); + requiredColumns.forEach((name, columns) -> { + if (!allInputs.contains(name)) { + throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "Table function %s specifies required columns from table argument %s which cannot be found", node.getName(), name); + } + if (columns.isEmpty()) { + throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "Table function %s specifies empty list of required columns from table argument %s", node.getName(), name); + } + // the scope is recorded, because table arguments are already analyzed + Scope inputScope = analysis.getScope(tableArgumentsByName.get(name).getRelation()); + columns.stream() + .filter(column -> column < 0 || column >= inputScope.getRelationType().getAllFieldCount()) // hidden columns can be required as well as visible columns + .findFirst() + .ifPresent(column -> { + throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "Invalid index: %s of required column from table argument %s", column, name); + }); + }); + Set requiredInputs = ImmutableSet.copyOf(requiredColumns.keySet()); + allInputs.stream() + .filter(input -> !requiredInputs.contains(input)) + .findFirst() + .ifPresent(input -> { + throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "Table function %s does not specify required input columns from table argument %s", node.getName(), input); + }); + } + + private Descriptor verifyProperColumnsDescriptor(TableFunctionInvocation node, ConnectorTableFunction function, ReturnTypeSpecification returnTypeSpecification, Optional analyzedProperColumnsDescriptor) + { switch (returnTypeSpecification.getReturnType()) { case ReturnTypeSpecification.OnlyPassThrough.returnType: - throw new SemanticException(NOT_IMPLEMENTED, node, "Returning only pass through columns is not yet supported for table functions"); + if (analysis.isAliased(node)) { + // According to SQL standard ISO/IEC 9075-2, 7.6 , p. 409, + // table alias is prohibited for a table function with ONLY PASS THROUGH returned type. + throw new SemanticException(TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, node, "Alias specified for table function with ONLY PASS THROUGH return type"); + } + if (analyzedProperColumnsDescriptor.isPresent()) { + // If a table function has ONLY PASS THROUGH returned type, it does not produce any proper columns, + // so the function's analyze() method should not return the proper columns descriptor. + throw new SemanticException(TABLE_FUNCTION_AMBIGUOUS_RETURN_TYPE, node, "Returned relation type for table function %s is ambiguous", node.getName()); + } + if (function.getArguments().stream() + .filter(TableArgumentSpecification.class::isInstance) + .map(TableArgumentSpecification.class::cast) + .noneMatch(TableArgumentSpecification::isPassThroughColumns)) { + // According to SQL standard ISO/IEC 9075-2, 10.4 , p. 764, + // if there is no generic table parameter that specifies PASS THROUGH, then number of proper columns shall be positive. + // For GENERIC_TABLE and DescribedTable returned types, this is enforced by the Descriptor constructor, which requires positive number of fields. + // Here we enforce it for the remaining returned type specification: ONLY_PASS_THROUGH. + throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "A table function with ONLY_PASS_THROUGH return type must have a table argument with pass-through columns."); + } + return null; case ReturnTypeSpecification.GenericTable.returnType: - properColumnsDescriptor = analyzedProperColumnsDescriptor - .orElseThrow(() -> new SemanticException(MISSING_RETURN_TYPE, node, "Cannot determine returned relation type for table function " + node.getName())); - break; + // According to SQL standard ISO/IEC 9075-2, 7.6
, p. 409, + // table alias is mandatory for a polymorphic table function invocation which produces proper columns. + // We don't enforce this requirement. + return analyzedProperColumnsDescriptor + .orElseThrow(() -> new SemanticException(TABLE_FUNCTION_MISSING_RETURN_TYPE, node, "Cannot determine returned relation type for table function " + node.getName())); default: // returned type is statically declared at function declaration and cannot be overridden + // According to SQL standard ISO/IEC 9075-2, 7.6
, p. 409, + // table alias is mandatory for a polymorphic table function invocation which produces proper columns. + // We don't enforce this requirement. if (analyzedProperColumnsDescriptor.isPresent()) { - throw new SemanticException(AMBIGUOUS_RETURN_TYPE, node, "Returned relation type for table function %s is ambiguous", node.getName()); + // If a table function has statically declared returned type, it is returned in TableFunctionMetadata + // so the function's analyze() method should not return the proper columns descriptor. + throw new SemanticException(TABLE_FUNCTION_AMBIGUOUS_RETURN_TYPE, node, "Returned relation type for table function %s is ambiguous", node.getName()); } - properColumnsDescriptor = ((ReturnTypeSpecification.DescribedTable) returnTypeSpecification).getDescriptor(); + return ((ReturnTypeSpecification.DescribedTable) returnTypeSpecification).getDescriptor(); } - // currently we don't support input tables, so the output consists of proper columns only - List fields = properColumnsDescriptor.getFields().stream() - // per spec, field names are mandatory - .map(field -> Field.newUnqualified((field.getName()), field.getType().orElseThrow(() -> new IllegalStateException("missing returned type for proper field")))) - .collect(toImmutableList()); - - return createAndAssignScope(node, scope, fields); } - private Map analyzeArguments(Node node, List argumentSpecifications, List arguments) + private ArgumentsAnalysis analyzeArguments(TableFunctionInvocation node, List argumentSpecifications, Optional scope) { + List arguments = node.getArguments(); Node errorLocation = node; if (!arguments.isEmpty()) { errorLocation = arguments.get(0); } - if (argumentSpecifications.size() < arguments.size()) { - throw new SemanticException(INVALID_ARGUMENTS, errorLocation, "Too many arguments. Expected at most %s arguments, got %s arguments", argumentSpecifications.size(), arguments.size()); + throw new SemanticException(TABLE_FUNCTION_INVALID_ARGUMENTS, errorLocation, "Too many arguments. Expected at most %s arguments, got %s arguments", argumentSpecifications.size(), arguments.size()); } if (argumentSpecifications.isEmpty()) { - return ImmutableMap.of(); + return new ArgumentsAnalysis(ImmutableMap.of(), ImmutableList.of()); } boolean argumentsPassedByName = !arguments.isEmpty() && arguments.stream().allMatch(argument -> argument.getName().isPresent()); boolean argumentsPassedByPosition = arguments.stream().allMatch(argument -> !argument.getName().isPresent()); if (!argumentsPassedByName && !argumentsPassedByPosition) { - throw new SemanticException(INVALID_ARGUMENTS, errorLocation, "All arguments must be passed by name or all must be passed positionally"); + throw new SemanticException(TABLE_FUNCTION_INVALID_ARGUMENTS, errorLocation, "All arguments must be passed by name or all must be passed positionally"); } if (argumentsPassedByName) { - return mapTableFunctionsArgsByName(argumentSpecifications, arguments, errorLocation); + return mapTableFunctionsArgsByName(argumentSpecifications, arguments, errorLocation, scope); } else { - return mapTableFunctionArgsByPosition(argumentSpecifications, arguments, errorLocation); + return mapTableFunctionArgsByPosition(argumentSpecifications, arguments, errorLocation, scope); } } - private Map mapTableFunctionsArgsByName(List argumentSpecifications, List arguments, Node errorLocation) + private ArgumentsAnalysis mapTableFunctionsArgsByName(List argumentSpecifications, List arguments, Node errorLocation, Optional scope) { ImmutableMap.Builder passedArguments = ImmutableMap.builder(); + ImmutableList.Builder tableArgumentAnalyses = ImmutableList.builder(); Map argumentSpecificationsByName = new HashMap<>(); for (ArgumentSpecification argumentSpecification : argumentSpecifications) { if (argumentSpecificationsByName.put(argumentSpecification.getName(), argumentSpecification) != null) { @@ -1447,46 +1555,51 @@ private Map mapTableFunctionsArgsByName(List new IllegalStateException("Missing table function argument name")).getCanonicalValue(); if (!uniqueArgumentNames.add(argumentName)) { - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Duplicate argument name: %s", argumentName); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Duplicate argument name: %s", argumentName); } ArgumentSpecification argumentSpecification = argumentSpecificationsByName.remove(argumentName); if (argumentSpecification == null) { - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Unexpected argument name: %s", argumentName); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Unexpected argument name: %s", argumentName); } - passedArguments.put(argumentSpecification.getName(), analyzeArgument(argumentSpecification, argument)); + ArgumentAnalysis argumentAnalysis = analyzeArgument(argumentSpecification, argument, scope); + passedArguments.put(argumentSpecification.getName(), argumentAnalysis.getArgument()); + argumentAnalysis.getTableArgumentAnalysis().ifPresent(tableArgumentAnalyses::add); } // apply defaults for not specified arguments for (Map.Entry entry : argumentSpecificationsByName.entrySet()) { ArgumentSpecification argumentSpecification = entry.getValue(); passedArguments.put(argumentSpecification.getName(), analyzeDefault(argumentSpecification, errorLocation)); } - return passedArguments.build(); + return new ArgumentsAnalysis(passedArguments.buildOrThrow(), tableArgumentAnalyses.build()); } - private Map mapTableFunctionArgsByPosition(List argumentSpecifications, List arguments, Node errorLocation) + private ArgumentsAnalysis mapTableFunctionArgsByPosition(List argumentSpecifications, List arguments, Node errorLocation, Optional scope) { ImmutableMap.Builder passedArguments = ImmutableMap.builder(); + ImmutableList.Builder tableArgumentAnalyses = ImmutableList.builder(); for (int i = 0; i < arguments.size(); i++) { TableFunctionArgument argument = arguments.get(i); ArgumentSpecification argumentSpecification = argumentSpecifications.get(i); // TODO args passed positionally - can one only pass some prefix of args? - passedArguments.put(argumentSpecification.getName(), analyzeArgument(argumentSpecification, argument)); + ArgumentAnalysis argumentAnalysis = analyzeArgument(argumentSpecification, argument, scope); + passedArguments.put(argumentSpecification.getName(), argumentAnalysis.getArgument()); + argumentAnalysis.getTableArgumentAnalysis().ifPresent(tableArgumentAnalyses::add); } // apply defaults for not specified arguments for (int i = arguments.size(); i < argumentSpecifications.size(); i++) { ArgumentSpecification argumentSpecification = argumentSpecifications.get(i); passedArguments.put(argumentSpecification.getName(), analyzeDefault(argumentSpecification, errorLocation)); } - return passedArguments.build(); + return new ArgumentsAnalysis(passedArguments.buildOrThrow(), tableArgumentAnalyses.build()); } - private Argument analyzeArgument(ArgumentSpecification argumentSpecification, TableFunctionArgument argument) + private ArgumentAnalysis analyzeArgument(ArgumentSpecification argumentSpecification, TableFunctionArgument argument, Optional scope) { String actualType = getArgumentTypeString(argument); switch (argumentSpecification.getArgumentType()){ case TableArgumentSpecification.argumentType: - return analyzeTableArgument(argument, argumentSpecification, actualType); + return analyzeTableArgument(argument, (TableArgumentSpecification) argumentSpecification, scope, actualType); case DescriptorArgumentSpecification.argumentType: - return analyzeDescriptorArgument(argument, argumentSpecification, actualType); + return analyzeDescriptorArgument(argument, (DescriptorArgumentSpecification) argumentSpecification, actualType); case ScalarArgumentSpecification.argumentType: return analyzeScalarArgument(argument, argumentSpecification, actualType); default: @@ -1497,13 +1610,15 @@ private Argument analyzeArgument(ArgumentSpecification argumentSpecification, Ta private Argument analyzeDefault(ArgumentSpecification argumentSpecification, Node errorLocation) { if (argumentSpecification.isRequired()) { - throw new SemanticException(MISSING_ARGUMENT, errorLocation, "Missing argument: " + argumentSpecification.getName()); + throw new SemanticException(TABLE_FUNCTION_MISSING_ARGUMENT, errorLocation, "Missing argument: " + argumentSpecification.getName()); } checkArgument(!(argumentSpecification instanceof TableArgumentSpecification), "invalid table argument specification: default set"); if (argumentSpecification instanceof DescriptorArgumentSpecification) { - throw new SemanticException(NOT_IMPLEMENTED, errorLocation, "Descriptor arguments are not yet supported for table functions"); + return DescriptorArgument.builder() + .descriptor((Descriptor) argumentSpecification.getDefaultValue()) + .build(); } if (argumentSpecification instanceof ScalarArgumentSpecification) { return ScalarArgument.builder() @@ -1521,20 +1636,20 @@ private String getArgumentTypeString(TableFunctionArgument argument) return argument.getValue().getArgumentTypeString(); } catch (IllegalArgumentException e) { - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Unexpected table function argument type: ", argument.getClass().getSimpleName()); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Unexpected table function argument type: ", argument.getClass().getSimpleName()); } } - private Argument analyzeScalarArgument(TableFunctionArgument argument, ArgumentSpecification argumentSpecification, String actualType) + private ArgumentAnalysis analyzeScalarArgument(TableFunctionArgument argument, ArgumentSpecification argumentSpecification, String actualType) { Type type = ((ScalarArgumentSpecification) argumentSpecification).getType(); if (!(argument.getValue() instanceof Expression)) { - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected expression, got %s", argumentSpecification.getName(), actualType); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected expression, got %s", argumentSpecification.getName(), actualType); } Expression expression = (Expression) argument.getValue(); // 'descriptor' as a function name is not allowed in this context if (expression instanceof FunctionCall && ((FunctionCall) expression).getName().hasSuffix(QualifiedName.of("descriptor"))) { // function name is always compared case-insensitive - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "'descriptor' function is not allowed as a table function argument"); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "'descriptor' function is not allowed as a table function argument"); } // inline parameters Expression inlined = ExpressionTreeRewriter.rewriteWith(new ExpressionRewriter() @@ -1554,40 +1669,263 @@ public Expression rewriteParameter(Parameter node, Void context, ExpressionTreeR }, expression); // currently, only constant arguments are supported Object constantValue = ExpressionInterpreter.evaluateConstantExpression(inlined, type, metadata, session, analysis.getParameters()); - return ScalarArgument.builder() - .type(type) - .value(constantValue) - .build(); + return new ArgumentAnalysis( + ScalarArgument.builder() + .type(type) + .value(constantValue) + .build(), + Optional.empty()); } - private Argument analyzeTableArgument(TableFunctionArgument argument, ArgumentSpecification argumentSpecification, String actualType) + private ArgumentAnalysis analyzeTableArgument(TableFunctionArgument argument, TableArgumentSpecification argumentSpecification, Optional scope, String actualType) { if (!(argument.getValue() instanceof TableFunctionTableArgument)) { if (argument.getValue() instanceof FunctionCall) { // probably an attempt to pass a table function call, which is not supported, and was parsed as a function call - throw new SemanticException(NOT_IMPLEMENTED, argument, "Invalid table argument %s. Table functions are not allowed as table function arguments", argumentSpecification.getName()); + throw new SemanticException(NOT_SUPPORTED, argument, "Invalid table argument %s. Table functions are not allowed as table function arguments", argumentSpecification.getName()); + } + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected table, got %s", argumentSpecification.getName(), actualType); + } + TableFunctionTableArgument tableArgument = (TableFunctionTableArgument) argument.getValue(); + + TableArgument.Builder argumentBuilder = TableArgument.builder(); + TableArgumentAnalysis.Builder analysisBuilder = TableArgumentAnalysis.builder(); + analysisBuilder.withArgumentName(argumentSpecification.getName()); + + // process the relation + Relation relation = tableArgument.getTable(); + analysisBuilder.withRelation(relation); + Scope argumentScope = process(relation, scope); + QualifiedName relationName = analysis.getRelationName(relation); + if (relationName != null) { + analysisBuilder.withName(relationName); + } + + argumentBuilder.rowType(RowType.from(argumentScope.getRelationType().getVisibleFields().stream() + .map(field -> new RowType.Field(field.getName(), field.getType())) + .collect(toImmutableList()))); + + // analyze PARTITION BY + if (tableArgument.getPartitionBy().isPresent()) { + if (argumentSpecification.isRowSemantics()) { + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Partitioning specified for table argument with row semantics", argumentSpecification.getName()); + } + List partitionBy = tableArgument.getPartitionBy().get(); + analysisBuilder.withPartitionBy(partitionBy); + partitionBy.stream() + .forEach(partitioningColumn -> { + validateAndGetInputField(partitioningColumn, argumentScope); + Type type = analyzeExpression(partitioningColumn, argumentScope).getType(partitioningColumn); + if (!type.isComparable()) { + throw new SemanticException(TYPE_MISMATCH, partitioningColumn, "%s is not comparable, and therefore cannot be used in PARTITION BY", type); + } + }); + argumentBuilder.partitionBy(partitionBy.stream() + // each expression is either an Identifier or a DereferenceExpression + .map(Expression::toString) + .collect(toImmutableList())); + } + + // analyze ORDER BY + if (tableArgument.getOrderBy().isPresent()) { + if (argumentSpecification.isRowSemantics()) { + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Ordering specified for table argument with row semantics", argumentSpecification.getName()); + } + OrderBy orderBy = tableArgument.getOrderBy().get(); + analysisBuilder.withOrderBy(orderBy); + orderBy.getSortItems().stream() + .map(SortItem::getSortKey) + .forEach(orderingColumn -> { + validateAndGetInputField(orderingColumn, argumentScope); + Type type = analyzeExpression(orderingColumn, argumentScope).getType(orderingColumn); + if (!type.isOrderable()) { + throw new SemanticException(TYPE_MISMATCH, orderingColumn, "%s is not orderable, and therefore cannot be used in ORDER BY", type); + } + }); + argumentBuilder.orderBy(orderBy.getSortItems().stream() + // each sort key is either an Identifier or a DereferenceExpression + .map(sortItem -> sortItem.getSortKey().toString()) + .collect(toImmutableList())); + } + + // analyze the PRUNE/KEEP WHEN EMPTY property + boolean pruneWhenEmpty = argumentSpecification.isPruneWhenEmpty(); + if (tableArgument.getEmptyTableTreatment().isPresent()) { + if (argumentSpecification.isRowSemantics()) { + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, tableArgument.getEmptyTableTreatment().get(), "Invalid argument %s. Empty behavior specified for table argument with row semantics", argumentSpecification.getName()); } - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected table, got %s", argumentSpecification.getName(), actualType); + pruneWhenEmpty = tableArgument.getEmptyTableTreatment().get().getTreatment() == EmptyTableTreatment.Treatment.PRUNE; } - // TODO analyze the argument - // 1. process the Relation - // 2. partitioning and ordering must only apply to tables with set semantics - // 3. validate partitioning and ordering using `validateAndGetInputField()` - // 4. validate the prune when empty property vs argument specification (forbidden for row semantics; override? -> check spec) - // 5. return Argument - throw new SemanticException(NOT_IMPLEMENTED, argument, "Table arguments are not yet supported for table functions"); + analysisBuilder.withPruneWhenEmpty(pruneWhenEmpty); + + // record remaining properties + analysisBuilder.withRowSemantics(argumentSpecification.isRowSemantics()); + analysisBuilder.withPassThroughColumns(argumentSpecification.isPassThroughColumns()); + + return new ArgumentAnalysis(argumentBuilder.build(), Optional.of(analysisBuilder.build())); } - private Argument analyzeDescriptorArgument(TableFunctionArgument argument, ArgumentSpecification argumentSpecification, String actualType) + private ArgumentAnalysis analyzeDescriptorArgument(TableFunctionArgument argument, DescriptorArgumentSpecification argumentSpecification, String actualType) { if (!(argument.getValue() instanceof TableFunctionDescriptorArgument)) { if (argument.getValue() instanceof FunctionCall && ((FunctionCall) argument.getValue()).getName().hasSuffix(QualifiedName.of("descriptor"))) { // function name is always compared case-insensitive // malformed descriptor which parsed as a function call - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Invalid descriptor argument %s. Descriptors should be formatted as 'DESCRIPTOR(name [type], ...)'", (Object) argumentSpecification.getName()); + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid descriptor argument %s. Descriptors should be formatted as 'DESCRIPTOR(name [type], ...)'", (Object) argumentSpecification.getName()); + } + throw new SemanticException(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected descriptor, got %s", argumentSpecification.getName(), actualType); + } + return new ArgumentAnalysis( + ((TableFunctionDescriptorArgument) argument.getValue()).getDescriptor() + .map(descriptor -> DescriptorArgument.builder() + .descriptor(new Descriptor(descriptor.getFields().stream() + .map(field -> new Descriptor.Field( + field.getName().getCanonicalValue(), + field.getType().map(type -> { + try { + return functionAndTypeResolver.getType(parseTypeSignature(type)); + } + catch (IllegalArgumentException | UnknownTypeException e) { + throw new SemanticException(TYPE_MISMATCH, field, "Unknown type: %s", type); + } + }))) + .collect(toImmutableList()))) + .build()) + .orElse(NULL_DESCRIPTOR), + Optional.empty()); + } + + private Field validateAndGetInputField(Expression expression, Scope inputScope) + { + QualifiedName qualifiedName; + if (expression instanceof Identifier) { + qualifiedName = QualifiedName.of(ImmutableList.of(((Identifier) expression))); + } + else if (expression instanceof DereferenceExpression) { + qualifiedName = getQualifiedName((DereferenceExpression) expression); + } + else { + throw new SemanticException(TABLE_FUNCTION_INVALID_COLUMN_REFERENCE, expression, "Expected column reference. Actual: %s", expression); + } + Optional field = inputScope.tryResolveField(expression, qualifiedName); + if (!field.isPresent() || !field.get().isLocal()) { + throw new SemanticException(TABLE_FUNCTION_COLUMN_NOT_FOUND, expression, "Column %s is not present in the input relation", expression); + } + + return field.get().getField(); + } + + private List> analyzeCopartitioning(List> copartitioning, List tableArgumentAnalyses) + { + // map table arguments by relation names. usa a multimap, because multiple arguments can have the same value, e.g. input_1 => tpch.tiny.orders, input_2 => tpch.tiny.orders + ImmutableMultimap.Builder unqualifiedInputsBuilder = ImmutableMultimap.builder(); + ImmutableMultimap.Builder qualifiedInputsBuilder = ImmutableMultimap.builder(); + tableArgumentAnalyses.stream() + .filter(argument -> argument.getName().isPresent()) + .forEach(argument -> { + QualifiedName name = argument.getName().get(); + if (name.getParts().size() == 1) { + unqualifiedInputsBuilder.put(name, argument); + } + else if (name.getParts().size() == 3) { + qualifiedInputsBuilder.put(name, argument); + } + else { + throw new IllegalStateException("relation name should be unqualified or fully qualified"); + } + }); + Multimap unqualifiedInputs = unqualifiedInputsBuilder.build(); + Multimap qualifiedInputs = qualifiedInputsBuilder.build(); + + ImmutableList.Builder> copartitionBuilder = ImmutableList.builder(); + Set referencedArguments = new HashSet<>(); + for (List nameList : copartitioning) { + ImmutableList.Builder copartitionListBuilder = ImmutableList.builder(); + + // resolve copartition tables as references to table arguments + for (QualifiedName name : nameList) { + Collection candidates = emptyList(); + if (name.getParts().size() == 1) { + // try to match unqualified name. it might be a reference to a CTE or an aliased relation + candidates = unqualifiedInputs.get(name); + } + if (candidates.isEmpty()) { + // qualify the name using current schema and catalog + // Since we lost the Identifier context, create a new one here + QualifiedObjectName fullyQualifiedName = createQualifiedObjectName(session, new Identifier(name.getOriginalParts().get(0).getValue()), name, metadata); + candidates = qualifiedInputs.get(QualifiedName.of(fullyQualifiedName.getCatalogName(), fullyQualifiedName.getSchemaName(), fullyQualifiedName.getObjectName())); + } + if (candidates.isEmpty()) { + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, name.getOriginalParts().get(0), "No table argument found for name: " + name); + } + if (candidates.size() > 1) { + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, name.getOriginalParts().get(0), "Ambiguous reference: multiple table arguments found for name: " + name); + } + TableArgumentAnalysis argument = getOnlyElement(candidates); + if (!referencedArguments.add(argument.getArgumentName())) { + // multiple references to argument in COPARTITION clause are implicitly prohibited by + // ISO/IEC TR REPORT 19075-7, p.33, Feature B203, “More than one copartition specification” + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, name.getOriginalParts().get(0), "Multiple references to table argument: %s in COPARTITION clause", name); + } + copartitionListBuilder.add(argument); + } + List copartitionList = copartitionListBuilder.build(); + + // analyze partitioning columns + copartitionList.stream() + .filter(argument -> !argument.getPartitionBy().isPresent()) + .findFirst().ifPresent(unpartitioned -> { + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, unpartitioned.getRelation(), "Table %s referenced in COPARTITION clause is not partitioned", unpartitioned.getName().orElseThrow(() -> new IllegalStateException("Missing unpartitioned TableArgumentAnalysis name"))); + }); + // TODO(#26147): make sure that copartitioned tables cannot have empty partitioning lists. + // ISO/IEC TR REPORT 19075-7, 4.5 Partitioning and ordering, p.25 is not clear: "With copartitioning, the copartitioned table arguments must have the same number of partitioning columns, + // and corresponding partitioning columns must be comparable. The DBMS effectively performs a full outer equijoin on the copartitioning columns" + copartitionList.stream() + .filter(argument -> argument.getPartitionBy().orElseThrow(() -> new IllegalStateException("PartitionBy not present in copartitionList")).isEmpty()) + .findFirst().ifPresent(partitionedOnEmpty -> { + // table is partitioned but no partitioning columns are specified (single partition) + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, partitionedOnEmpty.getRelation(), "No partitioning columns specified for table %s referenced in COPARTITION clause", partitionedOnEmpty.getName().orElseThrow(() -> new IllegalStateException("Missing partitionedOnEmpty TableArgumentAnalysis name"))); + }); + List> partitioningColumns = copartitionList.stream() + .map(TableArgumentAnalysis::getPartitionBy) + .map(opt -> opt.orElseThrow(() -> new IllegalStateException("PartitionBy not present in partitioningColumns"))) + .collect(toImmutableList()); + if (partitioningColumns.stream() + .map(List::size) + .distinct() + .count() > 1) { + throw new SemanticException(TABLE_FUNCTION_INVALID_COPARTITIONING, nameList.get(0).getOriginalParts().get(0), "Numbers of partitioning columns in copartitioned tables do not match"); } - throw new SemanticException(INVALID_FUNCTION_ARGUMENT, argument, "Invalid argument %s. Expected descriptor, got %s", argumentSpecification.getName(), actualType); + + // coerce corresponding copartition columns to common supertype + for (int index = 0; index < partitioningColumns.get(0).size(); index++) { + Type commonSuperType = analysis.getType(partitioningColumns.get(0).get(index)); + // find common supertype + for (List columnList : partitioningColumns) { + Optional superType = functionAndTypeResolver.getCommonSuperType(commonSuperType, analysis.getType(columnList.get(index))); + if (!superType.isPresent()) { + throw new SemanticException(TYPE_MISMATCH, nameList.get(0).getOriginalParts().get(0), "Partitioning columns in copartitioned tables have incompatible types"); + } + commonSuperType = superType.get(); + } + for (List columnList : partitioningColumns) { + Expression column = columnList.get(index); + Type type = analysis.getType(column); + if (!type.equals(commonSuperType)) { + if (!functionAndTypeResolver.canCoerce(type, commonSuperType)) { + throw new SemanticException(TYPE_MISMATCH, column, "Cannot coerce column of type %s to common supertype: %s", type.getDisplayName(), commonSuperType.getDisplayName()); + } + analysis.addCoercion(column, commonSuperType, functionAndTypeResolver.isTypeOnlyCoercion(type, commonSuperType)); + } + } + } + + // record the resolved copartition arguments by argument names + copartitionBuilder.add(copartitionList.stream() + .map(TableArgumentAnalysis::getArgumentName) + .collect(toImmutableList())); } - throw new SemanticException(NOT_IMPLEMENTED, argument, "Descriptor arguments are not yet supported for table functions"); + + return copartitionBuilder.build(); } @Override @@ -1601,6 +1939,7 @@ protected Scope visitTable(Table table, Optional scope) if (withQuery.isPresent()) { Query query = withQuery.get().getQuery(); analysis.registerNamedQuery(table, query, false); + analysis.setRelationName(table, table.getName()); // re-alias the fields with the name assigned to the query in the WITH declaration RelationType queryDescriptor = analysis.getOutputDescriptor(query); @@ -1652,6 +1991,7 @@ protected Scope visitTable(Table table, Optional scope) } QualifiedObjectName name = createQualifiedObjectName(session, table, table.getName(), metadata); + analysis.setRelationName(table, QualifiedName.of(name.getCatalogName(), name.getSchemaName(), name.getObjectName())); if (name.getObjectName().isEmpty()) { throw new SemanticException(MISSING_TABLE, table, "Table name is empty"); } @@ -2025,10 +2365,19 @@ private MaterializedViewStatus getMaterializedViewStatus(QualifiedObjectName mat @Override protected Scope visitAliasedRelation(AliasedRelation relation, Optional scope) { + analysis.setRelationName(relation, QualifiedName.of(relation.getAlias().getValue())); + analysis.addAliased(relation.getRelation()); Scope relationScope = process(relation.getRelation(), scope); - // todo this check should be inside of TupleDescriptor.withAlias, but the exception needs the node object RelationType relationType = relationScope.getRelationType(); + + // special-handle table function invocation + if (relation.getRelation() instanceof TableFunctionInvocation) { + return createAndAssignScope(relation, scope, + aliasTableFunctionInvocation(relation, relationType, (TableFunctionInvocation) relation.getRelation())); + } + + // todo this check should be inside of TupleDescriptor.withAlias, but the exception needs the node object if (relation.getColumnNames() != null) { int totalColumns = relationType.getVisibleFieldCount(); if (totalColumns != relation.getColumnNames().size()) { @@ -2059,6 +2408,85 @@ protected Scope visitAliasedRelation(AliasedRelation relation, Optional s return createAndAssignScope(relation, scope, descriptor); } + // As described by the SQL standard ISO/IEC 9075-2, 7.6
, p. 409 + private RelationType aliasTableFunctionInvocation(AliasedRelation relation, RelationType relationType, TableFunctionInvocation function) + { + TableFunctionInvocationAnalysis tableFunctionAnalysis = analysis.getTableFunctionAnalysis(function); + int properColumnsCount = tableFunctionAnalysis.getProperColumnsCount(); + + // check that relation alias is different from range variables of all table arguments + tableFunctionAnalysis.getTableArgumentAnalyses().stream() + .map(TableArgumentAnalysis::getName) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(name -> name.hasSuffix(QualifiedName.of(ImmutableList.of(relation.getAlias())))) + .findFirst() + .ifPresent(name -> { + throw new SemanticException(TABLE_FUNCTION_DUPLICATE_RANGE_VARIABLE, relation.getAlias(), "Relation alias: %s is a duplicate of input table name: %s", relation.getAlias(), name); + }); + + // build the new relation type. the alias must be applied to the proper columns only, + // and it must not shadow the range variables exposed by the table arguments + ImmutableList.Builder fieldsBuilder = ImmutableList.builder(); + // first, put the table function's proper columns with alias + if (relation.getColumnNames() != null) { + // check that number of column aliases matches number of table function's proper columns + if (properColumnsCount != relation.getColumnNames().size()) { + throw new SemanticException(MISMATCHED_COLUMN_ALIASES, relation, "Column alias list has %s entries but table function has %s proper columns", relation.getColumnNames().size(), properColumnsCount); + } + for (int i = 0; i < properColumnsCount; i++) { + // proper columns are not hidden, so we don't need to skip hidden fields + Field field = relationType.getFieldByIndex(i); + fieldsBuilder.add(Field.newQualified( + field.getNodeLocation(), + QualifiedName.of(ImmutableList.of(relation.getAlias())), + Optional.of(relation.getColumnNames().get(i).getCanonicalValue()), // although the canonical name is recorded, fields are resolved case-insensitive + field.getType(), + field.isHidden(), + field.getOriginTable(), + field.getOriginColumnName(), + field.isAliased())); + } + } + else { + for (int i = 0; i < properColumnsCount; i++) { + Field field = relationType.getFieldByIndex(i); + fieldsBuilder.add(Field.newQualified( + field.getNodeLocation(), + QualifiedName.of(ImmutableList.of(relation.getAlias())), + field.getName(), + field.getType(), + field.isHidden(), + field.getOriginTable(), + field.getOriginColumnName(), + field.isAliased())); + } + } + + // append remaining fields. They are not being aliased, so hidden fields are included + for (int i = properColumnsCount; i < relationType.getAllFieldCount(); i++) { + fieldsBuilder.add(relationType.getFieldByIndex(i)); + } + + List fields = fieldsBuilder.build(); + + // check that there are no duplicate names within the table function's proper columns + Set names = new HashSet<>(); + fields.subList(0, properColumnsCount).stream() + .map(Field::getName) + .filter(Optional::isPresent) + .map(Optional::get) + // field names are resolved case-insensitive + .map(name -> name.toLowerCase(ENGLISH)) + .forEach(name -> { + if (!names.add(name)) { + throw new SemanticException(DUPLICATE_COLUMN_NAME, relation.getRelation(), "Duplicate name of table function proper column: " + name); + } + }); + + return new RelationType(fields); + } + @Override protected Scope visitSampledRelation(SampledRelation relation, Optional scope) { @@ -2102,9 +2530,33 @@ protected Scope visitSampledRelation(SampledRelation relation, Optional s analysis.setSampleRatio(relation, samplePercentageValue / 100); Scope relationScope = process(relation.getRelation(), scope); + + // TABLESAMPLE cannot be applied to a polymorphic table function (SQL standard ISO/IEC 9075-2, 7.6
, p. 409) + // Note: the below method finds a table function immediately nested in SampledRelation, or aliased. + // Potentially, a table function could be also nested with intervening PatternRecognitionRelation. + // Such case is handled in visitPatternRecognitionRelation(). + validateNoNestedTableFunction(relation.getRelation(), "sample"); + return createAndAssignScope(relation, scope, relationScope.getRelationType()); } + // this method should run after the `base` relation is processed, so that it is + // determined whether the table function is polymorphic + private void validateNoNestedTableFunction(Relation base, String context) + { + TableFunctionInvocation tableFunctionInvocation = null; + if (base instanceof TableFunctionInvocation) { + tableFunctionInvocation = (TableFunctionInvocation) base; + } + else if (base instanceof AliasedRelation && + ((AliasedRelation) base).getRelation() instanceof TableFunctionInvocation) { + tableFunctionInvocation = (TableFunctionInvocation) ((AliasedRelation) base).getRelation(); + } + if (tableFunctionInvocation != null && analysis.isPolymorphicTableFunction(tableFunctionInvocation)) { + throw new SemanticException(TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, base, "Cannot apply %s to polymorphic table function invocation", context); + } + } + @Override protected Scope visitTableSubquery(TableSubquery node, Optional scope) { @@ -2513,6 +2965,12 @@ protected Scope visitUpdate(Update update, Optional scope) return createAndAssignScope(update, scope, Field.newUnqualified(update.getLocation(), "rows", BIGINT)); } + @Override + protected Scope visitMerge(Merge merge, Optional scope) + { + throw new PrestoException(StandardErrorCode.NOT_SUPPORTED, "This connector does not support MERGE INTO statements"); + } + private Scope analyzeJoinUsing(Join node, List columns, Optional scope, Scope left, Scope right) { List joinFields = new ArrayList<>(); @@ -2858,7 +3316,7 @@ public Expression rewriteIdentifier(Identifier reference, Void context, Expressi } if (expressions.size() == 1) { - return Iterables.getOnlyElement(expressions); + return getOnlyElement(expressions); } // otherwise, couldn't resolve name against output aliases, so fall through... @@ -3031,7 +3489,7 @@ else if (item instanceof SingleColumn) { name = QualifiedName.of(((Identifier) expression).getValue()); } else if (expression instanceof DereferenceExpression) { - name = DereferenceExpression.getQualifiedName((DereferenceExpression) expression); + name = getQualifiedName((DereferenceExpression) expression); } if (name != null) { @@ -3656,4 +4114,48 @@ private static boolean hasScopeAsLocalParent(Scope root, Scope parent) return false; } + + private static final class ArgumentAnalysis + { + private final Argument argument; + private final Optional tableArgumentAnalysis; + + public ArgumentAnalysis(Argument argument, Optional tableArgumentAnalysis) + { + this.argument = requireNonNull(argument, "argument is null"); + this.tableArgumentAnalysis = requireNonNull(tableArgumentAnalysis, "tableArgumentAnalysis is null"); + } + + public Argument getArgument() + { + return argument; + } + + public Optional getTableArgumentAnalysis() + { + return tableArgumentAnalysis; + } + } + + private static final class ArgumentsAnalysis + { + private final Map passedArguments; + private final List tableArgumentAnalyses; + + public ArgumentsAnalysis(Map passedArguments, List tableArgumentAnalyses) + { + this.passedArguments = ImmutableMap.copyOf(requireNonNull(passedArguments, "passedArguments is null")); + this.tableArgumentAnalyses = ImmutableList.copyOf(requireNonNull(tableArgumentAnalyses, "tableArgumentAnalyses is null")); + } + + public Map getPassedArguments() + { + return passedArguments; + } + + public List getTableArgumentAnalyses() + { + return tableArgumentAnalyses; + } + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 85a52a298bcb7..1050161cbdc5d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -3485,8 +3485,7 @@ else if (target instanceof InsertHandle) { return metadata.finishInsert(session, ((InsertHandle) target).getHandle(), fragments, statistics); } else if (target instanceof DeleteHandle) { - metadata.finishDelete(session, ((DeleteHandle) target).getHandle(), fragments); - return Optional.empty(); + return metadata.finishDeleteWithOutput(session, ((DeleteHandle) target).getHandle(), fragments); } else if (target instanceof RefreshMaterializedViewHandle) { return metadata.finishRefreshMaterializedView(session, ((RefreshMaterializedViewHandle) target).getHandle(), fragments, statistics); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java index 97c95e0d7bf05..1c6f151d70efc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.eventlistener.OutputColumnMetadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.SequenceNode; @@ -66,6 +67,21 @@ public Void visitTableWriter(TableWriterNode node, Void context) return null; } + @Override + public Void visitTableFinish(TableFinishNode node, Void context) + { + if (node.getTarget().isPresent() && node.getTarget().get() instanceof TableWriterNode.DeleteHandle) { + TableWriterNode.DeleteHandle deleteHandle = (TableWriterNode.DeleteHandle) node.getTarget().get(); + connectorId = deleteHandle.getConnectorId(); + checkState(schemaTableName == null || schemaTableName.equals(deleteHandle.getSchemaTableName()), + "cannot have more than a single create, insert or delete in a query"); + schemaTableName = deleteHandle.getSchemaTableName(); + outputColumns = deleteHandle.getOutputColumns(); + return null; + } + return super.visitTableFinish(node, context); + } + public Void visitSequence(SequenceNode node, Void context) { // Left children of sequence are ignored since they don't output anything diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java index 7199daeb239f1..10ad2b1f68e7d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner; import com.facebook.presto.Session; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.common.predicate.TupleDomain; @@ -114,7 +115,6 @@ import static com.facebook.presto.sql.NodeUtils.getSortItemsFromOrderBy; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.isNumericType; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getSourceLocation; -import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.PlannerUtils.newVariable; import static com.facebook.presto.sql.planner.PlannerUtils.toOrderingScheme; import static com.facebook.presto.sql.planner.PlannerUtils.toSortOrder; @@ -1112,7 +1112,7 @@ private FrameBoundPlanAndSymbols planFrameBound(PlanBuilder subPlan, PlanAndMapp // First, append filter to validate offset values. They mustn't be negative or null. VariableReferenceExpression offsetSymbol = coercions.get(frameOffset.get()); Expression zeroOffset = zeroOfType(TypeProvider.viewOf(variableAllocator.getVariables()).get(offsetSymbol)); - FunctionHandle fail = metadata.getFunctionAndTypeManager().resolveFunction(Optional.empty(), Optional.empty(), QualifiedObjectName.valueOf("presto.default.fail"), fromTypes(VARCHAR)); + CatalogSchemaName defaultNamespace = metadata.getFunctionAndTypeManager().getDefaultNamespace(); Expression predicate = new IfExpression( new ComparisonExpression( GREATER_THAN_OR_EQUAL, @@ -1121,7 +1121,7 @@ private FrameBoundPlanAndSymbols planFrameBound(PlanBuilder subPlan, PlanAndMapp TRUE_LITERAL, new Cast( new FunctionCall( - QualifiedName.of("presto", "default", "fail"), + QualifiedName.of(defaultNamespace.getCatalogName(), defaultNamespace.getSchemaName(), "fail"), ImmutableList.of(new Cast(new StringLiteral("Window frame offset value must not be negative or null"), VARCHAR.getTypeSignature().toString()))), BOOLEAN.getTypeSignature().toString())); subPlan = subPlan.withNewRoot(new FilterNode( diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java index 11241f2f3dce1..0bd4e47fc67b8 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RelationPlanner.java @@ -51,6 +51,7 @@ import com.facebook.presto.sql.analyzer.RelationId; import com.facebook.presto.sql.analyzer.RelationType; import com.facebook.presto.sql.analyzer.Scope; +import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; import com.facebook.presto.sql.planner.optimizations.SampleNodeUtil; @@ -86,7 +87,9 @@ import com.facebook.presto.sql.tree.SetOperation; import com.facebook.presto.sql.tree.SymbolReference; import com.facebook.presto.sql.tree.Table; +import com.facebook.presto.sql.tree.TableFunctionDescriptorArgument; import com.facebook.presto.sql.tree.TableFunctionInvocation; +import com.facebook.presto.sql.tree.TableFunctionTableArgument; import com.facebook.presto.sql.tree.TableSubquery; import com.facebook.presto.sql.tree.Union; import com.facebook.presto.sql.tree.Unnest; @@ -122,6 +125,7 @@ import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.isEqualComparisonExpression; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.resolveEnumLiteral; import static com.facebook.presto.sql.analyzer.FeaturesConfig.CteMaterializationStrategy.NONE; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED; import static com.facebook.presto.sql.analyzer.SemanticExceptions.notSupportedException; import static com.facebook.presto.sql.planner.PlannerUtils.newVariable; import static com.facebook.presto.sql.planner.TranslateExpressionsUtil.toRowExpression; @@ -303,6 +307,15 @@ private RelationPlan addColumnMasks(Table table, RelationPlan plan, SqlPlannerCo @Override protected RelationPlan visitTableFunctionInvocation(TableFunctionInvocation node, SqlPlannerContext context) { + node.getArguments().stream() + .forEach(argument -> { + if (argument.getValue() instanceof TableFunctionTableArgument) { + throw new SemanticException(NOT_SUPPORTED, argument, "Table arguments are not yet supported for table functions"); + } + if (argument.getValue() instanceof TableFunctionDescriptorArgument) { + throw new SemanticException(NOT_SUPPORTED, argument, "Descriptor arguments are not yet supported for table functions"); + } + }); Analysis.TableFunctionInvocationAnalysis functionAnalysis = analysis.getTableFunctionAnalysis(node); // TODO handle input relations: diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java index 8c89750b6d78a..3b24606a808ef 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/RowExpressionInterpreter.java @@ -264,7 +264,7 @@ public Object visitCall(CallExpression node, Object context) (!functionMetadata.isDeterministic() || hasUnresolvedValue(argumentValues) || isDynamicFilter(node) || - resolution.isFailFunction(functionHandle))) { + resolution.isJavaBuiltInFailFunction(functionHandle))) { return call(node.getDisplayName(), functionHandle, node.getType(), toRowExpressions(argumentValues, node.getArguments())); } diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index 337582257bad4..3aa2a3b544ca7 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -300,9 +300,12 @@ public boolean isTryFunction(FunctionHandle functionHandle) return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().getObjectName().equals("$internal$try"); } - public boolean isFailFunction(FunctionHandle functionHandle) + public boolean isJavaBuiltInFailFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("fail"))); + // todo: Revert this hack once constant folding support lands in C++. + // For now, we always use the presto.default.fail function even when the default namespace is switched. + // This is done for consistency since the BuiltInNamespaceRewriter rewrites the functionHandles to presto.default functionHandles. + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "fail")); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java index 10256d652f66b..caf443379fb4c 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/relational/RowExpressionOptimizer.java @@ -15,6 +15,8 @@ import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.expressions.RowExpressionRewriter; +import com.facebook.presto.expressions.RowExpressionTreeRewriter; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.ConnectorSession; @@ -24,10 +26,10 @@ import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ExpressionOptimizer; import com.facebook.presto.spi.relation.RowExpression; -import com.facebook.presto.spi.relation.RowExpressionVisitor; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.sql.planner.RowExpressionInterpreter; +import com.google.common.collect.ImmutableList; import jakarta.annotation.Nullable; import java.util.IdentityHashMap; @@ -105,7 +107,7 @@ public RowExpression convertToInterpreterNamespace(RowExpression expression) // No need to replace built-in namespaces if the default namespace is already the Java built-in namespace return expression; } - return expression.accept(new ReplaceBuiltInNamespaces(), null); + return RowExpressionTreeRewriter.rewriteWith(new ReplaceBuiltInNamespaces(), expression, null); } public RowExpression restoreOriginalNamespaces(RowExpression expression) @@ -113,20 +115,14 @@ public RowExpression restoreOriginalNamespaces(RowExpression expression) if (defaultToOriginalFunctionHandles.isEmpty()) { return expression; } - return expression.accept(new ReplaceOriginalNamespaces(), null); + return RowExpressionTreeRewriter.rewriteWith(new ReplaceOriginalNamespaces(), expression, null); } private class ReplaceBuiltInNamespaces - implements RowExpressionVisitor + extends RowExpressionRewriter { @Override - public RowExpression visitExpression(RowExpression expression, Void context) - { - return expression; - } - - @Override - public RowExpression visitCall(CallExpression call, Void context) + public RowExpression rewriteCall(CallExpression call, Void context, RowExpressionTreeRewriter treeRewriter) { FunctionHandle functionHandle = call.getFunctionHandle(); FunctionMetadata functionMetadata = functionAndTypeManager.getFunctionMetadata(functionHandle); @@ -143,7 +139,8 @@ public RowExpression visitCall(CallExpression call, Void context) } catch (PrestoException e) { if (e.getErrorCode().equals(FUNCTION_NOT_FOUND.toErrorCode())) { - return call; // If the function is not found in the Java built-in namespace, return the original call + // If the function is not found in the Java built-in namespace, let default rewriter handle it + return null; } throw e; // Rethrow other exceptions } @@ -152,39 +149,42 @@ public RowExpression visitCall(CallExpression call, Void context) format("FunctionHandle %s in the Java built-in namespace (%s) is not eligible to be evaluated in the coordinator", javaNamespaceFunctionHandle, JAVA_BUILTIN_NAMESPACE)); defaultToOriginalFunctionHandles.put(javaNamespaceFunctionHandle, functionHandle); + ImmutableList rewrittenArgs = call.getArguments().stream() + .map(arg -> treeRewriter.rewrite(arg, context)) + .collect(toImmutableList()); return new CallExpression( call.getSourceLocation(), call.getDisplayName(), javaNamespaceFunctionHandle, call.getType(), - call.getArguments()); + rewrittenArgs); } - return call; + + // Return null to let the default rewriter handle it (which will rewrite children automatically) + return null; } } private class ReplaceOriginalNamespaces - implements RowExpressionVisitor + extends RowExpressionRewriter { @Override - public RowExpression visitExpression(RowExpression expression, Void context) - { - return expression; - } - - @Override - public RowExpression visitCall(CallExpression call, Void context) + public RowExpression rewriteCall(CallExpression call, Void context, RowExpressionTreeRewriter treeRewriter) { if (defaultToOriginalFunctionHandles.containsKey(call.getFunctionHandle())) { FunctionHandle originalFunctionHandle = defaultToOriginalFunctionHandles.get(call.getFunctionHandle()); + ImmutableList rewrittenArgs = call.getArguments().stream() + .map(arg -> treeRewriter.rewrite(arg, context)) + .collect(toImmutableList()); return new CallExpression( call.getSourceLocation(), call.getDisplayName(), originalFunctionHandle, call.getType(), - call.getArguments()); + rewrittenArgs); } - return call; + // Return null to let the default rewriter handle it (which will rewrite children automatically) + return null; } } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/TestingAccessControlManager.java b/presto-main-base/src/main/java/com/facebook/presto/testing/TestingAccessControlManager.java index dd932e971e923..ca16e92866171 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/testing/TestingAccessControlManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/TestingAccessControlManager.java @@ -101,9 +101,16 @@ public class TestingAccessControlManager @Inject public TestingAccessControlManager(TransactionManager transactionManager) + { + this(transactionManager, true); + } + + public TestingAccessControlManager(TransactionManager transactionManager, boolean loadDefaultSystemAccessControl) { super(transactionManager); - setSystemAccessControl(AllowAllSystemAccessControl.NAME, ImmutableMap.of()); + if (loadDefaultSystemAccessControl) { + setSystemAccessControl(AllowAllSystemAccessControl.NAME, ImmutableMap.of()); + } } public static TestingPrivilege privilege(String entityName, TestingPrivilegeType type) diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/TestingPrestoServerModule.java b/presto-main-base/src/main/java/com/facebook/presto/testing/TestingPrestoServerModule.java new file mode 100644 index 0000000000000..a0139a22ac673 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/TestingPrestoServerModule.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.testing; + +import com.facebook.presto.eventlistener.EventListenerConfig; +import com.facebook.presto.eventlistener.EventListenerManager; +import com.facebook.presto.security.AccessControlManager; +import com.facebook.presto.server.GracefulShutdownHandler; +import com.facebook.presto.server.security.PrestoAuthenticatorManager; +import com.facebook.presto.spi.security.AccessControl; +import com.facebook.presto.storage.TempStorageManager; +import com.facebook.presto.transaction.TransactionManager; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; + +import static java.util.Objects.requireNonNull; + +public class TestingPrestoServerModule + implements Module +{ + private final boolean loadDefaultSystemAccessControl; + + public TestingPrestoServerModule(boolean loadDefaultSystemAccessControl) + { + this.loadDefaultSystemAccessControl = loadDefaultSystemAccessControl; + } + + @Override + public void configure(Binder binder) + { + binder.bind(PrestoAuthenticatorManager.class).in(Scopes.SINGLETON); + binder.bind(TestingEventListenerManager.class).in(Scopes.SINGLETON); + binder.bind(TestingTempStorageManager.class).in(Scopes.SINGLETON); + binder.bind(EventListenerManager.class).to(TestingEventListenerManager.class).in(Scopes.SINGLETON); + binder.bind(EventListenerConfig.class).in(Scopes.SINGLETON); + binder.bind(TempStorageManager.class).to(TestingTempStorageManager.class).in(Scopes.SINGLETON); + binder.bind(AccessControl.class).to(AccessControlManager.class).in(Scopes.SINGLETON); + binder.bind(GracefulShutdownHandler.class).in(Scopes.SINGLETON); + binder.bind(ProcedureTester.class).in(Scopes.SINGLETON); + } + + @Provides + @Singleton + public AccessControlManager createAccessControlManager(TransactionManager transactionManager) + { + requireNonNull(transactionManager, "transactionManager is null"); + return new TestingAccessControlManager(transactionManager, loadDefaultSystemAccessControl); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java b/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java index f43ce90779cb7..ba91087dcc245 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java +++ b/presto-main-base/src/main/java/com/facebook/presto/util/AnalyzerUtil.java @@ -102,14 +102,14 @@ public static AnalyzerContext getAnalyzerContext( public static void checkAccessPermissions(AccessControlReferences accessControlReferences, String query) { // Query check - checkAccessPermissionsForQuery(accessControlReferences, query); + checkQueryIntegrity(accessControlReferences, query); // Table checks checkAccessPermissionsForTable(accessControlReferences); // Table Column checks checkAccessPermissionsForColumns(accessControlReferences); } - private static void checkAccessPermissionsForQuery(AccessControlReferences accessControlReferences, String query) + public static void checkQueryIntegrity(AccessControlReferences accessControlReferences, String query) { AccessControlInfo queryAccessControlInfo = accessControlReferences.getQueryAccessControlInfo(); // Only check access if query gets analyzed @@ -124,7 +124,7 @@ private static void checkAccessPermissionsForQuery(AccessControlReferences acces } } - private static void checkAccessPermissionsForColumns(AccessControlReferences accessControlReferences) + public static void checkAccessPermissionsForColumns(AccessControlReferences accessControlReferences) { accessControlReferences.getTableColumnAndSubfieldReferencesForAccessControl() .forEach((accessControlInfo, tableColumnReferences) -> @@ -140,7 +140,7 @@ private static void checkAccessPermissionsForColumns(AccessControlReferences acc })); } - private static void checkAccessPermissionsForTable(AccessControlReferences accessControlReferences) + public static void checkAccessPermissionsForTable(AccessControlReferences accessControlReferences) { accessControlReferences.getTableReferences().forEach((accessControlRole, accessControlInfoForTables) -> accessControlInfoForTables.forEach(accessControlInfoForTable -> { AccessControlInfo accessControlInfo = accessControlInfoForTable.getAccessControlInfo(); diff --git a/presto-main-base/src/test/java/com/facebook/presto/connector/tvf/TestingTableFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/connector/tvf/TestingTableFunctions.java index f2831f4bf10e1..96373d826b50a 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/connector/tvf/TestingTableFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/connector/tvf/TestingTableFunctions.java @@ -42,7 +42,9 @@ import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.spi.function.table.ReturnTypeSpecification.DescribedTable; import static com.facebook.presto.spi.function.table.ReturnTypeSpecification.GenericTable.GENERIC_TABLE; +import static com.facebook.presto.spi.function.table.ReturnTypeSpecification.OnlyPassThrough.ONLY_PASS_THROUGH; import static io.airlift.slice.Slices.utf8Slice; import static java.util.Objects.requireNonNull; @@ -57,6 +59,11 @@ public class TestingTableFunctions .returnedType(new Descriptor(ImmutableList.of(new Descriptor.Field(COLUMN_NAME, Optional.of(BOOLEAN))))) .build(); + private static final TableFunctionAnalysis NO_DESCRIPTOR_ANALYSIS = TableFunctionAnalysis.builder() + .handle(HANDLE) + .requiredColumns("INPUT", ImmutableList.of(0)) + .build(); + public static class TestConnectorTableFunction extends AbstractConnectorTableFunction { @@ -84,7 +91,7 @@ public static class TestConnectorTableFunction2 public TestConnectorTableFunction2() { - super(SCHEMA_NAME, TEST_FUNCTION_2, ImmutableList.of(), ReturnTypeSpecification.OnlyPassThrough.ONLY_PASS_THROUGH); + super(SCHEMA_NAME, TEST_FUNCTION_2, ImmutableList.of(), ONLY_PASS_THROUGH); } @Override @@ -101,7 +108,7 @@ public static class NullArgumentsTableFunction public NullArgumentsTableFunction() { - super(SCHEMA_NAME, NULL_ARGUMENTS_FUNCTION, null, ReturnTypeSpecification.OnlyPassThrough.ONLY_PASS_THROUGH); + super(SCHEMA_NAME, NULL_ARGUMENTS_FUNCTION, null, ONLY_PASS_THROUGH); } @Override @@ -123,7 +130,7 @@ public DuplicateArgumentsTableFunction() ImmutableList.of( ScalarArgumentSpecification.builder().name("a").type(INTEGER).build(), ScalarArgumentSpecification.builder().name("a").type(INTEGER).build()), - ReturnTypeSpecification.OnlyPassThrough.ONLY_PASS_THROUGH); + ONLY_PASS_THROUGH); } @Override @@ -144,7 +151,7 @@ public MultipleRSTableFunction() MULTIPLE_SOURCES_FUNCTION, ImmutableList.of(TableArgumentSpecification.builder().name("t").rowSemantics().build(), TableArgumentSpecification.builder().name("t2").rowSemantics().build()), - ReturnTypeSpecification.OnlyPassThrough.ONLY_PASS_THROUGH); + ONLY_PASS_THROUGH); } @Override @@ -334,4 +341,180 @@ public SchemaFunctionName getSchemaFunctionName() return schemaFunctionName; } } + + public static class TableArgumentRowSemanticsFunction + extends AbstractConnectorTableFunction + { + public TableArgumentRowSemanticsFunction() + { + super( + SCHEMA_NAME, + "table_argument_row_semantics_function", + ImmutableList.of( + TableArgumentSpecification.builder() + .name("INPUT") + .rowSemantics() + .build()), + GENERIC_TABLE); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return TableFunctionAnalysis.builder() + .handle(HANDLE) + .returnedType(new Descriptor(ImmutableList.of(new Descriptor.Field(COLUMN_NAME, Optional.of(BOOLEAN))))) + .requiredColumns("INPUT", ImmutableList.of(0)) + .build(); + } + } + + public static class TwoTableArgumentsFunction + extends AbstractConnectorTableFunction + { + public TwoTableArgumentsFunction() + { + super( + SCHEMA_NAME, + "two_table_arguments_function", + ImmutableList.of( + TableArgumentSpecification.builder() + .name("INPUT1") + .build(), + TableArgumentSpecification.builder() + .name("INPUT2") + .build()), + GENERIC_TABLE); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return TableFunctionAnalysis.builder() + .handle(HANDLE) + .returnedType(new Descriptor(ImmutableList.of(new Descriptor.Field(COLUMN_NAME, Optional.of(BOOLEAN))))) + .requiredColumns("INPUT1", ImmutableList.of(0)) + .requiredColumns("INPUT2", ImmutableList.of(0)) + .build(); + } + } + + public static class OnlyPassThroughFunction + extends AbstractConnectorTableFunction + { + public OnlyPassThroughFunction() + { + super( + SCHEMA_NAME, + "only_pass_through_function", + ImmutableList.of( + TableArgumentSpecification.builder() + .name("INPUT") + .passThroughColumns() + .build()), + ONLY_PASS_THROUGH); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return NO_DESCRIPTOR_ANALYSIS; + } + } + + public static class MonomorphicStaticReturnTypeFunction + extends AbstractConnectorTableFunction + { + public MonomorphicStaticReturnTypeFunction() + { + super( + SCHEMA_NAME, + "monomorphic_static_return_type_function", + ImmutableList.of(), + new DescribedTable(Descriptor.descriptor( + ImmutableList.of("a", "b"), + ImmutableList.of(BOOLEAN, INTEGER)))); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return TableFunctionAnalysis.builder() + .handle(HANDLE) + .build(); + } + } + + public static class PolymorphicStaticReturnTypeFunction + extends AbstractConnectorTableFunction + { + public PolymorphicStaticReturnTypeFunction() + { + super( + SCHEMA_NAME, + "polymorphic_static_return_type_function", + ImmutableList.of(TableArgumentSpecification.builder() + .name("INPUT") + .build()), + new DescribedTable(Descriptor.descriptor( + ImmutableList.of("a", "b"), + ImmutableList.of(BOOLEAN, INTEGER)))); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return NO_DESCRIPTOR_ANALYSIS; + } + } + + public static class PassThroughFunction + extends AbstractConnectorTableFunction + { + public PassThroughFunction() + { + super( + SCHEMA_NAME, + "pass_through_function", + ImmutableList.of(TableArgumentSpecification.builder() + .name("INPUT") + .passThroughColumns() + .build()), + new DescribedTable(Descriptor.descriptor( + ImmutableList.of("x"), + ImmutableList.of(BOOLEAN)))); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return NO_DESCRIPTOR_ANALYSIS; + } + } + + public static class RequiredColumnsFunction + extends AbstractConnectorTableFunction + { + public RequiredColumnsFunction() + { + super( + SCHEMA_NAME, + "required_columns_function", + ImmutableList.of( + TableArgumentSpecification.builder() + .name("INPUT") + .build()), + GENERIC_TABLE); + } + + @Override + public TableFunctionAnalysis analyze(ConnectorSession session, ConnectorTransactionHandle transaction, Map arguments) + { + return TableFunctionAnalysis.builder() + .handle(HANDLE) + .returnedType(new Descriptor(ImmutableList.of(new Descriptor.Field("column", Optional.of(BOOLEAN))))) + .requiredColumns("INPUT", ImmutableList.of(0, 1)) + .build(); + } + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java b/presto-main-base/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java index 0925f42733a77..281997a3246b9 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java +++ b/presto-main-base/src/test/java/com/facebook/presto/execution/TestNodeScheduler.java @@ -1324,7 +1324,6 @@ public void testScheduleSplitsBasedOnTaskLoad() assertTrue(node.getNodeIdentifier().equals("other2") || node.getNodeIdentifier().equals("other3")); } - Map> splitsForTasks = new HashMap<>(); PlanNodeId planNodeId = new PlanNodeId("sourceId"); for (InternalNode node : assignments.keySet()) { Multimap splits = ArrayListMultimap.create(); diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index 64d33822ed2dc..c5ba12a7e8938 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -430,7 +430,7 @@ public DeleteTableHandle beginDelete(Session session, TableHandle tableHandle) } @Override - public void finishDelete(Session session, DeleteTableHandle tableHandle, Collection fragments) + public Optional finishDeleteWithOutput(Session session, DeleteTableHandle tableHandle, Collection fragments) { throw new UnsupportedOperationException(); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/security/TestStatsRecordingSystemAccessControl.java b/presto-main-base/src/test/java/com/facebook/presto/security/TestStatsRecordingSystemAccessControl.java new file mode 100644 index 0000000000000..70b1109c71b3c --- /dev/null +++ b/presto-main-base/src/test/java/com/facebook/presto/security/TestStatsRecordingSystemAccessControl.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.security; + +import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.spi.QueryId; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.security.AccessControlContext; +import com.facebook.presto.spi.security.SystemAccessControl; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Optional; + +import static com.facebook.presto.spi.testing.InterfaceTestUtils.assertAllMethodsOverridden; +import static org.testng.Assert.assertEquals; + +public class TestStatsRecordingSystemAccessControl +{ + public static final AccessControlContext CONTEXT = new AccessControlContext(new QueryId("query_id"), Optional.empty(), Collections.emptySet(), Optional.empty(), WarningCollector.NOOP, new RuntimeStats(), Optional.empty(), Optional.empty(), Optional.empty()); + + @Test + public void testEverythingDelegated() + { + assertAllMethodsOverridden(SystemAccessControl.class, StatsRecordingSystemAccessControl.class); + } + + @Test + public void testStatsRecording() + { + SystemAccessControl delegate = new AllowAllSystemAccessControl(); + StatsRecordingSystemAccessControl statsRecordingAccessControl = new StatsRecordingSystemAccessControl(delegate); + + assertEquals(statsRecordingAccessControl.getStats().getCheckCanAccessCatalog().getTime().getAllTime().getCount(), 0.0); + + statsRecordingAccessControl.checkCanAccessCatalog(null, CONTEXT, "test-catalog"); + + assertEquals(statsRecordingAccessControl.getStats().getCheckCanAccessCatalog().getTime().getAllTime().getCount(), 1.0); + } +} diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java index 5819eadadcc6a..03909e16bc1c3 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/AbstractAnalyzerTest.java @@ -23,9 +23,16 @@ import com.facebook.presto.connector.informationSchema.InformationSchemaConnector; import com.facebook.presto.connector.system.SystemConnector; import com.facebook.presto.connector.tvf.TestingTableFunctions.DescriptorArgumentFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.MonomorphicStaticReturnTypeFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.OnlyPassThroughFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.PassThroughFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.PolymorphicStaticReturnTypeFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.RequiredColumnsFunction; import com.facebook.presto.connector.tvf.TestingTableFunctions.SimpleTableFunction; import com.facebook.presto.connector.tvf.TestingTableFunctions.TableArgumentFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.TableArgumentRowSemanticsFunction; import com.facebook.presto.connector.tvf.TestingTableFunctions.TwoScalarArgumentsFunction; +import com.facebook.presto.connector.tvf.TestingTableFunctions.TwoTableArgumentsFunction; import com.facebook.presto.execution.warnings.WarningCollectorConfig; import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutor; @@ -159,7 +166,14 @@ public void setup() new SimpleTableFunction(), new TwoScalarArgumentsFunction(), new TableArgumentFunction(), - new DescriptorArgumentFunction())); + new DescriptorArgumentFunction(), + new TableArgumentRowSemanticsFunction(), + new TwoTableArgumentsFunction(), + new OnlyPassThroughFunction(), + new MonomorphicStaticReturnTypeFunction(), + new PolymorphicStaticReturnTypeFunction(), + new PassThroughFunction(), + new RequiredColumnsFunction())); Catalog tpchTestCatalog = createTestingCatalog(TPCH_CATALOG, TPCH_CONNECTOR_ID); catalogManager.registerCatalog(tpchTestCatalog); @@ -517,7 +531,12 @@ protected void assertFails(SemanticErrorCode error, int line, int column, @Langu protected void assertFails(SemanticErrorCode error, String message, @Language("SQL") String query) { - assertFails(CLIENT_SESSION, error, message, query); + assertFails(CLIENT_SESSION, error, message, query, false); + } + + protected void assertFailsExact(SemanticErrorCode error, String message, @Language("SQL") String query) + { + assertFails(CLIENT_SESSION, error, message, query, true); } protected void assertFails(Session session, SemanticErrorCode error, @Language("SQL") String query) @@ -525,6 +544,11 @@ protected void assertFails(Session session, SemanticErrorCode error, @Language(" assertFails(session, error, Optional.empty(), query); } + protected void assertFails(Session session, SemanticErrorCode error, String message, @Language("SQL") String query) + { + assertFails(session, error, message, query, false); + } + private void assertFails(Session session, SemanticErrorCode error, Optional location, @Language("SQL") String query) { try { @@ -553,7 +577,7 @@ private void assertFails(Session session, SemanticErrorCode error, Optional 'foo'))"); analyze("SELECT * FROM TABLE(system.two_arguments_function('foo', 1))"); analyze("SELECT * FROM TABLE(system.two_arguments_function(text => 'foo', number => 1))"); - assertFails(INVALID_ARGUMENTS, + assertFails(TABLE_FUNCTION_INVALID_ARGUMENTS, "line 1:51: All arguments must be passed by name or all must be passed positionally", "SELECT * FROM TABLE(system.two_arguments_function('foo', number => 1))"); - assertFails(INVALID_ARGUMENTS, + assertFails(TABLE_FUNCTION_INVALID_ARGUMENTS, "line 1:51: All arguments must be passed by name or all must be passed positionally", "SELECT * FROM TABLE(system.two_arguments_function(text => 'foo', 1))"); - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:66: Duplicate argument name: TEXT", "SELECT * FROM TABLE(system.two_arguments_function(text => 'foo', text => 'bar'))"); // argument names are resolved in the canonical form - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:66: Duplicate argument name: TEXT", "SELECT * FROM TABLE(system.two_arguments_function(text => 'foo', TeXt => 'bar'))"); - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:66: Unexpected argument name: BAR", "SELECT * FROM TABLE(system.two_arguments_function(text => 'foo', bar => 'bar'))"); - assertFails(MISSING_ARGUMENT, + assertFails(TABLE_FUNCTION_MISSING_ARGUMENT, "line 1:51: Missing argument: TEXT", "SELECT * FROM TABLE(system.two_arguments_function(number => 1))"); } @@ -1997,15 +2005,15 @@ public void testScalarArgument() { analyze("SELECT * FROM TABLE(system.two_arguments_function('foo', 1))"); - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:64: Invalid argument NUMBER. Expected expression, got descriptor", "SELECT * FROM TABLE(system.two_arguments_function(text => 'a', number => DESCRIPTOR(x integer, y boolean)))"); - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:64: 'descriptor' function is not allowed as a table function argument", "SELECT * FROM TABLE(system.two_arguments_function(text => 'a', number => DESCRIPTOR(1 + 2)))"); - assertFails(INVALID_FUNCTION_ARGUMENT, + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:64: Invalid argument NUMBER. Expected expression, got table", "SELECT * FROM TABLE(system.two_arguments_function(text => 'a', number => TABLE(t1)))"); @@ -2013,4 +2021,322 @@ public void testScalarArgument() "line 1:74: Constant expression cannot contain a subquery", "SELECT * FROM TABLE(system.two_arguments_function(text => 'a', number => (SELECT 1)))"); } + + @Test + public void testTableArgument() + { + // cannot pass a table function as the argument + assertFails(NOT_SUPPORTED, + "line 1:52: Invalid table argument INPUT. Table functions are not allowed as table function arguments", + "SELECT * FROM TABLE(system.table_argument_function(input => my_schema.my_table_function(1)))"); + + assertThatThrownBy(() -> analyze("SELECT * FROM TABLE(system.table_argument_function(input => my_schema.my_table_function(arg => 1)))")) + .isInstanceOf(ParsingException.class) + .hasMessageContaining("line 1:93: mismatched input '=>'."); + + // cannot pass a table function as the argument, also preceding nested table function with TABLE is incorrect + assertThatThrownBy(() -> analyze("SELECT * FROM TABLE(system.table_argument_function(input => TABLE(my_schema.my_table_function(1))))")) + .isInstanceOf(ParsingException.class) + .hasMessageContaining("line 1:94: mismatched input '('."); + + // a table passed as the argument must be preceded with TABLE + analyze("SELECT * FROM TABLE(system.table_argument_function(input => TABLE(t1)))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:52: Invalid argument INPUT. Expected table, got expression", + "SELECT * FROM TABLE(system.table_argument_function(input => t1))"); + + // a query passed as the argument must be preceded with TABLE + analyze("SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT * FROM t1)))"); + + assertThatThrownBy(() -> analyze("SELECT * FROM TABLE(system.table_argument_function(input => SELECT * FROM t1))")) + .isInstanceOf(ParsingException.class) + .hasMessageContaining("line 1:61: mismatched input 'SELECT'."); + + // query passed as the argument is correlated + analyze("SELECT * FROM t1 CROSS JOIN LATERAL (SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT 1 WHERE a > 0))))"); + + // wrong argument type + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:52: Invalid argument INPUT. Expected table, got expression", + "SELECT * FROM TABLE(system.table_argument_function(input => 'foo'))"); + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:52: Invalid argument INPUT. Expected table, got descriptor", + "SELECT * FROM TABLE(system.table_argument_function(input => DESCRIPTOR(x int, y int)))"); + } + + @Test + public void testTableArgumentProperties() + { + analyze("SELECT * FROM TABLE(system.table_argument_function(input => TABLE(t1) PARTITION BY a KEEP WHEN EMPTY ORDER BY b))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:66: Invalid argument INPUT. Partitioning specified for table argument with row semantics", + "SELECT * FROM TABLE(system.table_argument_row_semantics_function(input => TABLE(t1) PARTITION BY a))"); + + assertFails(TABLE_FUNCTION_COLUMN_NOT_FOUND, + "line 1:92: Column b is not present in the input relation", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT 1 a) PARTITION BY b))"); + + assertFails(TABLE_FUNCTION_INVALID_COLUMN_REFERENCE, + "line 1:88: Expected column reference. Actual: 1", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT 1 a) ORDER BY 1))"); + + assertFails(TYPE_MISMATCH, + "line 1:104: HyperLogLog is not comparable, and therefore cannot be used in PARTITION BY", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT approx_set(1) a) PARTITION BY a))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, "line 1:66: Invalid argument INPUT. Ordering specified for table argument with row semantics", + "SELECT * FROM TABLE(system.table_argument_row_semantics_function(input => TABLE(t1) ORDER BY a))"); + + assertFails(TABLE_FUNCTION_COLUMN_NOT_FOUND, + "line 1:88: Column b is not present in the input relation", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT 1 a) ORDER BY b))"); + + assertFails(TABLE_FUNCTION_INVALID_COLUMN_REFERENCE, + "line 1:88: Expected column reference. Actual: 1", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT 1 a) ORDER BY 1))"); + + assertFails(TYPE_MISMATCH, + "line 1:100: HyperLogLog is not orderable, and therefore cannot be used in ORDER BY", + "SELECT * FROM TABLE(system.table_argument_function(input => TABLE(SELECT approx_set(1) a) ORDER BY a))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:85: Invalid argument INPUT. Empty behavior specified for table argument with row semantics", + "SELECT * FROM TABLE(system.table_argument_row_semantics_function(input => TABLE(t1) PRUNE WHEN EMPTY))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:85: Invalid argument INPUT. Empty behavior specified for table argument with row semantics", + "SELECT * FROM TABLE(system.table_argument_row_semantics_function(input => TABLE(t1) KEEP WHEN EMPTY))"); + } + + @Test + public void testDescriptorArgument() + { + analyze("SELECT * FROM TABLE(system.descriptor_argument_function(schema => DESCRIPTOR(x integer, y boolean)))"); + + assertFailsExact(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:57: Invalid descriptor argument SCHEMA. Descriptors should be formatted as 'DESCRIPTOR(name [type], ...)'", + "SELECT * FROM TABLE(system.descriptor_argument_function(schema => DESCRIPTOR(1 + 2)))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:57: Invalid argument SCHEMA. Expected descriptor, got expression", + "SELECT * FROM TABLE(system.descriptor_argument_function(schema => 1))"); + + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:57: Invalid argument SCHEMA. Expected descriptor, got table", + "SELECT * FROM TABLE(system.descriptor_argument_function(schema => TABLE(t1)))"); + + assertFails(TYPE_MISMATCH, + "line 1:78: Unknown type: verybigint", + "SELECT * FROM TABLE(system.descriptor_argument_function(schema => DESCRIPTOR(x verybigint)))"); + } + + @Test + public void testCopartitioning() + { + // TABLE(t1) is matched by fully qualified name: tpch.s1.t1. It matches the second copartition item s1.t1. + // Aliased relation TABLE(SELECT 1, 2) t1(x, y) is matched by unqualified name. It matches the first copartition item t1. + analyze("SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY (a, b)," + + "input2 => TABLE(SELECT 1, 2) t1(x, y) PARTITION BY (x, y)" + + "COPARTITION (t1, s1.t1)))"); + + // Copartition items t1, t2 are first matched to arguments by unqualified names, and when no match is found, by fully qualified names. + // TABLE(tpch.s1.t1) is matched by fully qualified name. It matches the first copartition item t1. + // TABLE(s1.t2) is matched by unqualified name: tpch.s1.t2. It matches the second copartition item t2. + analyze("SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(tpch.s1.t1) PARTITION BY (a, b)," + + "input2 => TABLE(s1.t2) PARTITION BY (a, b)" + + "COPARTITION (t1, t2)))"); + + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:153: No table argument found for name: s1.foo", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY (a, b)," + + "input2 => TABLE(t2) PARTITION BY (a, b)" + + "COPARTITION (t1, s1.foo)))"); + + // Both table arguments are matched by fully qualified name: tpch.s1.t1 + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, "line 1:149: Ambiguous reference: multiple table arguments found for name: t1", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY (a, b)," + + "input2 => TABLE(t1) PARTITION BY (a, b)" + + "COPARTITION (t1, t2)))"); + + // Both table arguments are matched by unqualified name: t1 + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:185: Ambiguous reference: multiple table arguments found for name: t1", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(SELECT 1, 2) t1(a, b) PARTITION BY (a, b)," + + "input2 => TABLE(SELECT 3, 4) t1(c, d) PARTITION BY (c, d)" + + "COPARTITION (t1, t2)))"); + + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:153: Multiple references to table argument: t1 in COPARTITION clause", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY (a, b)," + + "input2 => TABLE(t2) PARTITION BY (a, b)" + + "COPARTITION (t1, t1)))"); + } + + @Test + public void testCopartitionColumns() + { + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:67: Table tpch.s1.t1 referenced in COPARTITION clause is not partitioned", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1)," + + "input2 => TABLE(t2) PARTITION BY (a, b)" + + "COPARTITION (t1, t2)))"); + + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:67: No partitioning columns specified for table tpch.s1.t1 referenced in COPARTITION clause", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY ()," + + "input2 => TABLE(t2) PARTITION BY ()" + + "COPARTITION (t1, t2)))"); + + assertFails(TABLE_FUNCTION_INVALID_COPARTITIONING, + "line 1:146: Numbers of partitioning columns in copartitioned tables do not match", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(t1) PARTITION BY (a, b)," + + "input2 => TABLE(t2) PARTITION BY (a)" + + "COPARTITION (t1, t2)))"); + + assertFails(TYPE_MISMATCH, + "line 1:169: Partitioning columns in copartitioned tables have incompatible types", + "SELECT * FROM TABLE(system.two_table_arguments_function(" + + "input1 => TABLE(SELECT 1) t1(a) PARTITION BY (a)," + + "input2 => TABLE(SELECT 'x') t2(b) PARTITION BY (b)" + + "COPARTITION (t1, t2)))"); + } + + @Test + public void testNullArguments() + { + // cannot pass null for table argument + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:52: Invalid argument INPUT. Expected table, got expression", + "SELECT * FROM TABLE(system.table_argument_function(input => null))"); + + // the wrong way to pass null for descriptor + assertFails(TABLE_FUNCTION_INVALID_FUNCTION_ARGUMENT, + "line 1:57: Invalid argument SCHEMA. Expected descriptor, got expression", + "SELECT * FROM TABLE(system.descriptor_argument_function(schema => null))"); + + // the right way to pass null for descriptor + analyze("SELECT * FROM TABLE(system.descriptor_argument_function(schema => CAST(null AS DESCRIPTOR)))"); + + // the default value for the argument schema is null + analyze("SELECT * FROM TABLE(system.descriptor_argument_function())"); + + analyze("SELECT * FROM TABLE(system.two_arguments_function(null, null))"); + + // the default value for the second argument is null + analyze("SELECT * FROM TABLE(system.two_arguments_function('a'))"); + } + + @Test + public void testTableFunctionInvocationContext() + { + // cannot specify relation alias for table function with ONLY PASS THROUGH return type + assertFails(TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, + "line 1:21: Alias specified for table function with ONLY PASS THROUGH return type", + "SELECT * FROM TABLE(system.only_pass_through_function(TABLE(t1))) f(x)"); + + // per SQL standard, relation alias is required for table function with GENERIC TABLE return type. We don't require it. + analyze("SELECT * FROM TABLE(system.two_arguments_function('a', 1)) f(x)"); + analyze("SELECT * FROM TABLE(system.two_arguments_function('a', 1))"); + + // per SQL standard, relation alias is required for table function with statically declared return type, only if the function is polymorphic. + // We don't require aliasing polymorphic functions. + analyze("SELECT * FROM TABLE(system.monomorphic_static_return_type_function())"); + analyze("SELECT * FROM TABLE(system.monomorphic_static_return_type_function()) f(x, y)"); + analyze("SELECT * FROM TABLE(system.polymorphic_static_return_type_function(input => TABLE(t1)))"); + analyze("SELECT * FROM TABLE(system.polymorphic_static_return_type_function(input => TABLE(t1))) f(x, y)"); + + // sampled + assertFails(TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, + "line 1:21: Cannot apply sample to polymorphic table function invocation", + "SELECT * FROM TABLE(system.only_pass_through_function(TABLE(t1))) TABLESAMPLE BERNOULLI (10)"); + + // aliased + sampled + assertFails(TABLE_FUNCTION_INVALID_TABLE_FUNCTION_INVOCATION, + "line 1:15: Cannot apply sample to polymorphic table function invocation", + "SELECT * FROM TABLE(system.two_arguments_function('a', 1)) f(x) TABLESAMPLE BERNOULLI (10)"); + } + + @Test + public void testTableFunctionAliasing() + { + // case-insensitive name matching + assertFails(TABLE_FUNCTION_DUPLICATE_RANGE_VARIABLE, + "line 1:64: Relation alias: T1 is a duplicate of input table name: tpch.s1.t1", + "SELECT * FROM TABLE(system.table_argument_function(TABLE(t1))) T1(x)"); + + assertFails(TABLE_FUNCTION_DUPLICATE_RANGE_VARIABLE, + "line 1:76: Relation alias: t1 is a duplicate of input table name: t1", + "SELECT * FROM TABLE(system.table_argument_function(TABLE(SELECT 1) T1(a))) t1(x)"); + + analyze("SELECT * FROM TABLE(system.table_argument_function(TABLE(t1) t2)) T1(x)"); + + // the original returned relation type is ("column" : BOOLEAN) + analyze("SELECT column FROM TABLE(system.two_arguments_function('a', 1)) table_alias"); + + analyze("SELECT column_alias FROM TABLE(system.two_arguments_function('a', 1)) table_alias(column_alias)"); + + analyze("SELECT table_alias.column_alias FROM TABLE(system.two_arguments_function('a', 1)) table_alias(column_alias)"); + + assertFails(MISSING_ATTRIBUTE, + "line 1:8: Column 'column' cannot be resolved", + "SELECT column FROM TABLE(system.two_arguments_function('a', 1)) table_alias(column_alias)"); + + assertFails(MISMATCHED_COLUMN_ALIASES, + "line 1:20: Column alias list has 3 entries but table function has 1 proper columns", + "SELECT column FROM TABLE(system.two_arguments_function('a', 1)) table_alias(col1, col2, col3)"); + + // the original returned relation type is ("a" : BOOLEAN, "b" : INTEGER) + analyze("SELECT column_alias_1, column_alias_2 FROM TABLE(system.monomorphic_static_return_type_function()) table_alias(column_alias_1, column_alias_2)"); + + assertFails(DUPLICATE_COLUMN_NAME, + "line 1:21: Duplicate name of table function proper column: col", + "SELECT * FROM TABLE(system.monomorphic_static_return_type_function()) table_alias(col, col)"); + + // case-insensitive name matching + assertFails(DUPLICATE_COLUMN_NAME, + "line 1:21: Duplicate name of table function proper column: col", + "SELECT * FROM TABLE(system.monomorphic_static_return_type_function()) table_alias(col, COL)"); + + // pass-through columns of an input table must not be aliased, and must be referenced by the original range variables of their corresponding table arguments + // the function pass_through_function has one proper column ("x" : BOOLEAN), and one table argument with pass-through property + // tha alias applies only to the proper column + analyze("SELECT table_alias.x, t1.a, t1.b, t1.c, t1.d FROM TABLE(system.pass_through_function(TABLE(t1))) table_alias"); + + analyze("SELECT table_alias.x, arg_alias.a, arg_alias.b, arg_alias.c, arg_alias.d FROM TABLE(system.pass_through_function(TABLE(t1) arg_alias)) table_alias"); + + assertFails(MISSING_ATTRIBUTE, + "line 1:23: 't1.a' cannot be resolved", + "SELECT table_alias.x, t1.a FROM TABLE(system.pass_through_function(TABLE(t1) arg_alias)) table_alias"); + + assertFails(MISSING_ATTRIBUTE, + "line 1:23: 'table_alias.a' cannot be resolved", + "SELECT table_alias.x, table_alias.a FROM TABLE(system.pass_through_function(TABLE(t1))) table_alias"); + } + + @Test + public void testTableFunctionRequiredColumns() + { + // the function required_column_function specifies columns 0 and 1 from table argument "INPUT" as required. + analyze("SELECT * FROM TABLE(system.required_columns_function(input => TABLE(t1)))"); + + analyze("SELECT * FROM TABLE(system.required_columns_function(input => TABLE(SELECT 1, 2, 3)))"); + + assertFails(TABLE_FUNCTION_IMPLEMENTATION_ERROR, + "Invalid index: 1 of required column from table argument INPUT", + "SELECT * FROM TABLE(system.required_columns_function(input => TABLE(SELECT 1)))"); + + // table s1.t5 has two columns. The second column is hidden. Table function can require a hidden column. + analyze("SELECT * FROM TABLE(system.required_columns_function(input => TABLE(s1.t5)))"); + } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java b/presto-main-base/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java index df9883832c579..0a100ac9f8757 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/relational/TestRowExpressionOptimizer.java @@ -15,9 +15,11 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.IntArrayBlock; +import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.type.ArrayType; import com.facebook.presto.common.type.RowType; import com.facebook.presto.common.type.StandardTypes; +import com.facebook.presto.common.type.Type; import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutor; import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors; @@ -33,6 +35,7 @@ import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.SpecialFormExpression; import com.facebook.presto.sql.analyzer.FunctionsConfig; @@ -42,10 +45,15 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.List; +import java.util.Optional; + import static com.facebook.airlift.testing.Assertions.assertInstanceOf; import static com.facebook.presto.block.BlockAssertions.toValues; import static com.facebook.presto.common.function.OperatorType.ADD; import static com.facebook.presto.common.function.OperatorType.EQUAL; +import static com.facebook.presto.common.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.common.function.OperatorType.LESS_THAN; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.DoubleType.DOUBLE; @@ -86,7 +94,15 @@ public class TestRowExpressionOptimizer RoutineCharacteristics.builder().setLanguage(CPP).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), "", notVersioned()); - public static final SqlInvokedFunction CPP_CUSTOM_FUNCTION = new SqlInvokedFunction( + private static final SqlInvokedFunction CPP_BAR = new SqlInvokedFunction( + new QualifiedObjectName("native", "default", "cbrt"), + ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), + parseTypeSignature(StandardTypes.DOUBLE), + "cbrt(x)", + RoutineCharacteristics.builder().setLanguage(CPP).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), + "", + notVersioned()); + private static final SqlInvokedFunction CPP_CUSTOM_FUNCTION = new SqlInvokedFunction( new QualifiedObjectName("native", "default", "cpp_custom_func"), ImmutableList.of(new Parameter("x", parseTypeSignature(StandardTypes.BIGINT))), parseTypeSignature(StandardTypes.BIGINT), @@ -94,6 +110,29 @@ public class TestRowExpressionOptimizer RoutineCharacteristics.builder().setLanguage(CPP).setDeterminism(DETERMINISTIC).setNullCallClause(RETURNS_NULL_ON_NULL_INPUT).build(), "", notVersioned()); + private static final String nativePrefix = "native.default"; + + private static final RowExpression CUBE_ROOT_EXP = call( + "cbrt", + new SqlFunctionHandle( + new SqlFunctionId( + QualifiedObjectName.valueOf(format("%s.cbrt", nativePrefix)), + ImmutableList.of(BIGINT.getTypeSignature())), + "1"), + DOUBLE, + ImmutableList.of( + constant(27L, BIGINT))); + + private static final RowExpression SQUARE_ROOT_EXP = call( + "sqrt", + new SqlFunctionHandle( + new SqlFunctionId( + QualifiedObjectName.valueOf(format("%s.sqrt", nativePrefix)), + ImmutableList.of(BIGINT.getTypeSignature())), + "1"), + DOUBLE, + ImmutableList.of( + constant(64L, BIGINT))); private FunctionAndTypeManager functionAndTypeManager; private RowExpressionOptimizer optimizer; @@ -176,57 +215,18 @@ public void testCastWithJsonParseOptimization() @Test public void testDefaultExpressionOptimizerUsesJavaNamespaceForBuiltInFunctions() { - String nativePrefix = "native.default"; - MetadataManager metadata = MetadataManager.createTestMetadataManager(new FunctionsConfig().setDefaultNamespacePrefix(nativePrefix)); + RowExpressionOptimizer nativeOptimizer = getNativeOptimizer(); + assertEquals(nativeOptimizer.optimize(SQUARE_ROOT_EXP, OPTIMIZED, SESSION), constant(8.0, DOUBLE)); + assertThrows(IllegalArgumentException.class, () -> optimizer.optimize(SQUARE_ROOT_EXP, OPTIMIZED, SESSION)); - metadata.getFunctionAndTypeManager().addFunctionNamespace( - "native", - new InMemoryFunctionNamespaceManager( - "native", - new SqlFunctionExecutors( - ImmutableMap.of( - CPP, FunctionImplementationType.CPP, - JAVA, FunctionImplementationType.JAVA), - new NoopSqlFunctionExecutor()), - new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("cpp"))); - metadata.getFunctionAndTypeManager().createFunction(CPP_FOO, true); - RowExpressionOptimizer nativeOptimizer = new RowExpressionOptimizer(metadata); - RowExpression simpleAddition = call( - "sqrt", - new SqlFunctionHandle( - new SqlFunctionId( - QualifiedObjectName.valueOf(format("%s.sqrt", nativePrefix)), - ImmutableList.of(BIGINT.getTypeSignature())), - "1"), - DOUBLE, - ImmutableList.of( - constant(4L, BIGINT))); - assertEquals(nativeOptimizer.optimize(simpleAddition, OPTIMIZED, SESSION), constant(2.0, DOUBLE)); - assertThrows(IllegalArgumentException.class, () -> optimizer.optimize(simpleAddition, OPTIMIZED, SESSION)); + assertEquals(nativeOptimizer.optimize(CUBE_ROOT_EXP, OPTIMIZED, SESSION), constant(3.0, DOUBLE)); + assertThrows(IllegalArgumentException.class, () -> optimizer.optimize(CUBE_ROOT_EXP, OPTIMIZED, SESSION)); } @Test public void testFunctionNotInPrestoDefaultNamespaceIsNotEvaluated() { - // Create a custom function that exists only in native namespace, not in presto.default - String nativePrefix = "native.default"; - MetadataManager metadata = MetadataManager.createTestMetadataManager(new FunctionsConfig().setDefaultNamespacePrefix(nativePrefix)); - - metadata.getFunctionAndTypeManager().addFunctionNamespace( - "native", - new InMemoryFunctionNamespaceManager( - "native", - new SqlFunctionExecutors( - ImmutableMap.of( - CPP, FunctionImplementationType.CPP, - JAVA, FunctionImplementationType.JAVA), - new NoopSqlFunctionExecutor()), - new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("cpp"))); - - // Create a custom function that only exists in native namespace - metadata.getFunctionAndTypeManager().createFunction(CPP_CUSTOM_FUNCTION, true); - - RowExpressionOptimizer nativeOptimizer = new RowExpressionOptimizer(metadata); + RowExpressionOptimizer nativeOptimizer = getNativeOptimizer(); // Create a call expression to the custom native function RowExpression customFunctionCall = call( @@ -248,6 +248,123 @@ public void testFunctionNotInPrestoDefaultNamespaceIsNotEvaluated() // Verify that the function handle remains the same (not replaced) CallExpression optimizedCall = (CallExpression) optimized; assertEquals(optimizedCall.getFunctionHandle().getCatalogSchemaName().toString(), nativePrefix); + + // Create a call expression to the custom native function with a sqrt call expression arg + RowExpression customFunctionWithCallExpressionCall = call( + "cpp_custom_func", + new SqlFunctionHandle( + new SqlFunctionId( + QualifiedObjectName.valueOf(format("%s.cpp_custom_func", nativePrefix)), + ImmutableList.of(BIGINT.getTypeSignature())), + "1"), + BIGINT, + ImmutableList.of(SQUARE_ROOT_EXP)); + + // The inner CallExpression should be optimized, but the outer shouldn't since the function doesn't exist in presto.default namespace + optimized = nativeOptimizer.optimize(customFunctionWithCallExpressionCall, OPTIMIZED, SESSION); + assertEquals( + optimized, + call( + "cpp_custom_func", + new SqlFunctionHandle( + new SqlFunctionId( + QualifiedObjectName.valueOf(format("%s.cpp_custom_func", nativePrefix)), + ImmutableList.of(BIGINT.getTypeSignature())), + "1"), + BIGINT, + ImmutableList.of(constant(8.0, DOUBLE)))); + assertInstanceOf(optimized, CallExpression.class); + // Verify that the function handle remains the same (not replaced) + optimizedCall = (CallExpression) optimized; + assertEquals(optimizedCall.getFunctionHandle().getCatalogSchemaName().toString(), nativePrefix); + assertEquals(optimizedCall.getChildren().get(0), constant(8.0, DOUBLE)); + } + + @Test + public void testSpecialFormExpressionsWhenDefaultNamespaceIsSwitched() + { + RowExpressionOptimizer nativeOptimizer = getNativeOptimizer(); + + RowExpression leftCondition = callOperator( + GREATER_THAN, + SQUARE_ROOT_EXP, + constant(5.0, DOUBLE)); + + RowExpression rightCondition = callOperator( + LESS_THAN, + CUBE_ROOT_EXP, + constant(10.0, DOUBLE)); + + RowExpression andExpr = new SpecialFormExpression( + SpecialFormExpression.Form.AND, + BOOLEAN, + ImmutableList.of(leftCondition, rightCondition)); + + RowExpression optimized = nativeOptimizer.optimize(andExpr, OPTIMIZED, SESSION); + assertEquals(optimized, constant(true, BOOLEAN)); + + // Lambda expressions inside Special form expressions + List lambdaTypes = ImmutableList.of(DOUBLE, DOUBLE); + LambdaDefinitionExpression lambda = + new LambdaDefinitionExpression( + Optional.empty(), + lambdaTypes, + ImmutableList.of("s", "x"), + callOperator(ADD, SQUARE_ROOT_EXP, CUBE_ROOT_EXP)); + + andExpr = new SpecialFormExpression( + SpecialFormExpression.Form.AND, + BOOLEAN, + ImmutableList.of(lambda, rightCondition)); + + // rightCondition is always true, hence the expression should be reduced to "(s, x) -> 11.0". + optimized = nativeOptimizer.optimize(andExpr, OPTIMIZED, SESSION); + + assertInstanceOf(optimized, LambdaDefinitionExpression.class); + LambdaDefinitionExpression lambdaExpression = (LambdaDefinitionExpression) optimized; + assertEquals(lambdaExpression.getArgumentTypes(), lambdaTypes); + assertEquals(lambdaExpression.getBody(), constant(11.0, DOUBLE)); + } + + @Test + public void testLambdaExpressionsWhenDefaultNamespaceIsSwitched() + { + RowExpressionOptimizer nativeOptimizer = getNativeOptimizer(); + + List lambdaTypes = ImmutableList.of(DOUBLE, DOUBLE); + + LambdaDefinitionExpression lambda = + new LambdaDefinitionExpression( + Optional.empty(), + lambdaTypes, + ImmutableList.of("s", "x"), + callOperator(ADD, SQUARE_ROOT_EXP, CUBE_ROOT_EXP)); + + LambdaDefinitionExpression nestedLambda = + new LambdaDefinitionExpression( + Optional.empty(), + ImmutableList.of(DOUBLE), + ImmutableList.of("y"), + lambda); + + RowExpression optimized = nativeOptimizer.optimize(lambda, OPTIMIZED, SESSION); + + assertInstanceOf(optimized, LambdaDefinitionExpression.class); + LambdaDefinitionExpression lambdaExpression = (LambdaDefinitionExpression) optimized; + assertEquals(lambdaExpression.getArgumentTypes(), lambdaTypes); + assertEquals(lambdaExpression.getBody(), constant(11.0, DOUBLE)); + + // Nested lambda + RowExpression optimizedOuterLambda = nativeOptimizer.optimize(nestedLambda, OPTIMIZED, SESSION); + + assertInstanceOf(optimizedOuterLambda, LambdaDefinitionExpression.class); + LambdaDefinitionExpression outerLambdaExpression = (LambdaDefinitionExpression) optimizedOuterLambda; + assertEquals(outerLambdaExpression.getArgumentTypes(), ImmutableList.of(DOUBLE)); + assertInstanceOf(outerLambdaExpression.getBody(), LambdaDefinitionExpression.class); + + LambdaDefinitionExpression innerLambdaExpression = (LambdaDefinitionExpression) outerLambdaExpression.getBody(); + assertEquals(innerLambdaExpression.getArgumentTypes(), lambdaTypes); + assertEquals(innerLambdaExpression.getBody(), constant(11.0, DOUBLE)); } private static RowExpression ifExpression(RowExpression condition, long trueValue, long falseValue) @@ -255,6 +372,34 @@ private static RowExpression ifExpression(RowExpression condition, long trueValu return new SpecialFormExpression(IF, BIGINT, ImmutableList.of(condition, constant(trueValue, BIGINT), constant(falseValue, BIGINT))); } + private static RowExpressionOptimizer getNativeOptimizer() + { + String nativePrefix = "native.default"; + MetadataManager metadata = MetadataManager.createTestMetadataManager(new FunctionsConfig().setDefaultNamespacePrefix(nativePrefix)); + + metadata.getFunctionAndTypeManager().addFunctionNamespace( + "native", + new InMemoryFunctionNamespaceManager( + "native", + new SqlFunctionExecutors( + ImmutableMap.of( + CPP, FunctionImplementationType.CPP, + JAVA, FunctionImplementationType.JAVA), + new NoopSqlFunctionExecutor()), + new SqlInvokedFunctionNamespaceManagerConfig().setSupportedFunctionLanguages("cpp"))); + metadata.getFunctionAndTypeManager().createFunction(CPP_FOO, true); + metadata.getFunctionAndTypeManager().createFunction(CPP_BAR, true); + // Create a custom function that only exists in native namespace + metadata.getFunctionAndTypeManager().createFunction(CPP_CUSTOM_FUNCTION, true); + return new RowExpressionOptimizer(metadata); + } + + private RowExpression callOperator(OperatorType operator, RowExpression left, RowExpression right) + { + FunctionHandle functionHandle = functionAndTypeManager.resolveOperator(operator, fromTypes(left.getType(), right.getType())); + return Expressions.call(operator.getOperator(), functionHandle, left.getType(), left, right); + } + private RowExpression optimize(RowExpression expression) { return optimizer.optimize(expression, OPTIMIZED, SESSION); diff --git a/presto-main/etc/config.properties b/presto-main/etc/config.properties index 164015db096d4..0ee18a1e4df0f 100644 --- a/presto-main/etc/config.properties +++ b/presto-main/etc/config.properties @@ -52,7 +52,7 @@ plugin.bundles=\ ../presto-hive-function-namespace/pom.xml,\ ../presto-delta/pom.xml,\ ../presto-hudi/pom.xml, \ - ../presto-sql-invoked-functions-plugin/pom.xml + ../presto-sql-helpers/presto-sql-invoked-functions-plugin/pom.xml presto.version=testversion node-scheduler.include-coordinator=true diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 5f737a7ab3bd6..944ec4e6faf8c 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-main @@ -308,23 +308,6 @@ reactor-netty-http - - io.micrometer - micrometer-core - 1.11.0 - - - org.hdrhistogram - HdrHistogram - - - - - io.micrometer - micrometer-registry-jmx - 1.11.0 - - io.projectreactor reactor-core diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index 1813c5eb06995..075934ece22cc 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -84,6 +84,8 @@ import com.facebook.presto.server.protocol.QueryBlockingRateLimiter; import com.facebook.presto.server.protocol.QueuedStatementResource; import com.facebook.presto.server.protocol.RetryCircuitBreaker; +import com.facebook.presto.server.remotetask.HttpClientConnectionPoolStats; +import com.facebook.presto.server.remotetask.HttpClientStats; import com.facebook.presto.server.remotetask.HttpRemoteTaskFactory; import com.facebook.presto.server.remotetask.ReactorNettyHttpClient; import com.facebook.presto.server.remotetask.ReactorNettyHttpClientConfig; @@ -277,6 +279,10 @@ protected void setup(Binder binder) ReactorNettyHttpClientConfig reactorNettyHttpClientConfig = buildConfigObject(ReactorNettyHttpClientConfig.class); if (reactorNettyHttpClientConfig.isReactorNettyHttpClientEnabled()) { binder.bind(ReactorNettyHttpClient.class).in(Scopes.SINGLETON); + binder.bind(HttpClientStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(HttpClientStats.class).withGeneratedName(); + binder.bind(HttpClientConnectionPoolStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(HttpClientConnectionPoolStats.class).withGeneratedName(); binder.bind(HttpClient.class).annotatedWith(ForScheduler.class).to(ReactorNettyHttpClient.class); } else { diff --git a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java index 4b7ae603a587c..2826ff4a4bf41 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java +++ b/presto-main/src/main/java/com/facebook/presto/server/HttpRequestSessionContext.java @@ -154,14 +154,7 @@ public HttpRequestSessionContext(HttpServletRequest servletRequest, SqlParserOpt String user = trimEmptyToNull(servletRequest.getHeader(PRESTO_USER)); assertRequest(user != null, "User must be set"); - identity = new Identity( - user, - Optional.ofNullable(servletRequest.getUserPrincipal()), - parseRoleHeaders(servletRequest), - parseExtraCredentials(servletRequest), - ImmutableMap.of(), - Optional.empty(), - Optional.empty()); + authorizedIdentity = authorizedIdentity(servletRequest); X509Certificate[] certs = (X509Certificate[]) servletRequest.getAttribute(X509_ATTRIBUTE); @@ -172,6 +165,16 @@ public HttpRequestSessionContext(HttpServletRequest servletRequest, SqlParserOpt certificates = ImmutableList.of(); } + identity = new Identity( + user, + Optional.ofNullable(servletRequest.getUserPrincipal()), + parseRoleHeaders(servletRequest), + parseExtraCredentials(servletRequest), + ImmutableMap.of(), + Optional.empty(), + Optional.empty(), + certificates); + source = servletRequest.getHeader(PRESTO_SOURCE); userAgent = servletRequest.getHeader(USER_AGENT); remoteUserAddress = !isNullOrEmpty(servletRequest.getHeader(X_FORWARDED_FOR)) ? servletRequest.getHeader(X_FORWARDED_FOR) : servletRequest.getRemoteAddr(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientConnectionPoolStats.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientConnectionPoolStats.java new file mode 100644 index 0000000000000..91513980c3ba4 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientConnectionPoolStats.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.server.remotetask; + +import com.facebook.airlift.stats.DistributionStat; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; +import reactor.netty.resources.ConnectionPoolMetrics; +import reactor.netty.resources.ConnectionProvider; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Singleton +public class HttpClientConnectionPoolStats + implements ConnectionProvider.MeterRegistrar +{ + private final ConcurrentHashMap poolMetrics = new ConcurrentHashMap<>(); + + private final DistributionStat activeConnections = new DistributionStat(); + private final DistributionStat totalConnections = new DistributionStat(); + private final DistributionStat idleConnections = new DistributionStat(); + private final DistributionStat pendingAcquires = new DistributionStat(); + private final DistributionStat maxConnections = new DistributionStat(); + private final DistributionStat maxPendingAcquires = new DistributionStat(); + + @Inject + public HttpClientConnectionPoolStats() + { + scheduleStatsExport(); + } + + @Override + public void registerMetrics(String poolName, String id, SocketAddress remoteAddress, ConnectionPoolMetrics metrics) + { + poolMetrics.put(createPoolKey(poolName, remoteAddress), metrics); + } + + private static String createPoolKey(String poolName, SocketAddress remoteAddress) + { + return poolName + ":" + formatSocketAddress(remoteAddress); + } + + private static String formatSocketAddress(SocketAddress socketAddress) + { + if (socketAddress != null) { + if (socketAddress instanceof InetSocketAddress) { + InetSocketAddress address = (InetSocketAddress) socketAddress; + return address.getHostString().replace(".", "_"); + } + else { + return socketAddress.toString().replace(".", "_"); + } + } + return "UNKNOWN"; + } + + private void scheduleStatsExport() + { + Executors.newSingleThreadScheduledExecutor() + .scheduleAtFixedRate( + () -> { + for (ConnectionPoolMetrics metrics : poolMetrics.values()) { + activeConnections.add(metrics.acquiredSize()); + totalConnections.add(metrics.allocatedSize()); + idleConnections.add(metrics.idleSize()); + pendingAcquires.add(metrics.pendingAcquireSize()); + maxConnections.add(metrics.maxAllocatedSize()); + maxPendingAcquires.add(metrics.maxPendingAcquireSize()); + } + }, + 0, + 1, + TimeUnit.SECONDS); + } + + @Managed + @Nested + public DistributionStat getActiveConnections() + { + return activeConnections; + } + + @Managed + @Nested + public DistributionStat getTotalConnections() + { + return totalConnections; + } + + @Managed + @Nested + public DistributionStat getIdleConnections() + { + return idleConnections; + } + + @Managed + @Nested + public DistributionStat getPendingAcquires() + { + return pendingAcquires; + } + + @Managed + @Nested + public DistributionStat getMaxConnections() + { + return maxConnections; + } + + @Managed + @Nested + public DistributionStat getMaxPendingAcquires() + { + return maxPendingAcquires; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientStats.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientStats.java new file mode 100644 index 0000000000000..6d5af54e80636 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpClientStats.java @@ -0,0 +1,382 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.server.remotetask; + +import com.facebook.airlift.stats.CounterStat; +import com.facebook.airlift.stats.DistributionStat; +import com.facebook.airlift.stats.TimeStat; +import com.google.inject.Singleton; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; +import reactor.netty.http.client.ContextAwareHttpClientMetricsRecorder; +import reactor.util.context.ContextView; + +import java.net.SocketAddress; +import java.time.Duration; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@Singleton +public class HttpClientStats + extends ContextAwareHttpClientMetricsRecorder +{ + // HTTP level metrics + private final TimeStat responseTime = new TimeStat(); + private final TimeStat dataReceivedTime = new TimeStat(); + private final TimeStat dataSentTime = new TimeStat(); + private final CounterStat errorsCount = new CounterStat(); + private final CounterStat bytesReceived = new CounterStat(); + private final CounterStat bytesSent = new CounterStat(); + private final DistributionStat payloadSize = new DistributionStat(); + + // Channel level metrics + private final TimeStat connectTime = new TimeStat(); + private final TimeStat tlsHandshakeTime = new TimeStat(); + private final TimeStat resolveAddressTime = new TimeStat(); + private final CounterStat channelErrorsCount = new CounterStat(); + private final CounterStat channelBytesReceived = new CounterStat(); + private final CounterStat channelBytesSent = new CounterStat(); + private final CounterStat connectionsOpened = new CounterStat(); + private final CounterStat connectionsClosed = new CounterStat(); + + // HTTP level metrics recording + + /** + * Records the time that is spent in consuming incoming data + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + * @param method The HTTP method + * @param status The HTTP status + * @param time The time in nanoseconds that is spent in consuming incoming data + */ + @Override + public void recordDataReceivedTime( + ContextView contextView, + SocketAddress remoteAddress, + String uri, + String method, + String status, + Duration time) + { + dataReceivedTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Records the time that is spent in sending outgoing data + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + * @param method The HTTP method + * @param time The time in nanoseconds that is spent in sending outgoing data + */ + @Override + public void recordDataSentTime( + ContextView contextView, + SocketAddress remoteAddress, + String uri, + String method, + Duration time) + { + dataSentTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Records the total time for the request/response + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + * @param method The HTTP method + * @param status The HTTP status + * @param time The total time in nanoseconds for the request/response + */ + @Override + public void recordResponseTime( + ContextView contextView, + SocketAddress remoteAddress, + String uri, + String method, + String status, + Duration time) + { + responseTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Increments the number of the errors that are occurred + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + */ + @Override + public void incrementErrorsCount(ContextView contextView, SocketAddress remoteAddress, String uri) + { + errorsCount.update(1); + } + + /** + * Records the amount of the data that is received, in bytes + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + * @param bytes The amount of the data that is received, in bytes + */ + @Override + public void recordDataReceived(ContextView contextView, SocketAddress remoteAddress, String uri, long bytes) + { + bytesReceived.update(bytes); + } + + /** + * Records the amount of the data that is sent, in bytes + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux + * @param remoteAddress The remote peer + * @param uri The requested URI + * @param bytes The amount of the data that is sent, in bytes + */ + @Override + public void recordDataSent(ContextView contextView, SocketAddress remoteAddress, String uri, long bytes) + { + bytesSent.update(bytes); + payloadSize.add(bytes); + } + + // Channel level metrics recording + + /** + * Increments the number of the errors that are occurred + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux pipeline + * @param remoteAddress The remote peer + */ + @Override + public void incrementErrorsCount(ContextView contextView, SocketAddress remoteAddress) + { + channelErrorsCount.update(1); + } + + /** + * Records the time that is spent for connecting to the remote address Relevant only when on the + * client + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux pipeline + * @param remoteAddress The remote peer + * @param time The time in nanoseconds that is spent for connecting to the remote address + * @param status The status of the operation + */ + @Override + public void recordConnectTime( + ContextView contextView, + SocketAddress remoteAddress, + Duration time, + String status) + { + connectTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Records the amount of the data that is received, in bytes + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux pipeline + * @param remoteAddress The remote peer + * @param bytes The amount of the data that is received, in bytes + */ + @Override + public void recordDataReceived(ContextView contextView, SocketAddress remoteAddress, long bytes) + { + channelBytesReceived.update(bytes); + } + + /** + * Records the amount of the data that is sent, in bytes + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux pipeline + * @param remoteAddress The remote peer + * @param bytes The amount of the data that is sent, in bytes + */ + @Override + public void recordDataSent(ContextView contextView, SocketAddress remoteAddress, long bytes) + { + channelBytesSent.update(bytes); + } + + /** + * Records the time that is spent for TLS handshake + * + * @param contextView The current {@link ContextView} associated with the Mono/Flux pipeline + * @param remoteAddress The remote peer + * @param time The time in nanoseconds that is spent for TLS handshake + * @param status The status of the operation + */ + @Override + public void recordTlsHandshakeTime( + ContextView contextView, + SocketAddress remoteAddress, + Duration time, + String status) + { + tlsHandshakeTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Records the time that is spent for resolving the remote address Relevant only when on the + * client + * + * @param remoteAddress The remote peer + * @param time the time in nanoseconds that is spent for resolving to the remote address + * @param status the status of the operation + */ + @Override + public void recordResolveAddressTime(SocketAddress remoteAddress, Duration time, String status) + { + resolveAddressTime.add(time.toMillis(), MILLISECONDS); + } + + /** + * Records a just accepted server connection + * + * @param localAddress the server local address + * @since 1.0.15 + */ + @Override + public void recordServerConnectionOpened(SocketAddress localAddress) + { + connectionsOpened.update(1); + } + + /** + * Records a just disconnected server connection + * + * @param localAddress the server local address + * @since 1.0.15 + */ + @Override + public void recordServerConnectionClosed(SocketAddress localAddress) + { + connectionsClosed.update(1); + } + + // JMX exposed metrics + + @Managed + @Nested + public TimeStat getResponseTime() + { + return responseTime; + } + + @Managed + @Nested + public TimeStat getDataReceivedTime() + { + return dataReceivedTime; + } + + @Managed + @Nested + public TimeStat getDataSentTime() + { + return dataSentTime; + } + + @Managed + @Nested + public CounterStat getErrorsCount() + { + return errorsCount; + } + + @Managed + @Nested + public CounterStat getBytesReceived() + { + return bytesReceived; + } + + @Managed + @Nested + public CounterStat getBytesSent() + { + return bytesSent; + } + + @Managed + @Nested + public DistributionStat getPayloadSize() + { + return payloadSize; + } + + @Managed + @Nested + public TimeStat getConnectTime() + { + return connectTime; + } + + @Managed + @Nested + public TimeStat getTlsHandshakeTime() + { + return tlsHandshakeTime; + } + + @Managed + @Nested + public TimeStat getResolveAddressTime() + { + return resolveAddressTime; + } + + @Managed + @Nested + public CounterStat getChannelErrorsCount() + { + return channelErrorsCount; + } + + @Managed + @Nested + public CounterStat getChannelBytesReceived() + { + return channelBytesReceived; + } + + @Managed + @Nested + public CounterStat getChannelBytesSent() + { + return channelBytesSent; + } + + @Managed + @Nested + public CounterStat getConnectionsOpened() + { + return connectionsOpened; + } + + @Managed + @Nested + public CounterStat getConnectionsClosed() + { + return connectionsClosed; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java index 9ef85adcae21a..638d6cf3ac60f 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/HttpRemoteTaskWithEventLoop.java @@ -88,12 +88,13 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; -import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import static com.facebook.airlift.http.client.HttpStatus.NO_CONTENT; @@ -183,16 +184,15 @@ public final class HttpRemoteTaskWithEventLoop private long currentRequestLastTaskUpdate; private final SetMultimap pendingSplits = HashMultimap.create(); - private volatile int pendingSourceSplitCount; - private volatile long pendingSourceSplitsWeight; + private final AtomicInteger pendingSourceSplitCount = new AtomicInteger(); + private final AtomicLong pendingSourceSplitsWeight = new AtomicLong(); private final SetMultimap pendingNoMoreSplitsForLifespan = HashMultimap.create(); // The keys of this map represent all plan nodes that have "no more splits". // The boolean value of each entry represents whether the "no more splits" notification is pending delivery to workers. private final Map noMoreSplits = new HashMap<>(); private OutputBuffers outputBuffers; private final FutureStateChange whenSplitQueueHasSpace = new FutureStateChange<>(); - private volatile boolean splitQueueHasSpace; - private OptionalLong whenSplitQueueHasSpaceThreshold = OptionalLong.empty(); + private volatile long whenSplitQueueWeightThreshold = Long.MAX_VALUE; private final boolean summarizeTaskInfo; @@ -235,6 +235,9 @@ public final class HttpRemoteTaskWithEventLoop private final SafeEventLoopGroup.SafeEventLoop taskEventLoop; private final String loggingPrefix; + private long startTime; + private long startedTime; + public static HttpRemoteTaskWithEventLoop createHttpRemoteTaskWithEventLoop( Session session, TaskId taskId, @@ -431,8 +434,8 @@ private HttpRemoteTaskWithEventLoop(Session session, pendingSourceSplitsWeight = addExact(pendingSourceSplitsWeight, SplitWeight.rawValueSum(tableScanSplits, Split::getSplitWeight)); } } - this.pendingSourceSplitCount = pendingSourceSplitCount; - this.pendingSourceSplitsWeight = pendingSourceSplitsWeight; + this.pendingSourceSplitCount.set(pendingSourceSplitCount); + this.pendingSourceSplitsWeight.set(pendingSourceSplitsWeight); List bufferStates = outputBuffers.getBuffers() .keySet().stream() @@ -535,9 +538,12 @@ public URI getRemoteTaskLocation() @Override public void start() { + startTime = System.nanoTime(); safeExecuteOnEventLoop(() -> { // to start we just need to trigger an update started = true; + startedTime = System.nanoTime(); + schedulerStatsTracker.recordStartWaitForEventLoop(startedTime - startTime); scheduleUpdate(); taskStatusFetcher.start(); @@ -555,32 +561,37 @@ public void addSplits(Multimap splitsBySource) return; } + int count = 0; + long weight = 0; + for (Entry> entry : splitsBySource.asMap().entrySet()) { + PlanNodeId sourceId = entry.getKey(); + Collection splits = entry.getValue(); + + if (tableScanPlanNodeIds.contains(sourceId)) { + count += splits.size(); + weight += splits.stream().map(Split::getSplitWeight) + .mapToLong(SplitWeight::getRawValue) + .sum(); + } + } + if (count != 0) { + pendingSourceSplitCount.addAndGet(count); + pendingSourceSplitsWeight.addAndGet(weight); + updateTaskStats(); + } + safeExecuteOnEventLoop(() -> { boolean updateNeeded = false; for (Entry> entry : splitsBySource.asMap().entrySet()) { PlanNodeId sourceId = entry.getKey(); Collection splits = entry.getValue(); - boolean isTableScanSource = tableScanPlanNodeIds.contains(sourceId); checkState(!noMoreSplits.containsKey(sourceId), "noMoreSplits has already been set for %s", sourceId); - int added = 0; - long addedWeight = 0; for (Split split : splits) { - if (pendingSplits.put(sourceId, new ScheduledSplit(nextSplitId++, sourceId, split))) { - if (isTableScanSource) { - added++; - addedWeight = addExact(addedWeight, split.getSplitWeight().getRawValue()); - } - } - } - if (isTableScanSource) { - pendingSourceSplitCount += added; - pendingSourceSplitsWeight = addExact(pendingSourceSplitsWeight, addedWeight); - updateTaskStats(); + pendingSplits.put(sourceId, new ScheduledSplit(nextSplitId++, sourceId, split)); } updateNeeded = true; } - updateSplitQueueSpace(); if (updateNeeded) { needsUpdate = true; @@ -722,9 +733,7 @@ public PartitionedSplitsInfo getPartitionedSplitsInfo() @SuppressWarnings("FieldAccessNotGuarded") public PartitionedSplitsInfo getUnacknowledgedPartitionedSplitsInfo() { - int count = pendingSourceSplitCount; - long weight = pendingSourceSplitsWeight; - return PartitionedSplitsInfo.forSplitCountAndWeightSum(count, weight); + return PartitionedSplitsInfo.forSplitCountAndWeightSum(pendingSourceSplitCount.get(), pendingSourceSplitsWeight.get()); } @Override @@ -749,7 +758,7 @@ public int getUnacknowledgedPartitionedSplitCount() @SuppressWarnings("FieldAccessNotGuarded") private int getPendingSourceSplitCount() { - return pendingSourceSplitCount; + return pendingSourceSplitCount.get(); } private long getQueuedPartitionedSplitsWeight() @@ -764,7 +773,7 @@ private long getQueuedPartitionedSplitsWeight() @SuppressWarnings("FieldAccessNotGuarded") private long getPendingSourceSplitsWeight() { - return pendingSourceSplitsWeight; + return pendingSourceSplitsWeight.get(); } @Override @@ -782,35 +791,45 @@ public void addFinalTaskInfoListener(StateChangeListener stateChangeLi @Override public ListenableFuture whenSplitQueueHasSpace(long weightThreshold) { - if (splitQueueHasSpace) { + setSplitQueueWeightThreshold(weightThreshold); + + if (splitQueueHasSpace()) { return immediateFuture(null); } SettableFuture future = SettableFuture.create(); safeExecuteOnEventLoop(() -> { - if (whenSplitQueueHasSpaceThreshold.isPresent()) { - checkArgument(weightThreshold == whenSplitQueueHasSpaceThreshold.getAsLong(), "Multiple split queue space notification thresholds not supported"); + if (splitQueueHasSpace()) { + future.set(null); } else { - whenSplitQueueHasSpaceThreshold = OptionalLong.of(weightThreshold); - updateSplitQueueSpace(); - } - if (splitQueueHasSpace) { - future.set(null); + whenSplitQueueHasSpace.createNewListener().addListener(() -> future.set(null), taskEventLoop); } - whenSplitQueueHasSpace.createNewListener().addListener(() -> future.set(null), taskEventLoop); }, "whenSplitQueueHasSpace"); return future; } + private void setSplitQueueWeightThreshold(long weightThreshold) + { + long currentValue = whenSplitQueueWeightThreshold; + if (currentValue != Long.MAX_VALUE) { + checkArgument(weightThreshold == currentValue, "Multiple split queue space notification thresholds not supported"); + } + else { + whenSplitQueueWeightThreshold = weightThreshold; + } + } + + private boolean splitQueueHasSpace() + { + return getUnacknowledgedPartitionedSplitCount() < maxUnacknowledgedSplits && + getQueuedPartitionedSplitsWeight() < whenSplitQueueWeightThreshold; + } + private void updateSplitQueueSpace() { verify(taskEventLoop.inEventLoop()); - - // Must check whether the unacknowledged split count threshold is reached even without listeners registered yet - splitQueueHasSpace = getUnacknowledgedPartitionedSplitCount() < maxUnacknowledgedSplits && - (!whenSplitQueueHasSpaceThreshold.isPresent() || getQueuedPartitionedSplitsWeight() < whenSplitQueueHasSpaceThreshold.getAsLong()); // Only trigger notifications if a listener might be registered - if (splitQueueHasSpace && whenSplitQueueHasSpaceThreshold.isPresent()) { + if (splitQueueHasSpace()) { whenSplitQueueHasSpace.complete(null, taskEventLoop); } } @@ -838,12 +857,13 @@ private void processTaskUpdate(TaskInfo newValue, List sources) //Once it is converted to thrift we can use the isThrift enabled flag here. updateTaskInfo(newValue); + int removed = 0; + long removedWeight = 0; + // remove acknowledged splits, which frees memory for (TaskSource source : sources) { PlanNodeId planNodeId = source.getPlanNodeId(); boolean isTableScanSource = tableScanPlanNodeIds.contains(planNodeId); - int removed = 0; - long removedWeight = 0; for (ScheduledSplit split : source.getSplits()) { if (pendingSplits.remove(planNodeId, split)) { if (isTableScanSource) { @@ -858,14 +878,14 @@ private void processTaskUpdate(TaskInfo newValue, List sources) for (Lifespan lifespan : source.getNoMoreSplitsForLifespan()) { pendingNoMoreSplitsForLifespan.remove(planNodeId, lifespan); } - if (isTableScanSource) { - pendingSourceSplitCount -= removed; - pendingSourceSplitsWeight -= removedWeight; - } } // Update stats before split queue space to ensure node stats are up to date before waking up the scheduler - updateTaskStats(); - updateSplitQueueSpace(); + if (removed != 0) { + pendingSourceSplitCount.addAndGet(-removed); + pendingSourceSplitsWeight.addAndGet(-removedWeight); + updateTaskStats(); + updateSplitQueueSpace(); + } } private void onSuccessTaskInfo(TaskInfo result) @@ -1115,10 +1135,9 @@ private void cleanUpTask() // clear pending splits to free memory pendingSplits.clear(); - pendingSourceSplitCount = 0; - pendingSourceSplitsWeight = 0; + pendingSourceSplitCount.set(0); + pendingSourceSplitsWeight.set(0); updateTaskStats(); - splitQueueHasSpace = true; whenSplitQueueHasSpace.complete(null, taskEventLoop); // cancel pending request @@ -1297,6 +1316,7 @@ public void success(TaskInfo value) processTaskUpdate(value, sources); updateErrorTracker.requestSucceeded(); if (oldestTaskUpdateTime != 0) { + schedulerStatsTracker.recordDeliveredUpdates(deliveredUpdates); schedulerStatsTracker.recordTaskUpdateDeliveredTime(System.nanoTime() - oldestTaskUpdateTime); } } @@ -1350,6 +1370,7 @@ private void updateStats(long currentRequestStartNanos) verify(taskEventLoop.inEventLoop()); Duration requestRoundTrip = Duration.nanosSince(currentRequestStartNanos); stats.updateRoundTripMillis(requestRoundTrip.toMillis()); + schedulerStatsTracker.recordRoundTripTime(requestRoundTrip.toMillis() * 1000000); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClient.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClient.java index bd071f81ffd47..a018bebaaa8be 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClient.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClient.java @@ -26,8 +26,6 @@ import com.google.common.collect.ListMultimap; import com.google.common.util.concurrent.SettableFuture; import com.google.inject.Inject; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.jmx.JmxMeterRegistry; import io.netty.channel.ChannelOption; import io.netty.channel.epoll.Epoll; import io.netty.handler.codec.http.HttpHeaders; @@ -39,7 +37,6 @@ import reactor.core.Disposable; import reactor.core.publisher.Mono; import reactor.netty.ByteBufFlux; -import reactor.netty.channel.MicrometerChannelMetricsRecorder; import reactor.netty.http.HttpProtocol; import reactor.netty.http.client.Http2AllocationStrategy; import reactor.netty.http.client.HttpClient; @@ -64,11 +61,10 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Function; import static com.facebook.airlift.security.pem.PemReader.loadPrivateKey; import static com.facebook.airlift.security.pem.PemReader.readCertificateChain; -import static io.micrometer.core.instrument.Clock.SYSTEM; -import static io.micrometer.jmx.JmxConfig.DEFAULT; import static io.netty.handler.ssl.ApplicationProtocolConfig.Protocol.ALPN; import static io.netty.handler.ssl.ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT; import static io.netty.handler.ssl.ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE; @@ -91,10 +87,14 @@ public class ReactorNettyHttpClient private final Duration requestTimeout; private HttpClient httpClient; + private final HttpClientConnectionPoolStats connectionPoolStats; + private final HttpClientStats httpClientStats; @Inject - public ReactorNettyHttpClient(ReactorNettyHttpClientConfig config) + public ReactorNettyHttpClient(ReactorNettyHttpClientConfig config, HttpClientConnectionPoolStats connectionPoolStats, HttpClientStats httpClientStats) { + this.connectionPoolStats = connectionPoolStats; + this.httpClientStats = httpClientStats; SslContext sslContext = null; if (config.isHttpsEnabled()) { try { @@ -151,6 +151,11 @@ public ReactorNettyHttpClient(ReactorNettyHttpClientConfig config) */ ConnectionProvider pool = ConnectionProvider.builder("shared-pool") .maxConnections(config.getMaxConnections()) + .fifo() + .maxIdleTime(java.time.Duration.of(config.getMaxIdleTime().toMillis(), MILLIS)) + .evictInBackground(java.time.Duration.of(config.getEvictBackgroundTime().toMillis(), MILLIS)) + .pendingAcquireTimeout(java.time.Duration.of(config.getPendingAcquireTimeout().toMillis(), MILLIS)) + .metrics(true, () -> connectionPoolStats) .allocationStrategy((Http2AllocationStrategy.builder() .maxConnections(config.getMaxConnections()) .maxConcurrentStreams(config.getMaxStreamPerChannel()) @@ -159,10 +164,6 @@ public ReactorNettyHttpClient(ReactorNettyHttpClientConfig config) LoopResources loopResources = LoopResources.create("event-loop", config.getSelectorThreadCount(), config.getEventLoopThreadCount(), true, false); - // Add the JMX MeterRegistry to the global Metrics registry - JmxMeterRegistry jmxMeterRegistry = new JmxMeterRegistry(DEFAULT, SYSTEM); - Metrics.addRegistry(jmxMeterRegistry); - // Create HTTP/2 client SslContext finalSslContext = sslContext; this.httpClient = HttpClient @@ -170,10 +171,16 @@ public ReactorNettyHttpClient(ReactorNettyHttpClientConfig config) .create(pool) .protocol(HttpProtocol.H2, HttpProtocol.HTTP11) .runOn(loopResources, true) - .http2Settings(settings -> settings.maxConcurrentStreams(config.getMaxStreamPerChannel())) + .http2Settings(settings -> { + settings.maxConcurrentStreams(config.getMaxStreamPerChannel()); + settings.initialWindowSize((int) (config.getMaxInitialWindowSize().toBytes())); + settings.maxFrameSize((int) (config.getMaxFrameSize().toBytes())); + }) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) config.getConnectTimeout().getValue()) - // Track the metrics for all the tcp connections - .metrics(true, () -> new MicrometerChannelMetricsRecorder("reactor.netty.http.client", "tcp", false)); + .option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.TCP_NODELAY, true) + // Track HTTP client metrics + .metrics(true, () -> httpClientStats, Function.identity()); if (config.isHttpsEnabled()) { if (finalSslContext == null) { diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClientConfig.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClientConfig.java index ab068ca31988a..63ccc174b364b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClientConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ReactorNettyHttpClientConfig.java @@ -15,11 +15,13 @@ import com.facebook.airlift.configuration.Config; import com.facebook.airlift.configuration.ConfigDescription; +import com.facebook.airlift.units.DataSize; import com.facebook.airlift.units.Duration; import jakarta.validation.constraints.Min; import java.util.Optional; +import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.TimeUnit.SECONDS; public class ReactorNettyHttpClientConfig @@ -33,6 +35,11 @@ public class ReactorNettyHttpClientConfig private int eventLoopThreadCount = Runtime.getRuntime().availableProcessors(); private Duration connectTimeout = new Duration(10, SECONDS); private Duration requestTimeout = new Duration(10, SECONDS); + private Duration maxIdleTime = new Duration(45, SECONDS); + private Duration evictBackgroundTime = new Duration(15, SECONDS); + private Duration pendingAcquireTimeout = new Duration(2, SECONDS); + private DataSize maxInitialWindowSize = new DataSize(25, MEGABYTE); + private DataSize maxFrameSize = new DataSize(8, MEGABYTE); private String keyStorePath; private String keyStorePassword; private String trustStorePath; @@ -154,6 +161,66 @@ public ReactorNettyHttpClientConfig setRequestTimeout(Duration requestTimeout) return this; } + public Duration getMaxIdleTime() + { + return maxIdleTime; + } + + @Config("reactor.max-idle-time") + public ReactorNettyHttpClientConfig setMaxIdleTime(Duration maxIdleTime) + { + this.maxIdleTime = maxIdleTime; + return this; + } + + public Duration getEvictBackgroundTime() + { + return evictBackgroundTime; + } + + @Config("reactor.evict-background-time") + public ReactorNettyHttpClientConfig setEvictBackgroundTime(Duration evictBackgroundTime) + { + this.evictBackgroundTime = evictBackgroundTime; + return this; + } + + public Duration getPendingAcquireTimeout() + { + return pendingAcquireTimeout; + } + + @Config("reactor.pending-acquire-timeout") + public ReactorNettyHttpClientConfig setPendingAcquireTimeout(Duration pendingAcquireTimeout) + { + this.pendingAcquireTimeout = pendingAcquireTimeout; + return this; + } + + public DataSize getMaxInitialWindowSize() + { + return maxInitialWindowSize; + } + + @Config("reactor.max-initial-window-size") + public ReactorNettyHttpClientConfig setMaxInitialWindowSize(DataSize maxInitialWindowSize) + { + this.maxInitialWindowSize = maxInitialWindowSize; + return this; + } + + public DataSize getMaxFrameSize() + { + return maxFrameSize; + } + + @Config("reactor.max-frame-size") + public ReactorNettyHttpClientConfig setMaxFrameSize(DataSize maxFrameSize) + { + this.maxFrameSize = maxFrameSize; + return this; + } + public String getKeyStorePath() { return keyStorePath; diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index cb0a4850a197c..3f19772431415 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -39,7 +39,6 @@ import com.facebook.presto.cost.StatsCalculator; import com.facebook.presto.dispatcher.DispatchManager; import com.facebook.presto.dispatcher.QueryPrerequisitesManagerModule; -import com.facebook.presto.eventlistener.EventListenerConfig; import com.facebook.presto.eventlistener.EventListenerManager; import com.facebook.presto.execution.QueryInfo; import com.facebook.presto.execution.QueryManager; @@ -74,7 +73,6 @@ import com.facebook.presto.spi.eventlistener.EventListener; import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; -import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; @@ -85,11 +83,10 @@ import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.sql.planner.Plan; import com.facebook.presto.sql.planner.sanity.PlanCheckerProviderManager; -import com.facebook.presto.storage.TempStorageManager; import com.facebook.presto.testing.ProcedureTester; import com.facebook.presto.testing.TestingAccessControlManager; import com.facebook.presto.testing.TestingEventListenerManager; -import com.facebook.presto.testing.TestingTempStorageManager; +import com.facebook.presto.testing.TestingPrestoServerModule; import com.facebook.presto.testing.TestingWarningCollectorModule; import com.facebook.presto.transaction.TransactionManager; import com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule; @@ -289,6 +286,42 @@ public TestingPrestoServer( List additionalModules, Optional dataDirectory) throws Exception + { + this( + resourceManager, + resourceManagerEnabled, + catalogServer, + catalogServerEnabled, + coordinatorSidecar, + coordinatorSidecarEnabled, + coordinator, + skipLoadingResourceGroupConfigurationManager, + true, + properties, + environment, + discoveryUri, + parserOptions, + additionalModules, + dataDirectory); + } + + public TestingPrestoServer( + boolean resourceManager, + boolean resourceManagerEnabled, + boolean catalogServer, + boolean catalogServerEnabled, + boolean coordinatorSidecar, + boolean coordinatorSidecarEnabled, + boolean coordinator, + boolean skipLoadingResourceGroupConfigurationManager, + boolean loadDefaultSystemAccessControl, + Map properties, + String environment, + URI discoveryUri, + SqlParserOptions parserOptions, + List additionalModules, + Optional dataDirectory) + throws Exception { this.resourceManager = resourceManager; this.catalogServer = catalogServer; @@ -328,18 +361,9 @@ public TestingPrestoServer( .add(new NodeTtlFetcherManagerModule()) .add(new ClusterTtlProviderManagerModule()) .add(new ClientRequestFilterModule()) + .add(new TestingPrestoServerModule(loadDefaultSystemAccessControl)) .add(binder -> { - binder.bind(TestingAccessControlManager.class).in(Scopes.SINGLETON); - binder.bind(TestingEventListenerManager.class).in(Scopes.SINGLETON); - binder.bind(TestingTempStorageManager.class).in(Scopes.SINGLETON); - binder.bind(AccessControlManager.class).to(TestingAccessControlManager.class).in(Scopes.SINGLETON); - binder.bind(EventListenerManager.class).to(TestingEventListenerManager.class).in(Scopes.SINGLETON); - binder.bind(EventListenerConfig.class).in(Scopes.SINGLETON); - binder.bind(TempStorageManager.class).to(TestingTempStorageManager.class).in(Scopes.SINGLETON); - binder.bind(AccessControl.class).to(AccessControlManager.class).in(Scopes.SINGLETON); binder.bind(ShutdownAction.class).to(TestShutdownAction.class).in(Scopes.SINGLETON); - binder.bind(GracefulShutdownHandler.class).in(Scopes.SINGLETON); - binder.bind(ProcedureTester.class).in(Scopes.SINGLETON); binder.bind(RequestBlocker.class).in(Scopes.SINGLETON); newSetBinder(binder, Filter.class, TheServlet.class).addBinding() .to(RequestBlocker.class).in(Scopes.SINGLETON); @@ -385,7 +409,7 @@ public TestingPrestoServer( transactionManager = injector.getInstance(TransactionManager.class); sqlParser = injector.getInstance(SqlParser.class); metadata = injector.getInstance(Metadata.class); - accessControl = injector.getInstance(TestingAccessControlManager.class); + accessControl = (TestingAccessControlManager) injector.getInstance(AccessControlManager.class); procedureTester = injector.getInstance(ProcedureTester.class); splitManager = injector.getInstance(SplitManager.class); pageSourceManager = injector.getInstance(PageSourceManager.class); diff --git a/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClient.java b/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClient.java index a49b69f6f8644..061615cf37fcd 100644 --- a/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClient.java +++ b/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClient.java @@ -20,6 +20,8 @@ import com.facebook.airlift.json.JsonObjectMapperProvider; import com.facebook.airlift.units.Duration; import com.facebook.presto.execution.TaskStatus; +import com.facebook.presto.server.remotetask.HttpClientConnectionPoolStats; +import com.facebook.presto.server.remotetask.HttpClientStats; import com.facebook.presto.server.remotetask.ReactorNettyHttpClient; import com.facebook.presto.server.remotetask.ReactorNettyHttpClientConfig; import com.facebook.presto.server.smile.AdaptingJsonResponseHandler; @@ -88,7 +90,7 @@ public static void setUp() ReactorNettyHttpClientConfig reactorNettyHttpClientConfig = new ReactorNettyHttpClientConfig() .setRequestTimeout(new Duration(30, TimeUnit.SECONDS)) .setConnectTimeout(new Duration(30, TimeUnit.SECONDS)); - reactorNettyHttpClient = new ReactorNettyHttpClient(reactorNettyHttpClientConfig); + reactorNettyHttpClient = new ReactorNettyHttpClient(reactorNettyHttpClientConfig, new HttpClientConnectionPoolStats(), new HttpClientStats()); } @AfterClass diff --git a/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClientConfig.java b/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClientConfig.java index 60d78542769d6..29d913fb3adab 100644 --- a/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClientConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/remotetask/TestReactorNettyHttpClientConfig.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.remotetask; +import com.facebook.airlift.units.DataSize; import com.facebook.airlift.units.Duration; import com.facebook.presto.server.remotetask.ReactorNettyHttpClientConfig; import com.google.common.collect.ImmutableMap; @@ -23,6 +24,7 @@ import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE; import static java.util.concurrent.TimeUnit.SECONDS; public class TestReactorNettyHttpClientConfig @@ -40,6 +42,11 @@ public void testDefaults() .setEventLoopThreadCount(Runtime.getRuntime().availableProcessors()) .setConnectTimeout(new Duration(10, SECONDS)) .setRequestTimeout(new Duration(10, SECONDS)) + .setMaxIdleTime(new Duration(45, SECONDS)) + .setEvictBackgroundTime(new Duration(15, SECONDS)) + .setPendingAcquireTimeout(new Duration(2, SECONDS)) + .setMaxInitialWindowSize(new DataSize(25, MEGABYTE)) // 25MB + .setMaxFrameSize(new DataSize(8, MEGABYTE)) // 8MB .setKeyStorePath(null) .setKeyStorePassword(null) .setTrustStorePath(null) @@ -59,6 +66,11 @@ public void testExplicitPropertyMappings() .put("reactor.event-loop-thread-count", "150") .put("reactor.connect-timeout", "2s") .put("reactor.request-timeout", "1s") + .put("reactor.max-idle-time", "120s") + .put("reactor.evict-background-time", "120s") + .put("reactor.pending-acquire-timeout", "10s") + .put("reactor.max-initial-window-size", "10MB") + .put("reactor.max-frame-size", "4MB") .put("reactor.keystore-path", "/var/abc/def/presto.jks") .put("reactor.truststore-path", "/var/abc/def/presto.jks") .put("reactor.keystore-password", "password") @@ -75,6 +87,11 @@ public void testExplicitPropertyMappings() .setEventLoopThreadCount(150) .setConnectTimeout(new Duration(2, SECONDS)) .setRequestTimeout(new Duration(1, SECONDS)) + .setMaxIdleTime(new Duration(120, SECONDS)) + .setEvictBackgroundTime(new Duration(120, SECONDS)) + .setPendingAcquireTimeout(new Duration(10, SECONDS)) + .setMaxInitialWindowSize(new DataSize(10, MEGABYTE)) // 10MB + .setMaxFrameSize(new DataSize(4, MEGABYTE)) // 4MB .setKeyStorePath("/var/abc/def/presto.jks") .setTrustStorePath("/var/abc/def/presto.jks") .setKeyStorePassword("password") diff --git a/presto-matching/pom.xml b/presto-matching/pom.xml index f2d2c8e3bc2f1..87fd346bb77ea 100644 --- a/presto-matching/pom.xml +++ b/presto-matching/pom.xml @@ -18,7 +18,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-matching diff --git a/presto-memory-context/pom.xml b/presto-memory-context/pom.xml index 27341f7917bd8..d4dd96a181ace 100644 --- a/presto-memory-context/pom.xml +++ b/presto-memory-context/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-memory-context diff --git a/presto-memory/pom.xml b/presto-memory/pom.xml index 9c912cf2eeb04..8c608e4f47d5d 100644 --- a/presto-memory/pom.xml +++ b/presto-memory/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-memory diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 110e307f6416e..075effcff022a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 8b4d7b916f61e..7794261a6dd79 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 59af1ad5ed4fd..8ff8676de5ce7 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-mysql @@ -173,7 +173,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index 6b0e3ede52709..9679bd8618699 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-native-execution diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index 29836ba760385..da8f6debfa0e3 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -15,7 +15,7 @@ add_subdirectory(http) add_subdirectory(common) add_subdirectory(thrift) add_subdirectory(connectors) -add_subdirectory(dynamic_registry) +add_subdirectory(functions) add_library( presto_server_lib diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 75a2bb1eedfe4..80b3531ceb05b 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -28,6 +28,7 @@ #include "presto_cpp/main/common/Utils.h" #include "presto_cpp/main/connectors/Registration.h" #include "presto_cpp/main/connectors/SystemConnector.h" +#include "presto_cpp/main/functions/FunctionMetadata.h" #include "presto_cpp/main/http/HttpConstants.h" #include "presto_cpp/main/http/filters/AccessLogFilter.h" #include "presto_cpp/main/http/filters/HttpEndpointLatencyFilter.h" @@ -39,7 +40,6 @@ #include "presto_cpp/main/operators/PartitionAndSerialize.h" #include "presto_cpp/main/operators/ShuffleRead.h" #include "presto_cpp/main/operators/UnsafeRowExchangeSource.h" -#include "presto_cpp/main/types/FunctionMetadata.h" #include "presto_cpp/main/types/PrestoToVeloxQueryPlan.h" #include "presto_cpp/main/types/VeloxPlanConversion.h" #include "velox/common/base/Counters.h" @@ -1264,14 +1264,12 @@ std::vector PrestoServer::registerVeloxConnectors( // make sure connector type is supported getPrestoToVeloxConnector(connectorName); - - std::shared_ptr connector = - velox::connector::getConnectorFactory(connectorName) - ->newConnector( - catalogName, - std::move(properties), - connectorIoExecutor_.get(), - connectorCpuExecutor_.get()); + auto connector = getConnectorFactory(connectorName) + ->newConnector( + catalogName, + std::move(properties), + connectorIoExecutor_.get(), + connectorCpuExecutor_.get()); velox::connector::registerConnector(connector); } } diff --git a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp index dfab8629aa2c4..e9be6b9aaad8b 100644 --- a/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoToVeloxQueryConfig.cpp @@ -70,6 +70,10 @@ void updateFromSessionConfigs( } } + if (session.startTime) { + queryConfigs[velox::core::QueryConfig::kSessionStartTime] = std::to_string(session.startTime); + } + if (session.source) { queryConfigs[velox::core::QueryConfig::kSource] = *session.source; } diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index 4bcd0b9ff563b..21baee18838f0 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -541,6 +541,38 @@ SessionProperties::SessionProperties() { false, QueryConfig::kUnnestSplitOutput, std::to_string(c.unnestSplitOutput())); + + addSessionProperty( + kPreferredOutputBatchBytes, + "Prefered memory budget for operator output batches. Used in tandem with average row size estimates when available.", + BIGINT(), + 10UL * 1048576, + QueryConfig::kPreferredOutputBatchBytes, + std::to_string(c.preferredOutputBatchBytes())); + + addSessionProperty( + kPreferredOutputBatchRows, + "Preferred row count per operator output batch. Used when average row size estimates are unknown.", + INTEGER(), + 1024, + QueryConfig::kPreferredOutputBatchRows, + std::to_string(c.preferredOutputBatchRows())); + + addSessionProperty( + kMaxOutputBatchRows, + "Upperbound for row count per output batch, used together with preferred_output_batch_bytes and average row size estimates.", + INTEGER(), + 10'000, + QueryConfig::kMaxOutputBatchRows, + std::to_string(c.maxOutputBatchRows())); + + addSessionProperty( + kRowSizeTrackingEnabled, + "A fallback for average row size estimate when not supported for certain readers. Turned on by default.", + BOOLEAN(), + true, + QueryConfig::kRowSizeTrackingEnabled, + std::to_string(c.rowSizeTrackingEnabled())); } const std::string SessionProperties::toVeloxConfig( diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index effb56b7f57b3..24ed4bf8208b5 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -339,6 +339,29 @@ class SessionProperties { /// a single output for each input batch. static constexpr const char* kUnnestSplitOutput = "native_unnest_split_output"; + + /// Preferred size of batches in bytes to be returned by operators from + /// Operator::getOutput. It is used when an estimate of average row size is + /// known. Otherwise kPreferredOutputBatchRows is used. + static constexpr const char* kPreferredOutputBatchBytes = + "preferred_output_batch_bytes"; + + /// Preferred number of rows to be returned by operators from + /// Operator::getOutput. It is used when an estimate of average row size is + /// not known. When the estimate of average row size is known, + /// kPreferredOutputBatchBytes is used. + static constexpr const char* kPreferredOutputBatchRows = + "preferred_output_batch_rows"; + + /// Max number of rows that could be return by operators from + /// Operator::getOutput. It is used when an estimate of average row size is + /// known and kPreferredOutputBatchBytes is used to compute the number of + /// output rows. + static constexpr const char* kMaxOutputBatchRows = "max_output_batch_rows"; + + /// Enable (reader) row size tracker as a fallback to file level row size estimates. + static constexpr const char* kRowSizeTrackingEnabled = + "row_size_tracking_enabled"; static SessionProperties* instance(); diff --git a/presto-native-execution/presto_cpp/main/TaskManager.cpp b/presto-native-execution/presto_cpp/main/TaskManager.cpp index ca5cee1aa9678..ab0fb8809c816 100644 --- a/presto-native-execution/presto_cpp/main/TaskManager.cpp +++ b/presto-native-execution/presto_cpp/main/TaskManager.cpp @@ -55,53 +55,53 @@ void cancelAbandonedTasksInternal(const TaskMap& taskMap, int32_t abandonedMs) { } } -// If spilling is enabled and the given Task can spill, then this helper -// generates the spilling directory path for the Task, and sets the path to it -// in the Task. -static void maybeSetupTaskSpillDirectory( +// If spilling is enabled and the task plan fragment can spill, then this helper +// generates the disk spilling options for the task. +std::optional getTaskSpillOptions( + const TaskId& taskId, const core::PlanFragment& planFragment, - exec::Task& execTask, - const std::string& baseSpillDirectory) { - if (baseSpillDirectory.empty() || - !planFragment.canSpill(execTask.queryCtx()->queryConfig())) { - return; + const std::shared_ptr& queryCtx, + const std::string& baseSpillDir) { + if (baseSpillDir.empty() || !planFragment.canSpill(queryCtx->queryConfig())) { + return std::nullopt; } - const auto includeNodeInSpillPath = + common::SpillDiskOptions spillDiskOpts; + const bool includeNodeInSpillPath = SystemConfig::instance()->includeNodeInSpillPath(); auto nodeConfig = NodeConfig::instance(); const auto [taskSpillDirPath, dateSpillDirPath] = TaskManager::buildTaskSpillDirectoryPath( - baseSpillDirectory, + baseSpillDir, nodeConfig->nodeInternalAddress(), nodeConfig->nodeId(), - execTask.queryCtx()->queryId(), - execTask.taskId(), + queryCtx->queryId(), + taskId, includeNodeInSpillPath); - execTask.setSpillDirectory(taskSpillDirPath, /*alreadyCreated=*/false); - - execTask.setCreateSpillDirectoryCb( - [spillDir = taskSpillDirPath, dateStrDir = dateSpillDirPath]() { - auto fs = filesystems::getFileSystem(dateStrDir, nullptr); - // First create the top level directory (date string of the query) with - // TTL or other configs if set. - filesystems::DirectoryOptions options; - // Do not fail if the directory already exist because another process - // may have already created the dateStrDir. - options.failIfExists = false; - auto config = SystemConfig::instance()->spillerDirectoryCreateConfig(); - if (!config.empty()) { - options.values.emplace( - filesystems::DirectoryOptions::kMakeDirectoryConfig.toString(), - config); - } - fs->mkdir(dateStrDir, options); - - // After the parent directory is created, - // then create the spill directory for the actual task. - fs->mkdir(spillDir); - return spillDir; - }); + spillDiskOpts.spillDirPath = taskSpillDirPath; + spillDiskOpts.spillDirCreated = false; + spillDiskOpts.spillDirCreateCb = [spillDir = taskSpillDirPath, + dateDir = dateSpillDirPath]() { + auto fs = filesystems::getFileSystem(dateDir, nullptr); + // First create the top level directory (date string of the query) with + // TTL or other configs if set. + filesystems::DirectoryOptions options; + // Do not fail if the directory already exist because another process + // may have already created the dateStrDir. + options.failIfExists = false; + auto config = SystemConfig::instance()->spillerDirectoryCreateConfig(); + if (!config.empty()) { + options.values.emplace( + filesystems::DirectoryOptions::kMakeDirectoryConfig.toString(), + config); + } + fs->mkdir(dateDir, options); + // After the parent directory is created, + // then create the spill directory for the actual task. + fs->mkdir(spillDir); + return spillDir; + }; + return spillDiskOpts; } // Keep outstanding Promises in RequestHandler's state itself. @@ -554,6 +554,10 @@ std::unique_ptr TaskManager::createOrUpdateTaskImpl( prestoTask->updateInfoLocked(summarize)); } + const auto baseSpillDir = *(baseSpillDir_.rlock()); + auto spillDiskOpts = + getTaskSpillOptions(taskId, planFragment, queryCtx, baseSpillDir); + // Uses a temp variable to store the created velox task to destroy it // under presto task lock if spill directory setup fails. Otherwise, the // concurrent task creation retry from the coordinator might see the @@ -567,12 +571,8 @@ std::unique_ptr TaskManager::createOrUpdateTaskImpl( std::move(queryCtx), exec::Task::ExecutionMode::kParallel, static_cast(nullptr), - prestoTask->id.stageId()); - // TODO: move spill directory creation inside velox task execution - // whenever spilling is triggered. It will reduce the unnecessary file - // operations on remote storage. - const auto baseSpillDir = *(baseSpillDir_.rlock()); - maybeSetupTaskSpillDirectory(planFragment, *newExecTask, baseSpillDir); + prestoTask->id.stageId(), + spillDiskOpts); prestoTask->task = std::move(newExecTask); prestoTask->info.needsPlan = false; diff --git a/presto-native-execution/presto_cpp/main/common/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/CMakeLists.txt index c19e2c0b7eb2a..582bdb3b0fd51 100644 --- a/presto-native-execution/presto_cpp/main/common/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/CMakeLists.txt @@ -18,7 +18,7 @@ set_property(TARGET presto_exception PROPERTY JOB_POOL_LINK presto_link_job_pool) target_link_libraries(presto_common velox_common_config velox_core - velox_exception) + velox_exception velox_presto_serializer) set_property(TARGET presto_common PROPERTY JOB_POOL_LINK presto_link_job_pool) if(PRESTO_ENABLE_TESTING) diff --git a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt index 267118dd1cd2f..fb83b2f843062 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt @@ -29,6 +29,7 @@ target_link_libraries( velox_file velox_functions_prestosql velox_function_registry + velox_presto_serializer velox_presto_types velox_window ${RE2} diff --git a/presto-native-execution/presto_cpp/main/connectors/Registration.cpp b/presto-native-execution/presto_cpp/main/connectors/Registration.cpp index d6f6555fb8a22..1650dafd16763 100644 --- a/presto-native-execution/presto_cpp/main/connectors/Registration.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/Registration.cpp @@ -28,44 +28,49 @@ namespace { constexpr char const* kHiveHadoop2ConnectorName = "hive-hadoop2"; constexpr char const* kIcebergConnectorName = "iceberg"; -void registerConnectorFactories() { - // These checks for connector factories can be removed after we remove the - // registrations from the Velox library. - if (!velox::connector::hasConnectorFactory( - velox::connector::hive::HiveConnectorFactory::kHiveConnectorName)) { - velox::connector::registerConnectorFactory( - std::make_shared()); - velox::connector::registerConnectorFactory( - std::make_shared( - kHiveHadoop2ConnectorName)); - } - if (!velox::connector::hasConnectorFactory( - velox::connector::tpch::TpchConnectorFactory::kTpchConnectorName)) { - velox::connector::registerConnectorFactory( - std::make_shared()); - } - - // Register Velox connector factory for iceberg. - // The iceberg catalog is handled by the hive connector factory. - if (!velox::connector::hasConnectorFactory(kIcebergConnectorName)) { - velox::connector::registerConnectorFactory( - std::make_shared( - kIcebergConnectorName)); - } - +const std::unordered_map< + std::string, + const std::shared_ptr>& +connectorFactories() { + static const std::unordered_map< + std::string, + const std::shared_ptr> + factories = { + {velox::connector::hive::HiveConnectorFactory::kHiveConnectorName, + std::make_shared()}, + {kHiveHadoop2ConnectorName, + std::make_shared( + kHiveHadoop2ConnectorName)}, + {velox::connector::tpch::TpchConnectorFactory::kTpchConnectorName, + std::make_shared()}, + {kIcebergConnectorName, + std::make_shared( + kIcebergConnectorName)}, #ifdef PRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR - if (!velox::connector::hasConnectorFactory( - ArrowFlightConnectorFactory::kArrowFlightConnectorName)) { - velox::connector::registerConnectorFactory( - std::make_shared()); - } + {ArrowFlightConnectorFactory::kArrowFlightConnectorName, + std::make_shared()}, #endif + }; + return factories; } + } // namespace -void registerConnectors() { - registerConnectorFactories(); +velox::connector::ConnectorFactory* getConnectorFactory( + const std::string& connectorName) { + { + auto it = connectorFactories().find(connectorName); + if (it != connectorFactories().end()) { + return it->second.get(); + } + } + if (!velox::connector::hasConnectorFactory(connectorName)) { + VELOX_FAIL("ConnectorFactory with name '{}' not registered", connectorName); + } + return velox::connector::getConnectorFactory(connectorName).get(); +} +void registerConnectors() { registerPrestoToVeloxConnector(std::make_unique( velox::connector::hive::HiveConnectorFactory::kHiveConnectorName)); registerPrestoToVeloxConnector( diff --git a/presto-native-execution/presto_cpp/main/connectors/Registration.h b/presto-native-execution/presto_cpp/main/connectors/Registration.h index c95aefaacfcaa..ee46dce009b9f 100644 --- a/presto-native-execution/presto_cpp/main/connectors/Registration.h +++ b/presto-native-execution/presto_cpp/main/connectors/Registration.h @@ -13,8 +13,18 @@ */ #pragma once +#include + +// Forward declaration for ConnectorFactory. +namespace facebook::velox::connector { +class ConnectorFactory; +} // namespace facebook::velox::connector + namespace facebook::presto { +velox::connector::ConnectorFactory* getConnectorFactory( + const std::string& connectorName); + void registerConnectors(); } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h index f491a4b3b98a9..5e71ca277f0a7 100644 --- a/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/SystemConnector.h @@ -92,7 +92,7 @@ class SystemDataSource : public velox::connector::DataSource { return completedBytes_; } - std::unordered_map runtimeStats() + std::unordered_map getRuntimeStats() override { return {}; } diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp index 5c02ac051e076..f39dbcb8215ee 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.cpp @@ -183,7 +183,7 @@ std::optional ArrowFlightDataSource::next( auto output = projectOutputColumns(chunk.data); completedRows_ += output->size(); - completedBytes_ += output->inMemoryBytes(); + completedBytes_ += output->estimateFlatSize(); return output; } diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.h b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.h index d1c431575c677..3e893164a049a 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.h +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/ArrowFlightConnector.h @@ -97,7 +97,7 @@ class ArrowFlightDataSource : public velox::connector::DataSource { return completedRows_; } - std::unordered_map runtimeStats() + std::unordered_map getRuntimeStats() override { return {}; } diff --git a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp index 1fecf9e31977a..2b692880f073d 100644 --- a/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp +++ b/presto-native-execution/presto_cpp/main/connectors/arrow_flight/tests/utils/ArrowFlightConnectorTestBase.cpp @@ -25,16 +25,9 @@ namespace facebook::presto::test { void ArrowFlightConnectorTestBase::SetUp() { OperatorTestBase::SetUp(); - - if (!velox::connector::hasConnectorFactory( - presto::ArrowFlightConnectorFactory::kArrowFlightConnectorName)) { - velox::connector::registerConnectorFactory( - std::make_shared()); - } + presto::ArrowFlightConnectorFactory factory; velox::connector::registerConnector( - velox::connector::getConnectorFactory( - ArrowFlightConnectorFactory::kArrowFlightConnectorName) - ->newConnector(kFlightConnectorId, config_)); + factory.newConnector(kFlightConnectorId, config_)); ArrowFlightConfig config(config_); if (config.defaultServerPort().has_value()) { diff --git a/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt new file mode 100644 index 0000000000000..88aa887dd4002 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/CMakeLists.txt @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(presto_function_metadata OBJECT FunctionMetadata.cpp) +target_link_libraries(presto_function_metadata velox_function_registry) + +add_subdirectory(dynamic_registry) + +if(PRESTO_ENABLE_TESTING) + add_subdirectory(tests) +endif() diff --git a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp b/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp similarity index 99% rename from presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp rename to presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp index 70fe223bab1ad..ddcaa6f86cba3 100644 --- a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp +++ b/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.cpp @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "presto_cpp/main/types/FunctionMetadata.h" +#include "presto_cpp/main/functions/FunctionMetadata.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/exec/Aggregate.h" #include "velox/exec/WindowFunction.h" diff --git a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.h b/presto-native-execution/presto_cpp/main/functions/FunctionMetadata.h similarity index 100% rename from presto-native-execution/presto_cpp/main/types/FunctionMetadata.h rename to presto-native-execution/presto_cpp/main/functions/FunctionMetadata.h diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/CMakeLists.txt rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/CMakeLists.txt diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/DynamicFunctionRegistrar.h b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/DynamicFunctionRegistrar.h similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/DynamicFunctionRegistrar.h rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/DynamicFunctionRegistrar.h diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/README.md b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/README.md rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/README.md diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/examples/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/examples/CMakeLists.txt rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/CMakeLists.txt diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicFunction.cpp b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicFunction.cpp similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicFunction.cpp rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicFunction.cpp diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicNonDefaultFunction.cpp b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicNonDefaultFunction.cpp similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicNonDefaultFunction.cpp rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicNonDefaultFunction.cpp diff --git a/presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicVarcharFunction.cpp b/presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp similarity index 100% rename from presto-native-execution/presto_cpp/main/dynamic_registry/examples/DynamicVarcharFunction.cpp rename to presto-native-execution/presto_cpp/main/functions/dynamic_registry/examples/DynamicVarcharFunction.cpp diff --git a/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt new file mode 100644 index 0000000000000..cbe76a3312c86 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/functions/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(presto_function_metadata_test FunctionMetadataTest.cpp) + +add_test( + NAME presto_function_metadata_test + COMMAND presto_function_metadata_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries( + presto_function_metadata_test + presto_function_metadata + presto_protocol + presto_type_test_utils + velox_aggregates + velox_functions_prestosql + velox_window + GTest::gtest + GTest::gtest_main) diff --git a/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp b/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp similarity index 98% rename from presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp rename to presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp index ea177f980533d..10b78e2ccef51 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp +++ b/presto-native-execution/presto_cpp/main/functions/tests/FunctionMetadataTest.cpp @@ -14,7 +14,7 @@ #include #include "presto_cpp/main/common/tests/test_json.h" -#include "presto_cpp/main/types/FunctionMetadata.h" +#include "presto_cpp/main/functions/FunctionMetadata.h" #include "presto_cpp/main/types/tests/TestUtils.h" #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json b/presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/ApproxMostFrequent.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json b/presto-native-execution/presto_cpp/main/functions/tests/data/ArrayFrequency.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/ArrayFrequency.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Combinations.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Combinations.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json b/presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/CovarSamp.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json b/presto-native-execution/presto_cpp/main/functions/tests/data/ElementAt.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/ElementAt.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Greatest.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Greatest.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Greatest.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Lead.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Lead.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Lead.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Mod.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Mod.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Mod.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Mod.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Ntile.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Ntile.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json b/presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/SetAgg.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json b/presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/StddevSamp.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json b/presto-native-execution/presto_cpp/main/functions/tests/data/TransformKeys.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/TransformKeys.json diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json b/presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json similarity index 100% rename from presto-native-execution/presto_cpp/main/types/tests/data/Variance.json rename to presto-native-execution/presto_cpp/main/functions/tests/data/Variance.json diff --git a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp index d2be3dde677e2..e80c9b7909381 100644 --- a/presto-native-execution/presto_cpp/main/http/HttpClient.cpp +++ b/presto-native-execution/presto_cpp/main/http/HttpClient.cpp @@ -197,10 +197,7 @@ class ResponseHandler : public proxygen::HTTPTransactionHandler { folly::SemiFuture> initialize( std::shared_ptr self) { self_ = self; - auto [promise, future] = - folly::makePromiseContract>(); - promise_ = std::move(promise); - return std::move(future); + return promise_.getSemiFuture(); } void setTransaction(proxygen::HTTPTransaction* /* txn */) noexcept override {} diff --git a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp index bc8446841204d..daffa64442580 100644 --- a/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/PrestoToVeloxQueryConfigTest.cpp @@ -533,3 +533,39 @@ TEST_F(PrestoToVeloxQueryConfigTest, sessionAndExtraCredentialsOverload) { EXPECT_EQ("verify_key_xyz", raw.at("verification_key")); } } + +TEST_F(PrestoToVeloxQueryConfigTest, sessionStartTimeConfiguration) { + auto session = createBasicSession(); + + // Test with session start time set in SessionRepresentation + // The startTime is already set in createBasicSession() to 1234567890 + auto veloxConfig = QueryConfig{toVeloxConfigs(session)}; + + // Verify that session start time is properly passed through to VeloxQueryConfig + EXPECT_EQ(1234567890, veloxConfig.sessionStartTimeMs()); + + // Test with different session start time + session.startTime = 9876543210; + auto veloxConfig2 = QueryConfig{toVeloxConfigs(session)}; + + EXPECT_EQ(9876543210, veloxConfig2.sessionStartTimeMs()); + + // Test with zero start time (valid edge case) + session.startTime = 0; + auto veloxConfig3 = QueryConfig{toVeloxConfigs(session)}; + + EXPECT_EQ(0, veloxConfig3.sessionStartTimeMs()); + + // Test with negative start time (valid edge case) + session.startTime = -1000; + auto veloxConfig4 = QueryConfig{toVeloxConfigs(session)}; + + EXPECT_EQ(-1000, veloxConfig4.sessionStartTimeMs()); + + // Test with maximum value + session.startTime = std::numeric_limits::max(); + auto veloxConfig5 = QueryConfig{toVeloxConfigs(session)}; + + EXPECT_EQ( + std::numeric_limits::max(), veloxConfig5.sessionStartTimeMs()); +} diff --git a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp index 458afc4fadb37..af056aa289f2f 100644 --- a/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/QueryContextManagerTest.cpp @@ -57,6 +57,9 @@ TEST_F(QueryContextManagerTest, nativeSessionProperties) { {"native_debug_disable_expression_with_memoization", "true"}, {"native_debug_disable_expression_with_lazy_inputs", "true"}, {"native_selective_nimble_reader_enabled", "true"}, + {"preferred_output_batch_bytes", "10485760"}, + {"preferred_output_batch_rows", "1024"}, + {"row_size_tracking_enabled", "true"}, {"aggregation_spill_all", "true"}, {"native_expression_max_array_size_in_reduce", "99999"}, {"native_expression_max_compiled_regexes", "54321"}, @@ -75,6 +78,9 @@ TEST_F(QueryContextManagerTest, nativeSessionProperties) { EXPECT_TRUE(queryCtx->queryConfig().debugDisableExpressionsWithMemoization()); EXPECT_TRUE(queryCtx->queryConfig().debugDisableExpressionsWithLazyInputs()); EXPECT_TRUE(queryCtx->queryConfig().selectiveNimbleReaderEnabled()); + EXPECT_TRUE(queryCtx->queryConfig().rowSizeTrackingEnabled()); + EXPECT_EQ(queryCtx->queryConfig().preferredOutputBatchBytes(), 10UL << 20); + EXPECT_EQ(queryCtx->queryConfig().preferredOutputBatchRows(), 1024); EXPECT_EQ(queryCtx->queryConfig().spillWriteBufferSize(), 1024); EXPECT_EQ(queryCtx->queryConfig().exprMaxArraySizeInReduce(), 99999); EXPECT_EQ(queryCtx->queryConfig().exprMaxCompiledRegexes(), 54321); diff --git a/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp b/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp index 68c820486d953..02c47b8cff7a7 100644 --- a/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/ServerOperationTest.cpp @@ -140,18 +140,11 @@ TEST_F(ServerOperationTest, buildServerOp) { TEST_F(ServerOperationTest, taskEndpoint) { // Setup environment for TaskManager - if (!connector::hasConnectorFactory( - connector::hive::HiveConnectorFactory::kHiveConnectorName)) { - connector::registerConnectorFactory( - std::make_shared()); - } - auto hiveConnector = - connector::getConnectorFactory( - connector::hive::HiveConnectorFactory::kHiveConnectorName) - ->newConnector( - "test-hive", - std::make_shared( - std::unordered_map())); + connector::hive::HiveConnectorFactory factory; + auto hiveConnector = factory.newConnector( + "test-hive", + std::make_shared( + std::unordered_map())); connector::registerConnector(hiveConnector); const auto driverExecutor = std::make_shared( diff --git a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp index 02f84d83a77ed..1b8c33ff04d92 100644 --- a/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp +++ b/presto-native-execution/presto_cpp/main/tests/TaskManagerTest.cpp @@ -195,11 +195,6 @@ class TaskManagerTest : public exec::test::OperatorTestBase, static void SetUpTestCase() { OperatorTestBase::SetUpTestCase(); filesystems::registerLocalFileSystem(); - if (!connector::hasConnectorFactory( - connector::hive::HiveConnectorFactory::kHiveConnectorName)) { - connector::registerConnectorFactory( - std::make_shared()); - } test::setupMutableSystemConfig(); SystemConfig::instance()->setValue( std::string(SystemConfig::kMemoryArbitratorKind), "SHARED"); @@ -233,13 +228,11 @@ class TaskManagerTest : public exec::test::OperatorTestBase, registerPrestoToVeloxConnector(std::make_unique( connector::hive::HiveConnectorFactory::kHiveConnectorName)); - auto hiveConnector = - connector::getConnectorFactory( - connector::hive::HiveConnectorFactory::kHiveConnectorName) - ->newConnector( - kHiveConnectorId, - std::make_shared( - std::unordered_map())); + connector::hive::HiveConnectorFactory factory; + auto hiveConnector = factory.newConnector( + kHiveConnectorId, + std::make_shared( + std::unordered_map())); connector::registerConnector(hiveConnector); rowType_ = ROW({"c0", "c1"}, {INTEGER(), VARCHAR()}); diff --git a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt index 8b168254334aa..ad23798a46eab 100644 --- a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt @@ -30,9 +30,6 @@ target_link_libraries( set_property(TARGET presto_types PROPERTY JOB_POOL_LINK presto_link_job_pool) -add_library(presto_function_metadata OBJECT FunctionMetadata.cpp) -target_link_libraries(presto_function_metadata velox_function_registry) - add_library(presto_velox_plan_conversion OBJECT VeloxPlanConversion.cpp) target_link_libraries(presto_velox_plan_conversion velox_type) diff --git a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt index 8f98f3f76686f..fd7b823c30444 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt @@ -100,24 +100,6 @@ target_link_libraries( GTest::gtest GTest::gtest_main) -add_executable(presto_function_metadata_test FunctionMetadataTest.cpp) - -add_test( - NAME presto_function_metadata_test - COMMAND presto_function_metadata_test - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - -target_link_libraries( - presto_function_metadata_test - presto_function_metadata - presto_protocol - presto_type_test_utils - velox_aggregates - velox_functions_prestosql - velox_window - GTest::gtest - GTest::gtest_main) - add_executable(presto_to_velox_query_plan_test PrestoToVeloxQueryPlanTest.cpp) add_test( diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index e287ef7e0cb20..02deaf46e9ed5 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -120,6 +120,7 @@ public static class HiveQueryRunnerBuilder private Map extraCoordinatorProperties = new HashMap<>(); private Map hiveProperties = new HashMap<>(); private Map tpcdsProperties = new HashMap<>(); + private Optional pluginDirectory = Optional.empty(); private String security; private boolean addStorageFormatToPath; private boolean coordinatorSidecarEnabled; @@ -171,6 +172,12 @@ public HiveQueryRunnerBuilder setRemoteFunctionServerUds(Optional remote return this; } + public HiveQueryRunnerBuilder setPluginDirectory(Optional pluginDirectory) + { + this.pluginDirectory = pluginDirectory; + return this; + } + public HiveQueryRunnerBuilder setFailOnNestedLoopJoin(boolean failOnNestedLoopJoin) { this.failOnNestedLoopJoin = failOnNestedLoopJoin; @@ -225,6 +232,10 @@ public HiveQueryRunnerBuilder setCoordinatorSidecarEnabled(boolean coordinatorSi public HiveQueryRunnerBuilder setBuiltInWorkerFunctionsEnabled(boolean builtInWorkerFunctionsEnabled) { this.builtInWorkerFunctionsEnabled = builtInWorkerFunctionsEnabled; + if (builtInWorkerFunctionsEnabled) { + this.extraProperties.put("built-in-sidecar-functions-enabled", "true"); + } + return this; } @@ -277,7 +288,7 @@ public QueryRunner build() Optional> externalWorkerLauncher = Optional.empty(); if (this.useExternalWorkerLauncher) { externalWorkerLauncher = getExternalWorkerLauncher("hive", serverBinary, cacheMaxSize, remoteFunctionServerUds, - failOnNestedLoopJoin, coordinatorSidecarEnabled, builtInWorkerFunctionsEnabled, enableRuntimeMetricsCollection, enableSsdCache, implicitCastCharNToVarchar); + pluginDirectory, failOnNestedLoopJoin, coordinatorSidecarEnabled, builtInWorkerFunctionsEnabled, enableRuntimeMetricsCollection, enableSsdCache, implicitCastCharNToVarchar); } return HiveQueryRunner.createQueryRunner( ImmutableList.of(), @@ -366,7 +377,7 @@ public QueryRunner build() Optional> externalWorkerLauncher = Optional.empty(); if (this.useExternalWorkerLauncher) { externalWorkerLauncher = getExternalWorkerLauncher("iceberg", serverBinary, cacheMaxSize, remoteFunctionServerUds, - false, false, false, false, false, false); + Optional.empty(), false, false, false, false, false, false); } return IcebergQueryRunner.builder() .setExtraProperties(extraProperties) @@ -460,6 +471,7 @@ public static Optional> getExternalWorkerLaunc String prestoServerPath, int cacheMaxSize, Optional remoteFunctionServerUds, + Optional pluginDirectory, Boolean failOnNestedLoopJoin, boolean isCoordinatorSidecarEnabled, boolean isBuiltInWorkerFunctionsEnabled, @@ -513,6 +525,10 @@ else if (isBuiltInWorkerFunctionsEnabled) { "remote-function-server.signature.files.directory.path=%s%n", configProperties, REMOTE_FUNCTION_CATALOG_NAME, remoteFunctionServerUds.get(), jsonSignaturesPath); } + if (pluginDirectory.isPresent()) { + configProperties = format("%s%n" + "plugin.dir=%s%n", configProperties, pluginDirectory.get()); + } + if (failOnNestedLoopJoin) { configProperties = format("%s%n" + "velox-plan-validator-fail-on-nested-loop-join=true%n", configProperties); } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java index 4e5836a56bc97..59263a302113c 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestPrestoNativeBuiltInFunctions.java @@ -214,5 +214,8 @@ public void testUdfQueries() assertPlan("SELECT test_agg_function(cast(orderkey as integer), cast(orderkey as integer), cast(orderkey as bigint)) FROM tpch.tiny.orders", anyTree(any())); assertPlan("SELECT test_agg_function(5, cast(orderkey as smallint), orderkey) FROM tpch.tiny.orders", anyTree(any())); + + assertQuerySucceeds("SELECT ARRAY_SORT( ARRAY[ ARRAY['a', 'b', 'c'] ], (x, y) -> IF( COALESCE( x[1], '' ) > COALESCE( y[1], '' ), 1, IF( COALESCE( x[1], '' ) < COALESCE( y[1], '' ), -1, 0 ) ) )"); + assertQuerySucceeds("SELECT ARRAY_SORT( ARRAY[ ARRAY['a', 'b', 'c'] ], (x, y) -> IF( COALESCE( x[1], '' ) > COALESCE( y[1], '' ), cast(1 as bigint), IF( COALESCE( x[1], '' ) < COALESCE( y[1], '' ), cast(-1 as bigint), cast(0 as bigint) ) ) )"); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/spark/PrestoSparkNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/spark/PrestoSparkNativeQueryRunnerUtils.java index 3a386faece33a..4cb8fe15a1f52 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/spark/PrestoSparkNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/spark/PrestoSparkNativeQueryRunnerUtils.java @@ -18,6 +18,7 @@ import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; import com.facebook.presto.spark.execution.nativeprocess.NativeExecutionModule; +import com.facebook.presto.spark.execution.property.NativeExecutionConfigModule; import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.testing.QueryRunner; import com.google.common.collect.ImmutableList; @@ -100,27 +101,37 @@ public static Map getNativeExecutionSparkConfigs() public static PrestoSparkQueryRunner createHiveRunner() { - PrestoSparkQueryRunner queryRunner = createRunner("hive", new NativeExecutionModule( - Optional.of(ImmutableMap.of("hive", ImmutableMap.of("connector.name", "hive"))))); + PrestoSparkQueryRunner queryRunner = createRunner("hive", new NativeExecutionModule(), + new NativeExecutionConfigModule( + ImmutableMap.of(), + ImmutableMap.of( + "hive", + ImmutableMap.of("connector.name", "hive")))); PrestoNativeQueryRunnerUtils.setupJsonFunctionNamespaceManager(queryRunner, "external_functions.json", "json"); return queryRunner; } /** - * Similar to createHiveRunner(), but also add additional specified catalogs and their - * corresponding properties. This method exists because unlike Java, native execution does not - * allow adding catalogs in the tests after process starts. So any tests that need additional - * catalogs need to add them upon runner creation. + * Similar to createHiveRunner(), but also add additional specified system properties, catalogs + * and their corresponding properties. This method exists because unlike Java, native execution + * does not allow adding catalogs in the tests after process starts. So any tests that need + * additional catalogs need to add them upon runner creation. */ - public static PrestoSparkQueryRunner createHiveRunner(Map> additionalCatalogs) + public static PrestoSparkQueryRunner createHiveRunner( + Map additionalSystemConfigs, + Map> additionalCatalogs) { // Add connectors on the native side to make them available during execution. ImmutableMap.Builder> catalogBuilder = ImmutableMap.builder(); catalogBuilder.put("hive", ImmutableMap.of("connector.name", "hive")) .putAll(additionalCatalogs); - PrestoSparkQueryRunner queryRunner = createRunner("hive", new NativeExecutionModule( - Optional.of(catalogBuilder.build()))); + PrestoSparkQueryRunner queryRunner = createRunner( + "hive", + new NativeExecutionModule(), + new NativeExecutionConfigModule( + additionalSystemConfigs, + catalogBuilder.build())); // Add connectors on the Java side to make them visible during planning. additionalCatalogs.entrySet().stream().forEach(entry -> { @@ -140,7 +151,9 @@ public static PrestoSparkQueryRunner createHiveRunner(Map baseDir, Map additionalConfigProperties, Map additionalSparkProperties, ImmutableList nativeModules) diff --git a/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeGeneralQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeGeneralQueries.java index 33f7d76a61b57..66ac7035fd40c 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeGeneralQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeGeneralQueries.java @@ -35,6 +35,7 @@ protected QueryRunner createQueryRunner() { // Adding additional catalog needed in some tests in the suite. QueryRunner queryRunner = PrestoSparkNativeQueryRunnerUtils.createHiveRunner( + ImmutableMap.of(), ImmutableMap.of("hivecached", ImmutableMap.of("connector.name", "hive", "hive.storage-format", "DWRF", diff --git a/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleJsonQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleJsonQueries.java new file mode 100644 index 0000000000000..2e29cc7469f20 --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleJsonQueries.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spark; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.ExpectedQueryRunner; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createBucketedCustomer; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createBucketedLineitemAndOrders; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createCustomer; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createEmptyTable; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createLineitem; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createNation; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrders; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrdersEx; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrdersHll; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createPart; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createPartSupp; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createPartitionedNation; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createPrestoBenchTables; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createRegion; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createSupplier; + +public class TestPrestoSparkNativeSimpleJsonQueries + extends AbstractTestQueryFramework +{ + @Override + protected void createTables() + { + QueryRunner queryRunner = (QueryRunner) getExpectedQueryRunner(); + createLineitem(queryRunner); + createOrders(queryRunner); + createOrdersHll(queryRunner); + createOrdersEx(queryRunner); + createNation(queryRunner); + createRegion(queryRunner); + createPartitionedNation(queryRunner); + createBucketedCustomer(queryRunner); + createCustomer(queryRunner); + createPart(queryRunner); + createPartSupp(queryRunner); + createRegion(queryRunner); + createSupplier(queryRunner); + createEmptyTable(queryRunner); + createPrestoBenchTables(queryRunner); + createBucketedLineitemAndOrders(queryRunner); + } + + @Override + protected QueryRunner createQueryRunner() + { + return PrestoSparkNativeQueryRunnerUtils.createHiveRunner( + ImmutableMap.of("presto.default-namespace", "json.test_schema"), ImmutableMap.of()); + } + + @Override + protected ExpectedQueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoSparkNativeQueryRunnerUtils.createJavaQueryRunner(); + } + + /** + * Test native execution of cpp functions declared via a json file. `eq()` Scalar function & + * `sum()` Aggregate function are defined in `src/test/resources/external_functions.json` + */ + @Test + public void testJsonFileBasedFunction() + { + assertQuery("SELECT json.test_schema.eq(1, linenumber) FROM lineitem", + "SELECT 1 = linenumber FROM lineitem"); + assertQuery("SELECT json.test_schema.sum(linenumber) FROM lineitem", + "SELECT sum(linenumber) FROM lineitem"); + + // Test functions with complex types (array, map, and row). + assertQuery("SELECT json.test_schema.array_constructor(linenumber) FROM lineitem", + "SELECT array_constructor(linenumber) FROM lineitem"); + + assertQuery( + "SELECT json.test_schema.map(json.test_schema.array_constructor(linenumber), json.test_schema.array_constructor(linenumber)) FROM lineitem", + "SELECT map(array_constructor(linenumber), array_constructor(linenumber)) FROM lineitem"); + assertQuery( + "SELECT json.test_schema.map_entries(json.test_schema.map(json.test_schema.array_constructor(linenumber), json.test_schema.array_constructor(linenumber))) FROM lineitem", + "SELECT map_entries(map(array_constructor(linenumber), array_constructor(linenumber))) FROM lineitem"); + } + + /** + * Test aggregation using companion functions with partial and final aggregation steps handled + * by separate queries. The first query computes partial aggregation states and stores them in + * the avg_partial_states table. Subsequent queries read from avg_partial_states and aggregate + * the states to the final result. + */ + @Test + public void testAggregationCompanionFunction() + { + Session session = Session.builder(getSession()) + .setCatalogSessionProperty("hive", "collect_column_statistics_on_write", "false") + .setCatalogSessionProperty("hive", "orc_compression_codec", "ZSTD") + .build(); + try { + getQueryRunner().execute(session, + "CREATE TABLE avg_partial_states AS ( " + + "SELECT orderpriority, cast(json.test_schema.avg_partial(shippriority) as ROW(sum DOUBLE, count BIGINT)) as states " + + "FROM orders " + + "GROUP BY orderstatus, orderpriority " + + ")"); + + // Test group-by aggregation. + assertQuery( + "SELECT orderpriority, json.test_schema.avg_merge_extract_double(states) FROM avg_partial_states GROUP BY orderpriority", + "SELECT orderpriority, avg(shippriority) FROM orders GROUP BY orderpriority"); + assertQuery( + "SELECT orderpriority, json.test_schema.avg_extract_double(json.test_schema.avg_merge(states)) FROM avg_partial_states GROUP BY orderpriority", + "SELECT orderpriority, avg(shippriority) FROM orders GROUP BY orderpriority"); + + // Test global aggregation. + assertQuery( + "SELECT json.test_schema.avg_merge_extract_double(states) FROM avg_partial_states", + "SELECT avg(shippriority) FROM orders"); + assertQuery( + "SELECT json.test_schema.avg_extract_double(json.test_schema.avg_merge(states)) FROM avg_partial_states", + "SELECT avg(shippriority) FROM orders"); + } + finally { + getQueryRunner().execute("DROP TABLE IF EXISTS avg_partial_states"); + } + } +} diff --git a/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleQueries.java b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleQueries.java index 19c9a8a5679f7..a61d1fb8e4535 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleQueries.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeSimpleQueries.java @@ -17,7 +17,6 @@ import com.facebook.presto.testing.ExpectedQueryRunner; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestQueryFramework; -import org.testng.annotations.Ignore; import org.testng.annotations.Test; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; @@ -104,66 +103,6 @@ public void testFailures() assertQueryFails("SELECT orderkey / 0 FROM orders", "(?s).*division by zero.*"); } - /** - * Test native execution of cpp functions declared via a json file. - * `eq()` Scalar function & `sum()` Aggregate function are defined in `src/test/resources/external_functions.json` - */ - @Test - @Ignore("json schema based external function registraion is failing. Fix it and re-enable this test") - public void testJsonFileBasedFunction() - { - assertQuery("SELECT json.test_schema.eq(1, linenumber) FROM lineitem", "SELECT 1 = linenumber FROM lineitem"); - assertQuery("SELECT json.test_schema.sum(linenumber) FROM lineitem", "SELECT sum(linenumber) FROM lineitem"); - - // Test functions with complex types (array, map, and row). - assertQuery("SELECT json.test_schema.array_constructor(linenumber) FROM lineitem", "SELECT array_constructor(linenumber) FROM lineitem"); - - assertQuery("SELECT json.test_schema.map(json.test_schema.array_constructor(linenumber), json.test_schema.array_constructor(linenumber)) FROM lineitem", "SELECT map(array_constructor(linenumber), array_constructor(linenumber)) FROM lineitem"); - assertQuery("SELECT json.test_schema.map_entries(json.test_schema.map(json.test_schema.array_constructor(linenumber), json.test_schema.array_constructor(linenumber))) FROM lineitem", "SELECT map_entries(map(array_constructor(linenumber), array_constructor(linenumber))) FROM lineitem"); - } - - /** - * Test aggregation using companion functions with partial and final aggregation steps handled by separate queries. - * The first query computes partial aggregation states and stores them in the avg_partial_states table. - * Subsequent queries read from avg_partial_states and aggregate the states to the final result. - */ - @Test - @Ignore("json schema based external function registraion is failing. Fix it and re-enable this test") - public void testAggregationCompanionFunction() - { - Session session = Session.builder(getSession()) - .setCatalogSessionProperty("hive", "collect_column_statistics_on_write", "false") - .setCatalogSessionProperty("hive", "orc_compression_codec", "ZSTD") - .build(); - try { - getQueryRunner().execute(session, - "CREATE TABLE avg_partial_states AS ( " - + "SELECT orderpriority, cast(json.test_schema.avg_partial(shippriority) as ROW(sum DOUBLE, count BIGINT)) as states " - + "FROM orders " - + "GROUP BY orderstatus, orderpriority " - + ")"); - - // Test group-by aggregation. - assertQuery( - "SELECT orderpriority, json.test_schema.avg_merge_extract_double(states) FROM avg_partial_states GROUP BY orderpriority", - "SELECT orderpriority, avg(shippriority) FROM orders GROUP BY orderpriority"); - assertQuery( - "SELECT orderpriority, json.test_schema.avg_extract_double(json.test_schema.avg_merge(states)) FROM avg_partial_states GROUP BY orderpriority", - "SELECT orderpriority, avg(shippriority) FROM orders GROUP BY orderpriority"); - - // Test global aggregation. - assertQuery( - "SELECT json.test_schema.avg_merge_extract_double(states) FROM avg_partial_states", - "SELECT avg(shippriority) FROM orders"); - assertQuery( - "SELECT json.test_schema.avg_extract_double(json.test_schema.avg_merge(states)) FROM avg_partial_states", - "SELECT avg(shippriority) FROM orders"); - } - finally { - getQueryRunner().execute("DROP TABLE IF EXISTS avg_partial_states"); - } - } - @Test public void testRetryOnOutOfMemoryBroadcastJoin() { diff --git a/presto-native-execution/velox b/presto-native-execution/velox index 71a310307a734..6dce9b24e2052 160000 --- a/presto-native-execution/velox +++ b/presto-native-execution/velox @@ -1 +1 @@ -Subproject commit 71a310307a734e170206163efb504afb7e6d834c +Subproject commit 6dce9b24e2052463bcf43770652911af14077787 diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index d04424440469a..2e6d9f363b9d2 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-native-sidecar-plugin diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java index dd4122a04cc6d..a3bd8d4a84725 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java @@ -59,6 +59,7 @@ import static com.facebook.presto.common.type.StandardTypes.ROW; import static com.facebook.presto.common.type.StandardTypes.SMALLINT; import static com.facebook.presto.common.type.StandardTypes.TDIGEST; +import static com.facebook.presto.common.type.StandardTypes.TIME; import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP; import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.StandardTypes.TINYINT; @@ -84,6 +85,7 @@ public class NativeTypeManager TINYINT, BOOLEAN, DATE, + TIME, INTEGER, DOUBLE, SMALLINT, diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java index d3f584222c3b9..4dc3b460bf13a 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java @@ -219,7 +219,7 @@ public void testGeneralQueries() "date_trunc('hour', from_unixtime(orderkey, '-09:30')), date_trunc('minute', from_unixtime(orderkey, '+05:30')), " + "date_trunc('second', from_unixtime(orderkey, '+00:00')) FROM orders"); assertQuery("SELECT mod(orderkey, linenumber) FROM lineitem"); - assertQueryFails("SELECT IF(true, 0/0, 1)", "[\\s\\S]*/ by zero native.default.fail[\\s\\S]*"); + assertQueryFails("SELECT IF(true, 0/0, 1)", "/ by zero", true); assertQuery("select CASE WHEN true THEN 'Yes' ELSE 'No' END"); } diff --git a/presto-native-tests/CMakeLists.txt b/presto-native-tests/CMakeLists.txt new file mode 100644 index 0000000000000..6cd83f84951cf --- /dev/null +++ b/presto-native-tests/CMakeLists.txt @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.10) + +# set the project name +project(PrestoCpp) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) +message("Appending CMAKE_CXX_FLAGS with ${SCRIPT_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}") + +# Known warnings that are benign can be disabled. +set(DISABLED_WARNINGS + "-Wno-nullability-completeness -Wno-deprecated-declarations") + +# Important warnings that must be explicitly enabled. +set(ENABLE_WARNINGS "-Wreorder") + +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} ${DISABLED_WARNINGS} ${ENABLE_WARNINGS}") + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if(DEFINED ENV{INSTALL_PREFIX}) + message(STATUS "Dependency install directory set to: $ENV{INSTALL_PREFIX}") + list(APPEND CMAKE_PREFIX_PATH "$ENV{INSTALL_PREFIX}") +endif() + +find_package(gflags REQUIRED) + +find_library(GLOG glog) + +find_package(fmt REQUIRED) + +if(NOT TARGET gflags::gflags) + # This is a bit convoluted, but we want to be able to use gflags::gflags as a + # target even when velox is built as a subproject which uses + # `find_package(gflags)` which does not create a globally imported target that + # we can ALIAS. + add_library(gflags_gflags INTERFACE) + target_link_libraries(gflags_gflags INTERFACE gflags) + add_library(gflags::gflags ALIAS gflags_gflags) +endif() +include_directories(.) + +include_directories(${CMAKE_BINARY_DIR}) + +# Adding this down here prevents warnings in dependencies from stopping the +# build. +if("${TREAT_WARNINGS_AS_ERRORS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") +endif() + +if(DEFINED ENV{INSTALL_PREFIX}) + # Allow installed package headers to be picked up before brew/system package + # headers. We set this after Velox since Velox handles INSTALL_PREFIX its own + # way. + include_directories(BEFORE "$ENV{INSTALL_PREFIX}/include") +endif() + +add_subdirectory(presto_cpp) diff --git a/presto-native-tests/Makefile b/presto-native-tests/Makefile new file mode 100644 index 0000000000000..4147efc6d6070 --- /dev/null +++ b/presto-native-tests/Makefile @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +.PHONY: build clean debug release submodules + +BUILD_BASE_DIR=_build +BUILD_DIR=release +BUILD_TYPE=Release +TREAT_WARNINGS_AS_ERRORS ?= 1 +ENABLE_WALL ?= 1 +NUM_THREADS ?= $(shell getconf _NPROCESSORS_CONF 2>/dev/null || echo 1) +CMAKE_PREFIX_PATH ?= "/usr/local" + +EXTRA_CMAKE_FLAGS ?= "" + +CMAKE_FLAGS := -DTREAT_WARNINGS_AS_ERRORS=${TREAT_WARNINGS_AS_ERRORS} +CMAKE_FLAGS += -DENABLE_ALL_WARNINGS=${ENABLE_WALL} +CMAKE_FLAGS += -DCMAKE_PREFIX_PATH=$(CMAKE_PREFIX_PATH) +CMAKE_FLAGS += -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) + +SHELL := /bin/bash + +# Use Ninja if available. If Ninja is used, pass through parallelism control flags. +USE_NINJA ?= 1 +ifeq ($(USE_NINJA), 1) +ifneq ($(shell which ninja), ) +CMAKE_FLAGS += -GNinja -DMAX_LINK_JOBS=$(MAX_LINK_JOBS) -DMAX_HIGH_MEM_JOBS=$(MAX_HIGH_MEM_JOBS) +endif +endif + +ifndef USE_CCACHE +ifneq ($(shell which ccache), ) +CMAKE_FLAGS += -DCMAKE_CXX_COMPILER_LAUNCHER=ccache +endif +endif + +all: release #: Build the release version + +clean: #: Delete all build artifacts + rm -rf $(BUILD_BASE_DIR) + +cmake: submodules #: Use CMake to create a Makefile build system + cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) $(EXTRA_CMAKE_FLAGS) + +build: #: Build the software based in BUILD_DIR and BUILD_TYPE variables + cmake --build $(BUILD_BASE_DIR)/$(BUILD_DIR) -j $(NUM_THREADS) + +debug: #: Build with debugging symbols + $(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=Debug + $(MAKE) build BUILD_DIR=debug + +release: #: Build the release version + $(MAKE) cmake BUILD_DIR=release BUILD_TYPE=Release && \ + $(MAKE) build BUILD_DIR=release + +help: #: Show the help messages + @cat $(firstword $(MAKEFILE_LIST)) | \ + awk '/^[-a-z]+:/' | \ + awk -F: '{ printf("%-20s %s\n", $$1, $$NF) }' diff --git a/presto-native-tests/pom.xml b/presto-native-tests/pom.xml index 7e23670117b1d..fb100f8995ca9 100644 --- a/presto-native-tests/pom.xml +++ b/presto-native-tests/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-native-tests @@ -187,6 +187,12 @@ test + + com.facebook.presto + presto-spi + test + + com.facebook.airlift units @@ -199,6 +205,13 @@ ${project.version} test + + + com.facebook.presto + presto-sql-invoked-functions-plugin + ${project.version} + test + diff --git a/presto-native-tests/presto_cpp/CMakeLists.txt b/presto-native-tests/presto_cpp/CMakeLists.txt new file mode 100644 index 0000000000000..6a4085713e395 --- /dev/null +++ b/presto-native-tests/presto_cpp/CMakeLists.txt @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_subdirectory(tests) diff --git a/presto-native-tests/presto_cpp/tests/CMakeLists.txt b/presto-native-tests/presto_cpp/tests/CMakeLists.txt new file mode 100644 index 0000000000000..a056e86ef716b --- /dev/null +++ b/presto-native-tests/presto_cpp/tests/CMakeLists.txt @@ -0,0 +1,12 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_subdirectory(custom_functions) diff --git a/presto-native-tests/presto_cpp/tests/custom_functions/CMakeLists.txt b/presto-native-tests/presto_cpp/tests/custom_functions/CMakeLists.txt new file mode 100644 index 0000000000000..a2d62155f91d5 --- /dev/null +++ b/presto-native-tests/presto_cpp/tests/custom_functions/CMakeLists.txt @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set(PRESTO_AND_VELOX_INCLUDE_DIRS + ${CMAKE_SOURCE_DIR}/../presto-native-execution + ${CMAKE_SOURCE_DIR}/../presto-native-execution/velox +) + +add_library(presto_cpp_custom_functions_test SHARED CustomFunctions.cpp) + +set(CMAKE_DYLIB_TEST_LINK_LIBRARIES fmt::fmt + xsimd) + +target_include_directories(presto_cpp_custom_functions_test + PRIVATE + ${PRESTO_AND_VELOX_INCLUDE_DIRS} + presto_dynamic_function_registrar +) + +target_link_libraries(presto_cpp_custom_functions_test + PRIVATE + ${CMAKE_DYLIB_TEST_LINK_LIBRARIES}) + +if(APPLE) + set(COMMON_LIBRARY_LINK_OPTIONS "-Wl,-undefined,dynamic_lookup") +else() + set(COMMON_LIBRARY_LINK_OPTIONS "-Wl,--exclude-libs,ALL") +endif() + +target_link_options(presto_cpp_custom_functions_test PRIVATE + ${COMMON_LIBRARY_LINK_OPTIONS}) diff --git a/presto-native-tests/presto_cpp/tests/custom_functions/CustomFunctions.cpp b/presto-native-tests/presto_cpp/tests/custom_functions/CustomFunctions.cpp new file mode 100644 index 0000000000000..cf2b5f252fd03 --- /dev/null +++ b/presto-native-tests/presto_cpp/tests/custom_functions/CustomFunctions.cpp @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + #include + #include "presto_cpp/main/functions/dynamic_registry/DynamicFunctionRegistrar.h" + #include "velox/expression/SimpleFunctionRegistry.h" + // This file defines a function that will be dynamically linked and + // registered. This function implements a custom function in the definition, + // and this library (.so) needs to provide a `void registerExtensions()` + // C function in the top-level namespace. + // + // (note the extern "C" directive to prevent the compiler from mangling the + // symbol name). + +namespace custom::functionRegistry { + template + struct CustomAdd { + VELOX_DEFINE_FUNCTION_TYPES(T); + FOLLY_ALWAYS_INLINE bool call( + out_type& result, + const arg_type& x1, + const arg_type& x2) { + result = x1 + x2; + return true; + } + }; + + template + struct SumArray { + VELOX_DEFINE_FUNCTION_TYPES(T); + FOLLY_ALWAYS_INLINE bool call( + out_type& result, + const arg_type>& arr) { + int64_t sum = 0; + for (auto val : arr) { + if (val.has_value()) { + sum += val.value(); + } + } + result = sum; + return true; + } + }; + + template + struct MapSize { + VELOX_DEFINE_FUNCTION_TYPES(T); + FOLLY_ALWAYS_INLINE bool call( + out_type& result, + const arg_type>& m) { + result = m.size(); + return true; + } + }; + + template + struct SumNestedArrayElements { + VELOX_DEFINE_FUNCTION_TYPES(T); + FOLLY_ALWAYS_INLINE bool call( + out_type& result, + const arg_type>>& arr) { + int64_t sum = 0; + for (auto innerOpt : arr) { + if (innerOpt.has_value()) { + for (auto val : innerOpt.value()) { + if (val.has_value()) { + sum += val.value(); + } + } + } + } + result = sum; + return true; + } +}; +} // namespace custom::functionRegistry + +extern "C" { + void registerExtensions() { + facebook::presto::registerPrestoFunction< + custom::functionRegistry::CustomAdd, + int64_t, + int64_t, + int64_t>("dynamic_custom_add"); + + facebook::presto::registerPrestoFunction< + custom::functionRegistry::SumArray, + int64_t, + facebook::velox::Array>("sum_array"); + + facebook::presto::registerPrestoFunction< + custom::functionRegistry::MapSize, + int64_t, + facebook::velox::Map>("map_size"); + + facebook::presto::registerPrestoFunction< + custom::functionRegistry::SumNestedArrayElements, + int64_t, + facebook::velox::Array>>("sum_nested_array_elements"); + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java index d668c65f098ea..75284bfae9d89 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java @@ -36,7 +36,7 @@ public void init(String storageFormat, boolean charNToVarcharImplicitCast, boole if (sidecarEnabled) { charTypeUnsupportedError = ".*Unknown type: char.*"; timeTypeUnsupportedError = ".*Unknown type: time.*"; - approxDistinctUnsupportedSignatureError = ".*Unexpected parameters \\(timestamp with time zone.*\\) for function.*"; + approxDistinctUnsupportedSignatureError = ".*Unexpected parameters \\(time.*\\) for function.*"; } else { charTypeUnsupportedError = "Failed to parse type.*char"; @@ -75,12 +75,14 @@ public void testApproximateCountDistinct() approxDistinctUnsupportedSignatureError, true); // test time - assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", timeTypeUnsupportedError, true); - assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", timeTypeUnsupportedError, true); + // TODO: re-enable the timestamp related failures later. + //assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", approxDistinctUnsupportedSignatureError, true); + //assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", approxDistinctUnsupportedSignatureError, true); // test time with time zone - assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true); - assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true); + // TODO: re-enable the timestamp related failures later. + //assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true); + //assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true); // test short decimal assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0))) FROM orders", "SELECT 990"); diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/NativeTestsUtils.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/NativeTestsUtils.java index 12d4cabed0220..87382ba8ba445 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/NativeTestsUtils.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/NativeTestsUtils.java @@ -15,6 +15,13 @@ import com.facebook.presto.nativeworker.NativeQueryRunnerUtils; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableList; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Optional; import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder; import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder; @@ -33,6 +40,7 @@ public static QueryRunner createNativeQueryRunner(String storageFormat, boolean .setUseThrift(true) .setImplicitCastCharNToVarchar(charNToVarcharImplicitCast) .setCoordinatorSidecarEnabled(sidecarEnabled) + .setPluginDirectory(sidecarEnabled ? Optional.of(getCustomFunctionsPluginDirectory().toString()) : Optional.empty()) .build(); if (sidecarEnabled) { setupNativeSidecarPlugin(queryRunner); @@ -72,4 +80,35 @@ public static boolean getCharNToVarcharImplicitCastForTest(boolean sidecarEnable { return sidecarEnabled ? false : charNToVarcharImplicitCast; } + + public static Path getCustomFunctionsPluginDirectory() + throws Exception + { + Path prestoRoot = findPrestoRoot(); + + // All candidate paths relative to prestoRoot + List candidates = ImmutableList.of( + prestoRoot.resolve("presto-native-tests/_build/debug/presto_cpp/tests/custom_functions"), + prestoRoot.resolve("presto-native-tests/_build/release/presto_cpp/tests/custom_functions"), + prestoRoot.resolve("presto-native-execution/_build/debug/presto_cpp/main/functions/dynamic_registry/tests/custom_functions"), + prestoRoot.resolve("presto-native-execution/_build/release/presto_cpp/main/functions/dynamic_registry/tests/custom_functions")); + + return candidates.stream() + .filter(Files::exists) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Could not locate custom functions directory")); + } + + private static Path findPrestoRoot() + { + Path dir = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + while (dir != null) { + if (Files.exists(dir.resolve("presto-native-tests")) || + Files.exists(dir.resolve("presto-native-execution"))) { + return dir; + } + dir = dir.getParent(); + } + throw new IllegalStateException("Could not locate presto root directory."); + } } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestCustomFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestCustomFunctions.java new file mode 100644 index 0000000000000..0ad7b7537f75a --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestCustomFunctions.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.Test; +import org.testng.log4testng.Logger; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static com.facebook.presto.nativetests.NativeTestsUtils.getCustomFunctionsPluginDirectory; +import static java.lang.Boolean.parseBoolean; + +public class TestCustomFunctions + extends AbstractTestQueryFramework +{ + private static final Logger logger = Logger.getLogger(TestCustomFunctions.class); + private String storageFormat; + private boolean sidecarEnabled; + + @BeforeSuite + public void buildNativeLibrary() + throws IOException, InterruptedException + { + // If we built with examples on, do not try to build. + // This usually happens during the github pipeline. + try { + getCustomFunctionsPluginDirectory(); + } + catch (Exception e) { + Path prestoRoot = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + while (prestoRoot != null && !Files.exists(prestoRoot.resolve("presto-native-tests"))) { + prestoRoot = prestoRoot.getParent(); + } + if (prestoRoot == null) { + throw new IllegalStateException("Could not locate presto root directory."); + } + String workingDir = prestoRoot + .resolve("presto-native-tests").toAbsolutePath().toString(); + ProcessBuilder builder = new ProcessBuilder("make", "release"); + builder.directory(new File(workingDir)); + builder.redirectErrorStream(true); + Process process = builder.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + System.out.println("[BUILD OUTPUT] " + line); + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IllegalStateException("C++ build failed with exit code " + exitCode); + } + } + } + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = System.getProperty("storageFormat", "PARQUET"); + sidecarEnabled = parseBoolean(System.getProperty("sidecarEnabled", "true")); + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return NativeTestsUtils.createNativeQueryRunner(storageFormat, sidecarEnabled); + } + + @Override + protected void createTables() + { + NativeTestsUtils.createTables(storageFormat); + } + + /// Sidecar is needed to support custom functions in Presto C++. + @Test + public void testCustomFunctions() + { + if (sidecarEnabled) { + logger.info("Sidecar Enabled"); + // Scalar test + assertQuery( + "SELECT dynamic_custom_add(CAST(pow(2, 3) AS BIGINT), CAST(sqrt(25) AS BIGINT))", + "VALUES 13"); + + // Array test + assertQuery( + "SELECT sum_array(ARRAY[1,2,3])", + "VALUES 6"); + + // Nested array test + assertQuery( + "SELECT sum_nested_array_elements(ARRAY[ARRAY[1,2], ARRAY[3,4,5]])", + "VALUES 15"); + + // Map test + assertQuery( + "SELECT map_size(MAP(ARRAY[1,2], ARRAY[10,20]))", + "VALUES 2"); + } + else { + logger.info("Sidecar Disabled"); + // Scalar test + assertQueryFails( + "SELECT dynamic_custom_add(10, 5)", + "line 1:8: Function dynamic_custom_add not registered"); + + // Array test + assertQueryFails( + "SELECT sum_array(ARRAY[1,2,3])", + "line 1:8: Function sum_array not registered"); + + // Nested array test + assertQueryFails( + "SELECT sum_nested_array_elements(ARRAY[ARRAY[1,2], ARRAY[3,4,5]])", + "line 1:8: Function sum_nested_array_elements not registered"); + + // Map test + assertQueryFails( + "SELECT map_size(MAP(ARRAY[1,2], ARRAY[10,20]))", + "line 1:8: Function map_size not registered"); + } + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java index b54a9711940b9..1d7681a0c53ae 100644 --- a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestDistributedEngineOnlyQueries.java @@ -15,17 +15,13 @@ import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.AbstractTestEngineOnlyQueries; -import org.intellij.lang.annotations.Language; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.time.LocalDate; import java.time.LocalTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import static com.google.common.base.Preconditions.checkState; import static java.lang.Boolean.parseBoolean; +import static org.testng.Assert.assertEquals; public class TestDistributedEngineOnlyQueries extends AbstractTestEngineOnlyQueries @@ -68,71 +64,18 @@ protected void createTables() @Test public void testTimeLiterals() { - assertQueryFails("SELECT TIME '3:04:05'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '3:04:05.123'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '3:04:05'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '0:04:05'", timeTypeUnsupportedError); + assertEquals(computeScalar("SELECT TIME '3:04:05'"), LocalTime.of(3, 4, 5, 0)); + assertEquals(computeScalar("SELECT TIME '3:04:05.123'"), LocalTime.of(3, 4, 5, 123_000_000)); + assertQuery("SELECT TIME '3:04:05'"); + assertQuery("SELECT TIME '0:04:05'"); // TODO #7122 assertQueryFails(chicago, "SELECT TIME '3:04:05'", timeTypeUnsupportedError); // TODO #7122 assertQueryFails(kathmandu, "SELECT TIME '3:04:05'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '01:02:03.400 Z'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '01:02:03.400 UTC'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '3:04:05 +06:00'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '3:04:05 +0507'", timeTypeUnsupportedError); - assertQueryFails("SELECT TIME '3:04:05 +03'", timeTypeUnsupportedError); - } - - /// TIME datatype is not supported in Prestissimo. See issue: https://github.com/prestodb/presto/issues/18844. - @Override - @Test - public void testLocallyUnrepresentableTimeLiterals() - { - LocalTime localTimeThatDidNotOccurOn20120401 = LocalTime.of(2, 10); - checkState(ZoneId.systemDefault().getRules().getValidOffsets(localTimeThatDidNotOccurOn20120401.atDate(LocalDate.of(2012, 4, 1))).isEmpty(), "This test assumes certain JVM time zone"); - @Language("SQL") String sql = DateTimeFormatter.ofPattern("'SELECT TIME '''HH:mm:ss''").format(localTimeThatDidNotOccurOn20120401); - assertQueryFails(sql, timeTypeUnsupportedError); - } - - // todo: turn on these test cases when the sql invoked functions are extracted into a plugin module. - @Override - @Test(enabled = false) - public void testArraySplitIntoChunks() - { - } - - @Override - @Test(enabled = false) - public void testCrossJoinWithArrayNotContainsCondition() - { - } - - @Override - @Test(enabled = false) - public void testSamplingJoinChain() - { - } - - @Override - @Test(enabled = false) - public void testKeyBasedSampling() - { - } - - @Override - @Test(enabled = false) - public void testDefaultSamplingPercent() - { - } - - @Override - @Test(enabled = false) - public void testLeftJoinWithArrayContainsCondition() - { - } - - @Override - @Test(enabled = false) - public void testTry() - { + // TODO: re-enable the timestamp related test failures later. + //assertQueryFails("SELECT TIME '01:02:03.400 Z'", timeTypeUnsupportedError); + //assertQueryFails("SELECT TIME '01:02:03.400 UTC'", timeTypeUnsupportedError); + //assertQueryFails("SELECT TIME '3:04:05 +06:00'", timeTypeUnsupportedError); + //assertQueryFails("SELECT TIME '3:04:05 +0507'", timeTypeUnsupportedError); + //assertQueryFails("SELECT TIME '3:04:05 +03'", timeTypeUnsupportedError); } } diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestSqlInvokedFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestSqlInvokedFunctions.java new file mode 100644 index 0000000000000..416c5b86e7e06 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestSqlInvokedFunctions.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestSqlInvokedFunctions; +import com.google.common.collect.ImmutableMap; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder; +import static com.facebook.presto.sidecar.NativeSidecarPluginQueryRunnerUtils.setupNativeSidecarPlugin; +import static java.lang.Boolean.parseBoolean; + +public class TestSqlInvokedFunctions + extends AbstractTestSqlInvokedFunctions +{ + private String storageFormat; + private boolean sidecarEnabled; + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = System.getProperty("storageFormat", "PARQUET"); + sidecarEnabled = parseBoolean(System.getProperty("sidecarEnabled", "true")); + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + QueryRunner queryRunner = nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .setCoordinatorSidecarEnabled(sidecarEnabled) + .setExtraProperties(ImmutableMap.of("inline-sql-functions", "true")) + .build(); + if (sidecarEnabled) { + setupNativeSidecarPlugin(queryRunner); + } + else { + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + } + return queryRunner; + } + + @Override + protected void createTables() + { + NativeTestsUtils.createTables(storageFormat); + } + + /// Default implementation of Sql invoked function `map_keys_by_top_n_values` is not compatible with Velox. Sidecar + /// should be enabled to use the native compatible Sql invoked function from `native-sql-invoked-functions-plugin`. + @Override + @Test + public void testTry() + { + if (sidecarEnabled) { + super.testTry(); + } + else { + @Language("RegExp") String arraySortLambdaUnsupported = ".*array_sort with comparator lambda that cannot be rewritten into a transform is not supported.*"; + + assertQueryFails("SELECT\n" + + " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + + "FROM (\n" + + " VALUES\n" + + " MAP(\n" + + " ARRAY[1, 2], ARRAY[\n" + + " ARRAY[1, null],\n" + + " ARRAY[1, null]\n" + + " ]\n" + + " )\n" + + ") t(c0)", arraySortLambdaUnsupported); + + assertQueryFails("SELECT\n" + + " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + + "FROM (\n" + + " VALUES\n" + + " MAP(\n" + + " ARRAY[1, 2], ARRAY[\n" + + " ARRAY[null, null],\n" + + " ARRAY[1, 2]\n" + + " ]\n" + + " )\n" + + ") t(c0)", arraySortLambdaUnsupported); + + // Test try with array method with an input array containing null values. + // the error should be suppressed and just return null. + assertQuery("SELECT TRY(ARRAY_MAX(ARRAY [ARRAY[1, NULL], ARRAY[1, 2]]))", "SELECT NULL"); + } + } +} diff --git a/presto-node-ttl-fetchers/pom.xml b/presto-node-ttl-fetchers/pom.xml index 997901a2879cc..90980bdfef19f 100644 --- a/presto-node-ttl-fetchers/pom.xml +++ b/presto-node-ttl-fetchers/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-node-ttl-fetchers diff --git a/presto-open-telemetry/pom.xml b/presto-open-telemetry/pom.xml index 85260e23ff396..51991ab75af0c 100644 --- a/presto-open-telemetry/pom.xml +++ b/presto-open-telemetry/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-open-telemetry diff --git a/presto-openapi/pom.xml b/presto-openapi/pom.xml index 1b2369ab8b40b..d02c5e89ec0ba 100644 --- a/presto-openapi/pom.xml +++ b/presto-openapi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-openapi diff --git a/presto-oracle/pom.xml b/presto-oracle/pom.xml index 85f2ad2762d6a..bf8f5b9627a51 100644 --- a/presto-oracle/pom.xml +++ b/presto-oracle/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-oracle diff --git a/presto-oracle/src/main/java/com/facebook/presto/plugin/oracle/OracleClient.java b/presto-oracle/src/main/java/com/facebook/presto/plugin/oracle/OracleClient.java index ec7cf855bd0b2..adcb52324d132 100644 --- a/presto-oracle/src/main/java/com/facebook/presto/plugin/oracle/OracleClient.java +++ b/presto-oracle/src/main/java/com/facebook/presto/plugin/oracle/OracleClient.java @@ -114,14 +114,21 @@ protected void renameTable(JdbcIdentity identity, String catalogName, SchemaTabl throw new PrestoException(NOT_SUPPORTED, "Table rename across schemas is not supported in Oracle"); } - String newTableName = newTable.getTableName().toUpperCase(ENGLISH); - String oldTableName = oldTable.getTableName().toUpperCase(ENGLISH); - String sql = format( - "ALTER TABLE %s RENAME TO %s", - quoted(catalogName, oldTable.getSchemaName(), oldTableName), - quoted(newTableName)); - try (Connection connection = connectionFactory.openConnection(identity)) { + DatabaseMetaData metadata = connection.getMetaData(); + String schemaName = oldTable.getSchemaName(); + String newTableName = newTable.getTableName(); + String oldTableName = oldTable.getTableName(); + if (metadata.storesUpperCaseIdentifiers() && !caseSensitiveNameMatchingEnabled) { + schemaName = schemaName != null ? schemaName.toUpperCase(ENGLISH) : null; + newTableName = newTableName.toUpperCase(ENGLISH); + oldTableName = oldTableName.toUpperCase(ENGLISH); + } + + String sql = format( + "ALTER TABLE %s RENAME TO %s", + quoted(catalogName, schemaName, oldTableName), + quoted(newTableName)); execute(connection, sql); } catch (SQLException e) { diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 64071b4bc9e2c..b3d21a7971f40 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-orc diff --git a/presto-parquet/pom.xml b/presto-parquet/pom.xml index a87adc18151f7..fb557f48a7d43 100644 --- a/presto-parquet/pom.xml +++ b/presto-parquet/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-parquet diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index ea8a3cf23f2cb..c018d2332e487 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-parser diff --git a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 index 4a6731c100f92..6989c565d242f 100644 --- a/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 +++ b/presto-parser/src/main/antlr4/com/facebook/presto/sql/parser/SqlBase.g4 @@ -152,6 +152,8 @@ statement | UPDATE qualifiedName SET updateAssignment (',' updateAssignment)* (WHERE where=booleanExpression)? #update + | MERGE INTO qualifiedName (AS? identifier)? + USING relation ON expression mergeCase+ #mergeInto ; query @@ -528,6 +530,15 @@ filter : FILTER '(' WHERE booleanExpression ')' ; +mergeCase + : WHEN MATCHED THEN + UPDATE SET targetColumns+=identifier EQ values+=expression + (',' targetColumns+=identifier EQ values+=expression)* #mergeUpdate + | WHEN NOT MATCHED THEN + INSERT ('(' columns+=identifier (',' columns+=identifier)* ')')? + VALUES '(' values+=expression (',' values+=expression)* ')' #mergeInsert + ; + over : OVER '(' (PARTITION BY partition+=expression (',' partition+=expression)*)? @@ -553,7 +564,7 @@ frameBound ; updateAssignment - : identifier '=' expression + : identifier EQ expression ; explainOption @@ -678,7 +689,7 @@ nonReserved | JSON | KEEP | KEY | LANGUAGE | LAST | LATERAL | LEVEL | LIMIT | LOGICAL - | MAP | MATERIALIZED | MINUTE | MONTH + | MAP | MATCHED | MATERIALIZED | MERGE | MINUTE | MONTH | NAME | NFC | NFD | NFKC | NFKD | NO | NONE | NULLIF | NULLS | OF | OFFSET | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | PARTITION | PARTITIONS | POSITION | PRECEDING | PRIMARY | PRIVILEGES | PROPERTIES | PRUNE @@ -806,7 +817,9 @@ LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; LOGICAL: 'LOGICAL'; MAP: 'MAP'; +MATCHED: 'MATCHED'; MATERIALIZED: 'MATERIALIZED'; +MERGE: 'MERGE'; MINUTE: 'MINUTE'; MONTH: 'MONTH'; NAME: 'NAME'; diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/QueryUtil.java b/presto-parser/src/main/java/com/facebook/presto/sql/QueryUtil.java index 126c2b10331e0..02dc96e0094a0 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/QueryUtil.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/QueryUtil.java @@ -17,6 +17,7 @@ import com.facebook.presto.sql.tree.AllColumns; import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.GroupBy; @@ -64,6 +65,11 @@ public static Identifier quotedIdentifier(String name) return new Identifier(name, true); } + public static Expression nameReference(String first, String... rest) + { + return DereferenceExpression.from(QualifiedName.of(first, rest)); + } + public static SelectItem unaliasedName(String name) { return new SingleColumn(identifier(name)); @@ -148,6 +154,11 @@ public static Row row(Expression... values) return new Row(ImmutableList.copyOf(values)); } + public static Relation aliased(Relation relation, String alias) + { + return new AliasedRelation(relation, identifier(alias), null); + } + public static Relation aliased(Relation relation, String alias, List columnAliases) { return new AliasedRelation( diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java index d25d71123da11..ec0e4cc94a19e 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/SqlFormatter.java @@ -67,6 +67,10 @@ import com.facebook.presto.sql.tree.JoinUsing; import com.facebook.presto.sql.tree.Lateral; import com.facebook.presto.sql.tree.LikeClause; +import com.facebook.presto.sql.tree.Merge; +import com.facebook.presto.sql.tree.MergeCase; +import com.facebook.presto.sql.tree.MergeInsert; +import com.facebook.presto.sql.tree.MergeUpdate; import com.facebook.presto.sql.tree.NaturalJoin; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.Offset; @@ -678,6 +682,78 @@ protected Void visitIntersect(Intersect node, Integer indent) return null; } + @Override + protected Void visitMerge(Merge node, Integer indent) + { + builder.append("MERGE INTO ") + .append(node.getTargetTable().getName()); + + node.getTargetAlias().ifPresent(value -> builder + .append(' ') + .append(value)); + builder.append("\n"); + + append(indent + 1, "USING "); + + processRelation(node.getSource(), indent + 2); + + builder.append("\n"); + append(indent + 1, "ON "); + builder.append(formatExpression(node.getPredicate(), parameters)); + + for (MergeCase mergeCase : node.getMergeCases()) { + builder.append("\n"); + process(mergeCase, indent); + } + + return null; + } + + @Override + protected Void visitMergeInsert(MergeInsert node, Integer indent) + { + appendMergeCaseWhen(false); + append(indent + 1, "INSERT "); + + if (!node.getColumns().isEmpty()) { + builder.append(node.getColumns().stream() + .map((Identifier expression) -> ExpressionFormatter.formatExpression(expression, parameters)) + .collect(joining(", ", "(", ")"))); + } + + builder.append("\n"); + append(indent + 1, "VALUES "); + builder.append(node.getValues().stream() + .map((Expression expression) -> ExpressionFormatter.formatExpression(expression, parameters)) + .collect(joining(", ", "(", ")"))); + + return null; + } + + @Override + protected Void visitMergeUpdate(MergeUpdate node, Integer indent) + { + appendMergeCaseWhen(true); + append(indent + 1, "UPDATE SET"); + + boolean first = true; + for (MergeUpdate.Assignment assignment : node.getAssignments()) { + builder.append("\n"); + append(indent + 2, first ? " " : ", "); + builder.append(assignment.getTarget()) + .append(" = ") + .append(formatExpression(assignment.getValue(), parameters)); + first = false; + } + + return null; + } + + private void appendMergeCaseWhen(boolean matched) + { + builder.append(matched ? "WHEN MATCHED" : "WHEN NOT MATCHED").append(" THEN\n"); + } + @Override protected Void visitCreateView(CreateView node, Integer indent) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java index 2a21df87fff0f..8b85c6b822cd0 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/parser/AstBuilder.java @@ -109,6 +109,10 @@ import com.facebook.presto.sql.tree.LikePredicate; import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.Merge; +import com.facebook.presto.sql.tree.MergeCase; +import com.facebook.presto.sql.tree.MergeInsert; +import com.facebook.presto.sql.tree.MergeUpdate; import com.facebook.presto.sql.tree.NaturalJoin; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NodeLocation; @@ -469,6 +473,51 @@ public Node visitTruncateTable(SqlBaseParser.TruncateTableContext context) return new TruncateTable(getLocation(context), getQualifiedName(context.qualifiedName())); } + @Override + public Node visitMergeInto(SqlBaseParser.MergeIntoContext context) + { + Table table = new Table(getLocation(context), getQualifiedName(context.qualifiedName())); + Relation targetRelation = table; + if (context.identifier() != null) { + targetRelation = new AliasedRelation(table, (Identifier) visit(context.identifier()), null); + } + return new Merge( + getLocation(context), + targetRelation, + (Relation) visit(context.relation()), + (Expression) visit(context.expression()), + visit(context.mergeCase(), MergeCase.class)); + } + + @Override + public Node visitMergeInsert(SqlBaseParser.MergeInsertContext context) + { + return new MergeInsert( + getLocation(context), + visitIdentifiers(context.columns), + visit(context.values, Expression.class)); + } + + private List visitIdentifiers(List identifiers) + { + return identifiers.stream() + .map(identifier -> (Identifier) visit(identifier)) + .collect(toImmutableList()); + } + + @Override + public Node visitMergeUpdate(SqlBaseParser.MergeUpdateContext context) + { + ImmutableList.Builder assignments = ImmutableList.builder(); + for (int i = 0; i < context.targetColumns.size(); i++) { + assignments.add(new MergeUpdate.Assignment( + (Identifier) visit(context.targetColumns.get(i)), + (Expression) visit(context.values.get(i)))); + } + + return new MergeUpdate(getLocation(context), assignments.build()); + } + @Override public Node visitRenameTable(SqlBaseParser.RenameTableContext context) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java index 267f6638430f6..9ff0c41111561 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/AstVisitor.java @@ -527,6 +527,26 @@ protected R visitCallArgument(CallArgument node, C context) return visitNode(node, context); } + protected R visitMerge(Merge node, C context) + { + return visitStatement(node, context); + } + + protected R visitMergeCase(MergeCase node, C context) + { + return visitNode(node, context); + } + + protected R visitMergeInsert(MergeInsert node, C context) + { + return visitMergeCase(node, context); + } + + protected R visitMergeUpdate(MergeUpdate node, C context) + { + return visitMergeCase(node, context); + } + protected R visitTableElement(TableElement node, C context) { return visitNode(node, context); diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java index e6bd352a69748..9a545cf0df160 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/DefaultTraversalVisitor.java @@ -485,6 +485,34 @@ protected R visitUpdateAssignment(UpdateAssignment node, C context) return null; } + @Override + protected R visitMerge(Merge node, C context) + { + process(node.getTarget(), context); + process(node.getSource(), context); + process(node.getPredicate(), context); + node.getMergeCases().forEach(mergeCase -> process(mergeCase, context)); + return null; + } + + @Override + protected R visitMergeInsert(MergeInsert node, C context) + { + node.getColumns().forEach(column -> process(column, context)); + node.getValues().forEach(expression -> process(expression, context)); + return null; + } + + @Override + protected R visitMergeUpdate(MergeUpdate node, C context) + { + node.getAssignments().forEach(assignment -> { + process(assignment.getTarget(), context); + process(assignment.getValue(), context); + }); + return null; + } + @Override protected R visitCreateTableAsSelect(CreateTableAsSelect node, C context) { diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/Merge.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/Merge.java new file mode 100644 index 0000000000000..5b98d809773d0 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/Merge.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public final class Merge + extends Statement +{ + private final Relation target; + private final Relation source; + private final Expression predicate; + private final List mergeCases; + + public Merge( + NodeLocation location, + Relation target, + Relation source, + Expression predicate, + List mergeCases) + { + this(Optional.of(location), target, source, predicate, mergeCases); + } + + public Merge( + Optional location, + Relation target, + Relation source, + Expression predicate, + List mergeCases) + { + super(location); + this.target = requireNonNull(target, "target is null"); + checkArgument(target instanceof Table || target instanceof AliasedRelation, "target (%s) is neither a Table nor an AliasedRelation", target); + this.source = requireNonNull(source, "source is null"); + this.predicate = requireNonNull(predicate, "predicate is null"); + this.mergeCases = ImmutableList.copyOf(requireNonNull(mergeCases, "mergeCases is null")); + } + + public Relation getTarget() + { + return target; + } + + public Relation getSource() + { + return source; + } + + public Expression getPredicate() + { + return predicate; + } + + public List getMergeCases() + { + return mergeCases; + } + + public Table getTargetTable() + { + if (target instanceof Table) { + return (Table) target; + } + checkArgument(target instanceof AliasedRelation, "MERGE relation is neither a Table nor an AliasedRelation"); + return (Table) ((AliasedRelation) target).getRelation(); + } + + public Optional getTargetAlias() + { + if (target instanceof AliasedRelation) { + return Optional.of(((AliasedRelation) target).getAlias()); + } + return Optional.empty(); + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitMerge(this, context); + } + + @Override + public List getChildren() + { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(target); + builder.add(source); + builder.add(predicate); + builder.addAll(mergeCases); + return builder.build(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Merge merge = (Merge) o; + return Objects.equals(target, merge.target) && + Objects.equals(source, merge.source) && + Objects.equals(predicate, merge.predicate) && + Objects.equals(mergeCases, merge.mergeCases); + } + + @Override + public int hashCode() + { + return Objects.hash(target, source, predicate, mergeCases); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("target", target) + .add("relation", source) + .add("predicate", predicate) + .add("mergeCases", mergeCases) + .omitNullValues() + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeCase.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeCase.java new file mode 100644 index 0000000000000..d10c06a57f183 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeCase.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import java.util.List; +import java.util.Optional; + +public abstract class MergeCase + extends Node +{ + protected MergeCase(Optional location) + { + super(location); + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitMergeCase(this, context); + } + + public abstract List getSetColumns(); + + public abstract List getSetExpressions(); +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeInsert.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeInsert.java new file mode 100644 index 0000000000000..4ea5bb45fabe1 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeInsert.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class MergeInsert + extends MergeCase +{ + private final List columns; + private final List values; + + public MergeInsert(List columns, List values) + { + this(Optional.empty(), columns, values); + } + + public MergeInsert(NodeLocation location, List columns, List values) + { + this(Optional.of(location), columns, values); + } + + public MergeInsert(Optional location, List columns, List values) + { + super(location); + this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + this.values = ImmutableList.copyOf(requireNonNull(values, "values is null")); + } + + public List getColumns() + { + return columns; + } + + public List getValues() + { + return values; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitMergeInsert(this, context); + } + + @Override + public List getSetColumns() + { + return columns; + } + + @Override + public List getSetExpressions() + { + return values; + } + + @Override + public List getChildren() + { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.addAll(columns); + builder.addAll(values); + return builder.build(); + } + + @Override + public int hashCode() + { + return Objects.hash(columns, values); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + MergeInsert o = (MergeInsert) obj; + return Objects.equals(columns, o.columns) && + Objects.equals(values, o.values); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("columns", columns) + .add("values", values) + .omitNullValues() + .toString(); + } +} diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeUpdate.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeUpdate.java new file mode 100644 index 0000000000000..2254baaa18fe6 --- /dev/null +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/MergeUpdate.java @@ -0,0 +1,163 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.tree; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class MergeUpdate + extends MergeCase +{ + private final List assignments; + + public MergeUpdate(List assignments) + { + this(Optional.empty(), assignments); + } + + public MergeUpdate(NodeLocation location, List assignments) + { + this(Optional.of(location), assignments); + } + + public MergeUpdate(Optional location, List assignments) + { + super(location); + this.assignments = ImmutableList.copyOf(requireNonNull(assignments, "assignments is null")); + } + + public List getAssignments() + { + return assignments; + } + + @Override + public R accept(AstVisitor visitor, C context) + { + return visitor.visitMergeUpdate(this, context); + } + + @Override + public List getSetColumns() + { + return assignments.stream() + .map(Assignment::getTarget) + .collect(toImmutableList()); + } + + @Override + public List getSetExpressions() + { + return assignments.stream() + .map(Assignment::getValue) + .collect(toImmutableList()); + } + + @Override + public List getChildren() + { + ImmutableList.Builder builder = ImmutableList.builder(); + assignments.forEach(assignment -> { + builder.add(assignment.getTarget()); + builder.add(assignment.getValue()); + }); + return builder.build(); + } + + @Override + public int hashCode() + { + return Objects.hash(assignments); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + MergeUpdate o = (MergeUpdate) obj; + return Objects.equals(assignments, o.assignments); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("assignments", assignments) + .omitNullValues() + .toString(); + } + + public static class Assignment + { + private final Identifier target; + private final Expression value; + + public Assignment(Identifier target, Expression value) + { + this.target = requireNonNull(target, "target is null"); + this.value = requireNonNull(value, "value is null"); + } + + public Identifier getTarget() + { + return target; + } + + public Expression getValue() + { + return value; + } + + @Override + public int hashCode() + { + return Objects.hash(target, value); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Assignment o = (Assignment) obj; + return Objects.equals(target, o.target) && + Objects.equals(value, o.value); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("target", target) + .add("value", value) + .toString(); + } + } +} diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java index 1dc53fd2b9230..5b5632c3e16cb 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParser.java @@ -93,6 +93,9 @@ import com.facebook.presto.sql.tree.LikeClause; import com.facebook.presto.sql.tree.LogicalBinaryExpression; import com.facebook.presto.sql.tree.LongLiteral; +import com.facebook.presto.sql.tree.Merge; +import com.facebook.presto.sql.tree.MergeInsert; +import com.facebook.presto.sql.tree.MergeUpdate; import com.facebook.presto.sql.tree.NaturalJoin; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NodeLocation; @@ -178,7 +181,10 @@ import java.util.Optional; import java.util.stream.Collectors; +import static com.facebook.presto.sql.QueryUtil.aliased; +import static com.facebook.presto.sql.QueryUtil.equal; import static com.facebook.presto.sql.QueryUtil.identifier; +import static com.facebook.presto.sql.QueryUtil.nameReference; import static com.facebook.presto.sql.QueryUtil.query; import static com.facebook.presto.sql.QueryUtil.quotedIdentifier; import static com.facebook.presto.sql.QueryUtil.row; @@ -1644,6 +1650,41 @@ public void testWherelessUpdate() Optional.empty())); } + @Test + public void testMerge() + { + NodeLocation location = new NodeLocation(1, 1); + assertStatement("" + + "MERGE INTO product_sales AS s\n" + + " USING monthly_sales AS ms\n" + + " ON s.product_id = ms.product_id\n" + + "WHEN MATCHED THEN\n" + + " UPDATE SET\n" + + " sales = sales + ms.sales\n" + + " , last_sale = ms.sale_date\n" + + " , current_price = ms.price\n" + + "WHEN NOT MATCHED THEN\n" + + " INSERT (product_id, sales, last_sale, current_price)\n" + + " VALUES (ms.product_id, ms.sales, ms.sale_date, ms.price)", + new Merge( + location, + new AliasedRelation(location, table(QualifiedName.of("product_sales")), new Identifier("s"), null), + aliased(table(QualifiedName.of("monthly_sales")), "ms"), + equal(nameReference("s", "product_id"), nameReference("ms", "product_id")), + ImmutableList.of( + new MergeUpdate( + ImmutableList.of( + new MergeUpdate.Assignment(new Identifier("sales"), new ArithmeticBinaryExpression( + ArithmeticBinaryExpression.Operator.ADD, nameReference("sales"), nameReference("ms", "sales"))), + new MergeUpdate.Assignment(new Identifier("last_sale"), nameReference("ms", "sale_date")), + new MergeUpdate.Assignment(new Identifier("current_price"), nameReference("ms", "price")))), + new MergeInsert( + ImmutableList.of(new Identifier("product_id"), new Identifier("sales"), new Identifier("last_sale"), + new Identifier("current_price")), + ImmutableList.of(nameReference("ms", "product_id"), nameReference("ms", "sales"), + nameReference("ms", "sale_date"), nameReference("ms", "price")))))); + } + @Test public void testRenameTable() { diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java index d31ec9f5db64e..3dea840f8a7fc 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestSqlParserErrorHandling.java @@ -40,10 +40,10 @@ public Object[][] getStatements() return new Object[][] { {"", "line 1:1: mismatched input ''. Expecting: 'ALTER', 'ANALYZE', 'CALL', 'COMMIT', 'CREATE', 'DEALLOCATE', 'DELETE', 'DESC', 'DESCRIBE', 'DROP', 'EXECUTE', 'EXPLAIN', 'GRANT', " + - "'INSERT', 'PREPARE', 'REFRESH', 'RESET', 'REVOKE', 'ROLLBACK', 'SET', 'SHOW', 'START', 'TRUNCATE', 'UPDATE', 'USE', "}, + "'INSERT', 'MERGE', 'PREPARE', 'REFRESH', 'RESET', 'REVOKE', 'ROLLBACK', 'SET', 'SHOW', 'START', 'TRUNCATE', 'UPDATE', 'USE', "}, {"@select", "line 1:1: mismatched input '@'. Expecting: 'ALTER', 'ANALYZE', 'CALL', 'COMMIT', 'CREATE', 'DEALLOCATE', 'DELETE', 'DESC', 'DESCRIBE', 'DROP', 'EXECUTE', 'EXPLAIN', 'GRANT', " + - "'INSERT', 'PREPARE', 'REFRESH', 'RESET', 'REVOKE', 'ROLLBACK', 'SET', 'SHOW', 'START', 'TRUNCATE', 'UPDATE', 'USE', "}, + "'INSERT', 'MERGE', 'PREPARE', 'REFRESH', 'RESET', 'REVOKE', 'ROLLBACK', 'SET', 'SHOW', 'START', 'TRUNCATE', 'UPDATE', 'USE', "}, {"select * from foo where @what", "line 1:25: mismatched input '@'. Expecting: "}, {"select * from 'oops", diff --git a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java index b2aaac7baf2e5..1b9662102803e 100644 --- a/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java +++ b/presto-parser/src/test/java/com/facebook/presto/sql/parser/TestStatementBuilder.java @@ -295,6 +295,19 @@ public void testStatementBuilder() printStatement("alter table foo alter column bar drop not null"); printStatement("alter table if exists foo alter bar set not null"); printStatement("alter table if exists foo alter bar drop not null"); + + printStatement("" + + "merge into product_sales as s\n" + + " using monthly_sales as ms\n" + + " on s.product_id = ms.product_id\n" + + "when matched then\n" + + " update set\n" + + " sales = sales + ms.sales\n" + + " , last_sale = ms.sale_date\n" + + " , current_price = ms.price\n" + + "when not matched then\n" + + " insert (product_id, sales, last_sale, current_price)\n" + + " values (ms.product_id, ms.sales, ms.sale_date, ms.price)"); } @Test diff --git a/presto-password-authenticators/pom.xml b/presto-password-authenticators/pom.xml index 7e5f3e87811a2..a934422525d64 100644 --- a/presto-password-authenticators/pom.xml +++ b/presto-password-authenticators/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-password-authenticators diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml index 5abe5d9005762..a16f5ef3ae826 100644 --- a/presto-pinot-toolkit/pom.xml +++ b/presto-pinot-toolkit/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-pinot-toolkit diff --git a/presto-pinot/pom.xml b/presto-pinot/pom.xml index a54c4b07b0b2e..fe1e4646e4740 100644 --- a/presto-pinot/pom.xml +++ b/presto-pinot/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-pinot diff --git a/presto-plan-checker-router-plugin/pom.xml b/presto-plan-checker-router-plugin/pom.xml index db1c4b0f1736c..5ff52995f44d8 100644 --- a/presto-plan-checker-router-plugin/pom.xml +++ b/presto-plan-checker-router-plugin/pom.xml @@ -5,12 +5,12 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT com.facebook.presto.router presto-plan-checker-router-plugin - 0.295-SNAPSHOT + 0.296-SNAPSHOT jar presto-plan-checker-router-plugin Presto plan checker router plugin diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index f0aa2edb9267a..52fa42f6bdcef 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 09d1d66350676..d6b322bbd8db8 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-postgresql @@ -176,7 +176,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index c4bf61d33fd08..893ce92e0550b 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-product-tests diff --git a/presto-prometheus/pom.xml b/presto-prometheus/pom.xml index 70a22b905fee7..c459484d3485e 100644 --- a/presto-prometheus/pom.xml +++ b/presto-prometheus/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-prometheus diff --git a/presto-proxy/pom.xml b/presto-proxy/pom.xml index 404f11fd4d5bb..c162675125e88 100644 --- a/presto-proxy/pom.xml +++ b/presto-proxy/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-proxy diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index e07c2579fa922..807170661c76d 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 2e50641e05f8f..0c3feaa347f6f 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 398d59d621684..70c2131a720b9 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-redis @@ -260,18 +260,14 @@ + - test-redis-integration-smoke-test + ci-full-tests org.apache.maven.plugins maven-surefire-plugin - - - **/TestRedisIntegrationSmokeTest.java - - diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java index cab360b3eda21..b7016a6a12796 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisConnectorConfig.java @@ -94,6 +94,7 @@ public class RedisConnectorConfig private boolean tlsEnabled; private File truststorePath; + private boolean caseSensitiveNameMatchingEnabled; @NotNull public File getTableDescriptionDir() @@ -280,4 +281,16 @@ public RedisConnectorConfig setTruststorePath(File truststorePath) this.truststorePath = truststorePath; return this; } + + public boolean isCaseSensitiveNameMatchingEnabled() + { + return caseSensitiveNameMatchingEnabled; + } + + @Config("case-sensitive-name-matching") + public RedisConnectorConfig setCaseSensitiveNameMatchingEnabled(boolean caseSensitiveNameMatchingEnabled) + { + this.caseSensitiveNameMatchingEnabled = caseSensitiveNameMatchingEnabled; + return this; + } } diff --git a/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java b/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java index 8295dd7d0d90c..749e4160db8ce 100644 --- a/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java +++ b/presto-redis/src/main/java/com/facebook/presto/redis/RedisMetadata.java @@ -45,6 +45,7 @@ import static com.facebook.presto.redis.RedisHandleResolver.convertColumnHandle; import static com.facebook.presto.redis.RedisHandleResolver.convertLayout; import static com.facebook.presto.redis.RedisHandleResolver.convertTableHandle; +import static java.util.Locale.ROOT; import static java.util.Objects.requireNonNull; /** @@ -59,6 +60,7 @@ public class RedisMetadata private final String connectorId; private final boolean hideInternalColumns; + private boolean caseSensitiveNameMatchingEnabled; private final Supplier> redisTableDescriptionSupplier; @@ -76,6 +78,7 @@ public class RedisMetadata log.debug("Loading redis table definitions from %s", redisConnectorConfig.getTableDescriptionDir().getAbsolutePath()); this.redisTableDescriptionSupplier = Suppliers.memoize(redisTableDescriptionSupplier::get)::get; + this.caseSensitiveNameMatchingEnabled = redisConnectorConfig.isCaseSensitiveNameMatchingEnabled(); } @Override @@ -271,4 +274,10 @@ private void appendFields(ConnectorSession session, ImmutableList.Builder tableDescriptions = createTpchTableDescriptions(queryRunner.getCoordinator().getMetadata(), tables, dataFormat); - installRedisPlugin(embeddedRedis, queryRunner, tableDescriptions); + installRedisPlugin(embeddedRedis, queryRunner, tableDescriptions, ImmutableMap.of()); TestingPrestoClient prestoClient = queryRunner.getRandomClient(); diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/TestMinimalFunctionality.java b/presto-redis/src/test/java/com/facebook/presto/redis/TestMinimalFunctionality.java index eb68eea79ee9f..9bbbeced55ef3 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/TestMinimalFunctionality.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/TestMinimalFunctionality.java @@ -34,7 +34,7 @@ import java.util.Optional; import java.util.UUID; -import static com.facebook.presto.redis.util.RedisTestUtils.createEmptyTableDescription; +import static com.facebook.presto.redis.util.RedisTestUtils.createEmptyTableDescriptions; import static com.facebook.presto.redis.util.RedisTestUtils.installRedisPlugin; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.testing.assertions.Assert.assertEquals; @@ -76,10 +76,12 @@ public void spinUp() this.queryRunner = new StandaloneQueryRunner(SESSION); - installRedisPlugin(embeddedRedis, queryRunner, + installRedisPlugin( + embeddedRedis, + queryRunner, ImmutableMap.builder() - .put(createEmptyTableDescription(new SchemaTableName("default", tableName))) - .build()); + .putAll(createEmptyTableDescriptions( + new SchemaTableName("default", tableName))).build(), ImmutableMap.of()); } @AfterMethod(alwaysRun = true) diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java index deeb7dfc26e71..d00c45fd451c5 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisConnectorConfig.java @@ -39,7 +39,8 @@ public void testDefaults() .setRedisDataBaseIndex(0) .setRedisPassword(null) .setRedisScanCount(100) - .setHideInternalColumns(true)); + .setHideInternalColumns(true) + .setCaseSensitiveNameMatchingEnabled(false)); } @Test @@ -60,6 +61,7 @@ public void testExplicitPropertyMappings() .put("redis.tls.enabled", "true") .put("redis.tls.truststore-path", "/dev/null") .put("redis.password", "secret") + .put("case-sensitive-name-matching", "true") .build(); RedisConnectorConfig expected = new RedisConnectorConfig() @@ -76,7 +78,8 @@ public void testExplicitPropertyMappings() .setTruststorePath(new File("/dev/null")) .setRedisPassword("secret") .setRedisKeyDelimiter(",") - .setKeyPrefixSchemaTable(true); + .setKeyPrefixSchemaTable(true) + .setCaseSensitiveNameMatchingEnabled(true); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisIntegrationMixedCase.java b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisIntegrationMixedCase.java new file mode 100644 index 0000000000000..60fb3ac2424d2 --- /dev/null +++ b/presto-redis/src/test/java/com/facebook/presto/redis/TestRedisIntegrationMixedCase.java @@ -0,0 +1,223 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.redis; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.redis.util.EmbeddedRedis; +import com.facebook.presto.redis.util.JsonEncoder; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.security.AllowAllAccessControl; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.StandaloneQueryRunner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import redis.clients.jedis.Jedis; + +import java.util.Optional; +import java.util.UUID; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; +import static com.facebook.presto.redis.util.RedisTestUtils.createEmptyTableDescriptions; +import static com.facebook.presto.redis.util.RedisTestUtils.createTableDescriptionsWithColumns; +import static com.facebook.presto.redis.util.RedisTestUtils.installRedisPlugin; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.tests.QueryAssertions.assertContains; +import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestRedisIntegrationMixedCase +{ + private static final Session SESSION = testSessionBuilder() + .setCatalog("redis") + .setSchema("default") + .build(); + + private EmbeddedRedis embeddedRedis; + private final String lowerCaseTable = "test"; + private final String camelCaseTable = "Test"; + private final String upperCaseTable = "TEST"; + private final String testTable = "testTable"; + private StandaloneQueryRunner queryRunner; + + @BeforeClass + public void startRedis() + throws Exception + { + embeddedRedis = EmbeddedRedis.createEmbeddedRedis(); + embeddedRedis.start(); + } + + @AfterClass(alwaysRun = true) + public void stopRedis() + { + embeddedRedis.close(); + embeddedRedis = null; + } + + @BeforeMethod + public void spinUp() + throws Exception + { + this.queryRunner = new StandaloneQueryRunner(SESSION); + + installRedisPlugin( + embeddedRedis, + queryRunner, + ImmutableMap.builder() + .putAll(createEmptyTableDescriptions( + new SchemaTableName("default", lowerCaseTable), + new SchemaTableName("default", camelCaseTable), + new SchemaTableName("default", upperCaseTable))) + .putAll(createTableDescriptionsWithColumns( + ImmutableMap.of( + new SchemaTableName("default", testTable), + ImmutableList.of( + new RedisTableFieldDescription("id", BIGINT, "id", null, null, null, false), + new RedisTableFieldDescription("name", VARCHAR, "name", null, null, null, false), + new RedisTableFieldDescription("NAME", VARCHAR, "NAME", null, null, null, false), + new RedisTableFieldDescription("Address", VARCHAR, "Address", null, null, null, false), + new RedisTableFieldDescription("age", BIGINT, "age", null, null, null, false), + new RedisTableFieldDescription("BAND", VARCHAR, "BAND", null, null, null, false))))) + .build(), + ImmutableMap.of("case-sensitive-name-matching", "true")); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() + { + queryRunner.close(); + queryRunner = null; + } + + private void populateData(int count) + { + JsonEncoder jsonEncoder = new JsonEncoder(); + for (long i = 0; i < count; i++) { + Object value = ImmutableMap.of("id", Long.toString(i), "value", UUID.randomUUID().toString()); + try (Jedis jedis = embeddedRedis.getJedisPool().getResource()) { + jedis.set(lowerCaseTable + ":" + i, jsonEncoder.toString(value)); + jedis.set(camelCaseTable + ":" + i, jsonEncoder.toString(value)); + jedis.set(upperCaseTable + ":" + i, jsonEncoder.toString(value)); + } + } + } + + @Test + public void testTableExists() + { + QualifiedObjectName lowerCaseObjName = new QualifiedObjectName("redis", "default", lowerCaseTable); + QualifiedObjectName camelCaseObjName = new QualifiedObjectName("redis", "default", camelCaseTable); + QualifiedObjectName upperCaseObjName = new QualifiedObjectName("redis", "default", upperCaseTable); + transaction(queryRunner.getTransactionManager(), new AllowAllAccessControl()) + .singleStatement() + .execute(SESSION, session -> { + Optional lowerCaseHandle = queryRunner.getServer().getMetadata().getMetadataResolver(session).getTableHandle(lowerCaseObjName); + Optional camelCaseHandle = queryRunner.getServer().getMetadata().getMetadataResolver(session).getTableHandle(camelCaseObjName); + Optional upperCaseHandle = queryRunner.getServer().getMetadata().getMetadataResolver(session).getTableHandle(upperCaseObjName); + assertTrue(lowerCaseHandle.isPresent()); + assertTrue(camelCaseHandle.isPresent()); + assertTrue(upperCaseHandle.isPresent()); + }); + } + + private void assertTableCount(String tableName, long expectedCount) + { + MaterializedResult result = queryRunner.execute("SELECT count(1) from " + tableName); + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BIGINT).row(expectedCount).build(); + assertEquals(result, expected); + } + + @Test + public void testTableHasData() + { + assertTableCount(lowerCaseTable, 0L); + assertTableCount(camelCaseTable, 0L); + assertTableCount(upperCaseTable, 0L); + + int count = 10; + populateData(count); + + assertTableCount(lowerCaseTable, (long) count); + assertTableCount(camelCaseTable, (long) count); + assertTableCount(upperCaseTable, (long) count); + } + + @Test + public void testShowTables() + { + MaterializedResult result = queryRunner.execute("show tables from redis.default"); + assertEquals(result.getRowCount(), 4); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, createUnboundedVarcharType()) + .row("test") + .row("Test") + .row("TEST") + .row("testTable") + .build(); + + assertContains(result, expected); + } + + @Test + public void testShowColumns() + { + MaterializedResult result = queryRunner.execute("show columns from redis.default.testTable"); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, createUnboundedVarcharType()) + .row("id", "bigint", "", "", Long.valueOf(19), null, null) + .row("name", "varchar", "", "", null, null, Long.valueOf(2147483647)) + .row("NAME", "varchar", "", "", null, null, Long.valueOf(2147483647)) + .row("Address", "varchar", "", "", null, null, Long.valueOf(2147483647)) + .row("age", "bigint", "", "", Long.valueOf(19), null, null) + .row("BAND", "varchar", "", "", null, null, Long.valueOf(2147483647)) + .build(); + assertContains(result, expected); + } + + private void populateTestTable(long count) + { + JsonEncoder jsonEncoder = new JsonEncoder(); + for (long i = 0; i < count; i++) { + Object value = ImmutableMap.of("id", Long.toString(i), "name", "name_value_" + i, "NAME", "NAME_value_" + i, "Address", "Address_value_" + i, "age", Long.toString(25), "BAND", "BAND_value_" + i); + try (Jedis jedis = embeddedRedis.getJedisPool().getResource()) { + jedis.set(testTable + ":" + i, jsonEncoder.toString(value)); + } + } + } + + @Test + public void testSelect() + { + populateTestTable(3); + MaterializedResult result = queryRunner.execute("SELECT * FROM " + testTable); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), createUnboundedVarcharType()) + .row(Long.valueOf(0), "name_value_0", "NAME_value_0", "Address_value_0", Long.valueOf(25), "BAND_value_0") + .row(Long.valueOf(1), "name_value_1", "NAME_value_1", "Address_value_1", Long.valueOf(25), "BAND_value_1") + .row(Long.valueOf(2), "name_value_2", "NAME_value_2", "Address_value_2", Long.valueOf(25), "BAND_value_2") + .build(); + assertContains(result, expected); + } +} diff --git a/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java b/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java index e521268fbcda4..950b4fe008e0b 100644 --- a/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java +++ b/presto-redis/src/test/java/com/facebook/presto/redis/util/RedisTestUtils.java @@ -17,6 +17,8 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.redis.RedisPlugin; import com.facebook.presto.redis.RedisTableDescription; +import com.facebook.presto.redis.RedisTableFieldDescription; +import com.facebook.presto.redis.RedisTableFieldGroup; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.TestingPrestoClient; @@ -27,7 +29,11 @@ import java.io.IOException; import java.io.InputStream; import java.util.AbstractMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static java.lang.String.format; @@ -35,19 +41,20 @@ public final class RedisTestUtils { private RedisTestUtils() {} - public static void installRedisPlugin(EmbeddedRedis embeddedRedis, QueryRunner queryRunner, Map tableDescriptions) + public static void installRedisPlugin(EmbeddedRedis embeddedRedis, QueryRunner queryRunner, Map tableDescriptions, Map connectorProperties) { RedisPlugin redisPlugin = new RedisPlugin(); redisPlugin.setTableDescriptionSupplier(() -> tableDescriptions); queryRunner.installPlugin(redisPlugin); - Map redisConfig = ImmutableMap.of( - "redis.nodes", embeddedRedis.getConnectString() + ":" + embeddedRedis.getPort(), - "redis.table-names", Joiner.on(",").join(tableDescriptions.keySet()), - "redis.default-schema", "default", - "redis.hide-internal-columns", "true", - "redis.key-prefix-schema-table", "true"); - queryRunner.createCatalog("redis", "redis", redisConfig); + connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties)); + connectorProperties.putIfAbsent("redis.nodes", embeddedRedis.getConnectString() + ":" + embeddedRedis.getPort()); + connectorProperties.putIfAbsent("redis.table-names", Joiner.on(",").join(tableDescriptions.keySet())); + connectorProperties.putIfAbsent("redis.default-schema", "default"); + connectorProperties.putIfAbsent("redis.hide-internal-columns", "true"); + connectorProperties.putIfAbsent("redis.key-prefix-schema-table", "true"); + + queryRunner.createCatalog("redis", "redis", connectorProperties); } public static void loadTpchTable(EmbeddedRedis embeddedRedis, TestingPrestoClient prestoClient, String tableName, QualifiedObjectName tpchTableName, String dataFormat) @@ -76,14 +83,23 @@ public static Map.Entry loadTpchTableDes return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, tableDescription); } - public static Map.Entry createEmptyTableDescription(SchemaTableName schemaTableName) + public static Map createEmptyTableDescriptions(SchemaTableName... schemaTableNames) { - RedisTableDescription tableDescription = new RedisTableDescription( - schemaTableName.getTableName(), - schemaTableName.getSchemaName(), - null, - null); + return Arrays.stream(schemaTableNames) + .collect(Collectors.toMap( + schemaTableName -> schemaTableName, + schemaTableName -> new RedisTableDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), null, null))); + } - return new AbstractMap.SimpleImmutableEntry<>(schemaTableName, tableDescription); + public static Map createTableDescriptionsWithColumns( + Map> tablesWithColumns) + { + return tablesWithColumns.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, entry -> { SchemaTableName schemaTable = entry.getKey(); + List columns = entry.getValue(); + RedisTableFieldGroup valueGroup = new RedisTableFieldGroup("json", schemaTable.getTableName() + ":*", columns); + RedisTableFieldGroup keyGroup = null; + return new RedisTableDescription(schemaTable.getTableName(), schemaTable.getSchemaName(), keyGroup, valueGroup); + })); } } diff --git a/presto-redshift/pom.xml b/presto-redshift/pom.xml index c0ea409f6dcf9..c9f35e3cb307c 100644 --- a/presto-redshift/pom.xml +++ b/presto-redshift/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-redshift diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 838538606bc21..2430f6604de8b 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-resource-group-managers diff --git a/presto-router-example-plugin-scheduler/pom.xml b/presto-router-example-plugin-scheduler/pom.xml index 3377ca423f854..1269688dce06e 100644 --- a/presto-router-example-plugin-scheduler/pom.xml +++ b/presto-router-example-plugin-scheduler/pom.xml @@ -5,12 +5,12 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT com.facebook.presto.router presto-router-example-plugin-scheduler - 0.295-SNAPSHOT + 0.296-SNAPSHOT jar presto-router-example-plugin-scheduler Presto-router example custom plugin scheduler diff --git a/presto-router/pom.xml b/presto-router/pom.xml index 3a55686a187f2..6803384eb65db 100644 --- a/presto-router/pom.xml +++ b/presto-router/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-router diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 9f79458020224..663b75006895d 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-server diff --git a/presto-session-property-managers-common/pom.xml b/presto-session-property-managers-common/pom.xml index 098b68e7ea58c..e0244a9f74844 100644 --- a/presto-session-property-managers-common/pom.xml +++ b/presto-session-property-managers-common/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-session-property-managers-common diff --git a/presto-singlestore/pom.xml b/presto-singlestore/pom.xml index 4a8fb24172a67..618e4d6e8bff8 100644 --- a/presto-singlestore/pom.xml +++ b/presto-singlestore/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-singlestore @@ -179,7 +179,6 @@ org.jetbrains annotations - 19.0.0 test diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index d216e1fde78bb..d9ccddc626b9e 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java index bfabba20b632e..b30af13018b06 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java @@ -30,7 +30,6 @@ import com.facebook.presto.server.security.PrestoAuthenticatorManager; import com.facebook.presto.spark.classloader_interface.PrestoSparkBootstrapTimer; import com.facebook.presto.spark.classloader_interface.SparkProcessType; -import com.facebook.presto.spark.execution.property.NativeExecutionConfigModule; import com.facebook.presto.spi.security.AccessControl; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.parser.SqlParserOptions; @@ -61,7 +60,6 @@ public class PrestoSparkInjectorFactory private final SparkProcessType sparkProcessType; private final Map configProperties; private final Map> catalogProperties; - private final Optional> nativeWorkerConfigProperties; private final Optional> eventListenerProperties; private final Optional> accessControlProperties; private final Optional> sessionPropertyConfigurationProperties; @@ -75,7 +73,6 @@ public PrestoSparkInjectorFactory( SparkProcessType sparkProcessType, Map configProperties, Map> catalogProperties, - Optional> nativeWorkerConfigProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -88,7 +85,6 @@ public PrestoSparkInjectorFactory( sparkProcessType, configProperties, catalogProperties, - nativeWorkerConfigProperties, eventListenerProperties, accessControlProperties, sessionPropertyConfigurationProperties, @@ -103,7 +99,6 @@ public PrestoSparkInjectorFactory( SparkProcessType sparkProcessType, Map configProperties, Map> catalogProperties, - Optional> nativeWorkerConfigProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -117,7 +112,6 @@ public PrestoSparkInjectorFactory( this.configProperties = ImmutableMap.copyOf(requireNonNull(configProperties, "configProperties is null")); this.catalogProperties = requireNonNull(catalogProperties, "catalogProperties is null").entrySet().stream() .collect(toImmutableMap(Entry::getKey, entry -> ImmutableMap.copyOf(entry.getValue()))); - this.nativeWorkerConfigProperties = requireNonNull(nativeWorkerConfigProperties, "nativeWorkerConfigProperties is null").map(ImmutableMap::copyOf); this.eventListenerProperties = requireNonNull(eventListenerProperties, "eventListenerProperties is null").map(ImmutableMap::copyOf); this.accessControlProperties = requireNonNull(accessControlProperties, "accessControlProperties is null").map(ImmutableMap::copyOf); this.sessionPropertyConfigurationProperties = requireNonNull(sessionPropertyConfigurationProperties, "sessionPropertyConfigurationProperties is null").map(ImmutableMap::copyOf); @@ -166,11 +160,6 @@ public Injector create(PrestoSparkBootstrapTimer bootstrapTimer) modules.add(new TempStorageModule()); } - Map nativeWorkerConfigs = new HashMap<>( - this.nativeWorkerConfigProperties.orElse(ImmutableMap.of())); - nativeWorkerConfigs.put("node.environment", "spark"); - modules.add(new NativeExecutionConfigModule(nativeWorkerConfigs)); - modules.addAll(additionalModules); Bootstrap app = new Bootstrap(modules.build()); @@ -225,13 +214,17 @@ public Injector create(PrestoSparkBootstrapTimer bootstrapTimer) } } + FeaturesConfig featuresConfig = injector.getInstance(FeaturesConfig.class); if (sparkProcessType.equals(DRIVER) || - !injector.getInstance(FeaturesConfig.class).isInlineSqlFunctions()) { + (!featuresConfig.isNativeExecutionEnabled() + && !featuresConfig.isInlineSqlFunctions())) { if (functionNamespaceProperties.isPresent()) { - injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers(functionNamespaceProperties.get()); + injector.getInstance(StaticFunctionNamespaceStore.class) + .loadFunctionNamespaceManagers(functionNamespaceProperties.get()); } else { - injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers(); + injector.getInstance(StaticFunctionNamespaceStore.class) + .loadFunctionNamespaceManagers(); } } bootstrapTimer.endDriverModulesLoading(); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index 5cd272b85b277..f0f297904497b 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -121,7 +121,6 @@ import com.facebook.presto.spark.execution.PrestoSparkBroadcastTableCacheManager; import com.facebook.presto.spark.execution.PrestoSparkExecutionExceptionFactory; import com.facebook.presto.spark.execution.http.BatchTaskUpdateRequest; -import com.facebook.presto.spark.execution.property.NativeExecutionNodeConfig; import com.facebook.presto.spark.execution.shuffle.PrestoSparkLocalShuffleReadInfo; import com.facebook.presto.spark.execution.shuffle.PrestoSparkLocalShuffleWriteInfo; import com.facebook.presto.spark.execution.task.PrestoSparkNativeTaskExecutorFactory; @@ -283,7 +282,6 @@ protected void setup(Binder binder) configBinder(binder).bindConfig(SessionPropertyProviderConfig.class); configBinder(binder).bindConfig(PrestoSparkConfig.class); configBinder(binder).bindConfig(TracingConfig.class); - configBinder(binder).bindConfig(NativeExecutionNodeConfig.class); configBinder(binder).bindConfig(PlanCheckerProviderManagerConfig.class); configBinder(binder).bindConfig(SecurityConfig.class); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java index c86d3b30f94ef..72528bc875cea 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java @@ -20,14 +20,16 @@ import com.facebook.presto.spark.classloader_interface.PrestoSparkConfiguration; import com.facebook.presto.spark.classloader_interface.SparkProcessType; import com.facebook.presto.spark.execution.nativeprocess.NativeExecutionModule; +import com.facebook.presto.spark.execution.property.NativeExecutionConfigModule; import com.facebook.presto.sql.parser.SqlParserOptions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Injector; import com.google.inject.Module; +import java.util.HashMap; import java.util.List; -import java.util.Optional; +import java.util.Map; import static com.facebook.presto.spark.classloader_interface.PrestoSparkConfiguration.METADATA_STORAGE_TYPE_LOCAL; import static com.google.common.base.Preconditions.checkArgument; @@ -49,7 +51,6 @@ public IPrestoSparkService createService(SparkProcessType sparkProcessType, Pres sparkProcessType, properties.build(), configuration.getCatalogProperties(), - configuration.getNativeWorkerConfigProperties(), configuration.getEventListenerProperties(), configuration.getAccessControlProperties(), configuration.getSessionPropertyConfigurationProperties(), @@ -67,11 +68,20 @@ public IPrestoSparkService createService(SparkProcessType sparkProcessType, Pres protected List getAdditionalModules(PrestoSparkConfiguration configuration) { - checkArgument(METADATA_STORAGE_TYPE_LOCAL.equalsIgnoreCase(configuration.getMetadataStorageType()), "only local metadata storage is supported"); + checkArgument( + METADATA_STORAGE_TYPE_LOCAL.equalsIgnoreCase(configuration.getMetadataStorageType()), + "only local metadata storage is supported"); + + Map nativeWorkerConfigs = new HashMap<>( + configuration.getNativeWorkerConfigProperties().orElse(ImmutableMap.of())); + nativeWorkerConfigs.put("node.environment", "spark"); return ImmutableList.of( new PrestoSparkLocalMetadataStorageModule(), - // TODO: Need to let NativeExecutionModule addition be controlled by configuration as well. - new NativeExecutionModule(Optional.of(configuration.getCatalogProperties()))); + // TODO: Need to let NativeExecutionModule addition be controlled by configuration + // as well. + new NativeExecutionModule(), + new NativeExecutionConfigModule(nativeWorkerConfigs, + configuration.getNativeWorkerCatalogProperties().orElse(ImmutableMap.of()))); } protected SqlParserOptions getSqlParserOptions() diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionModule.java index 8df6a83395b35..727bd830ca998 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionModule.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.spark.execution.nativeprocess; -import com.facebook.presto.spark.execution.property.NativeExecutionCatalogProperties; import com.facebook.presto.spark.execution.property.PrestoSparkWorkerProperty; import com.facebook.presto.spark.execution.property.WorkerProperty; import com.facebook.presto.spark.execution.shuffle.PrestoSparkLocalShuffleInfoTranslator; @@ -21,7 +20,6 @@ import com.facebook.presto.spark.execution.task.ForNativeExecutionTask; import com.facebook.presto.spark.execution.task.NativeExecutionTaskFactory; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; @@ -30,8 +28,6 @@ import com.google.inject.TypeLiteral; import okhttp3.OkHttpClient; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.TimeUnit; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; @@ -39,15 +35,10 @@ public class NativeExecutionModule implements Module { - private Optional>> catalogProperties; - // In the future, we would make more bindings injected into NativeExecutionModule // to be able to test various configuration parameters @VisibleForTesting - public NativeExecutionModule(Optional>> catalogProperties) - { - this.catalogProperties = catalogProperties; - } + public NativeExecutionModule() {} @Override public void configure(Binder binder) @@ -67,10 +58,6 @@ protected void bindShuffle(Binder binder) protected void bindWorkerProperties(Binder binder) { - // Bind NativeExecutionCatalogProperties - this is not bound elsewhere - binder.bind(NativeExecutionCatalogProperties.class).toInstance( - new NativeExecutionCatalogProperties(catalogProperties.orElse(ImmutableMap.of()))); - // Bind worker property classes newOptionalBinder(binder, new TypeLiteral>() { }).setDefault().to(PrestoSparkWorkerProperty.class).in(Scopes.SINGLETON); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionProcess.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionProcess.java index c2c0ded8416a5..74afdd996f691 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionProcess.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/nativeprocess/NativeExecutionProcess.java @@ -85,7 +85,7 @@ public class NativeExecutionProcess private static final String NATIVE_EXECUTION_TASK_ERROR_MESSAGE = "Native process launch failed with multiple retries."; private static final String WORKER_CONFIG_FILE = "/config.properties"; private static final String WORKER_NODE_CONFIG_FILE = "/node.properties"; - private static final String WORKER_CONNECTOR_CONFIG_FILE = "/catalog/"; + private static final String WORKER_CONNECTOR_CONFIG_DIR = "/catalog/"; private static final String NATIVE_PROCESS_MEMORY_SPARK_CONF_NAME = "spark.memory.offHeap.size"; private static final int SIGSYS = 31; @@ -338,7 +338,7 @@ private void populateConfigurationFiles(String configBasePath) workerProperty.populateAllProperties( Paths.get(configBasePath, WORKER_CONFIG_FILE), Paths.get(configBasePath, WORKER_NODE_CONFIG_FILE), - Paths.get(configBasePath, WORKER_CONNECTOR_CONFIG_FILE)); // Directory path for catalogs + Paths.get(configBasePath, WORKER_CONNECTOR_CONFIG_DIR)); // Directory path for catalogs } private void updateWorkerProperties() diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogProperties.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogConfig.java similarity index 71% rename from presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogProperties.java rename to presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogConfig.java index a217c4e7f5ae8..61a873bd649cc 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogProperties.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionCatalogConfig.java @@ -13,6 +13,9 @@ */ package com.facebook.presto.spark.execution.property; +import javax.inject.Inject; +import javax.inject.Named; + import java.util.Map; import static java.util.Objects.requireNonNull; @@ -21,16 +24,20 @@ * This class holds catalog properties for native execution process. * Each catalog will generate a separate .properties file. */ -public class NativeExecutionCatalogProperties +public class NativeExecutionCatalogConfig { + public static final String NATIVE_EXECUTION_CATALOG_CONFIG = "native-execution-catalog-config"; + private final Map> catalogProperties; - public NativeExecutionCatalogProperties(Map> catalogProperties) + @Inject + public NativeExecutionCatalogConfig( + @Named(NATIVE_EXECUTION_CATALOG_CONFIG) Map> catalogProperties) { this.catalogProperties = requireNonNull(catalogProperties, "catalogProperties is null"); } - public Map> getAllCatalogProperties() + public Map> getAllProperties() { return catalogProperties; } diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionConfigModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionConfigModule.java index 38f6b365bdf79..35f133f00ea72 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionConfigModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/NativeExecutionConfigModule.java @@ -31,11 +31,15 @@ public class NativeExecutionConfigModule extends AbstractModule { private final Map systemConfigs; + private final Map> catalogConfigs; - public NativeExecutionConfigModule(Map systemConfigs) + public NativeExecutionConfigModule(Map systemConfigs, + Map> catalogConfigs) { this.systemConfigs = ImmutableMap.copyOf( requireNonNull(systemConfigs, "systemConfigs is null")); + this.catalogConfigs = ImmutableMap.copyOf( + requireNonNull(catalogConfigs, "catalogConfigs is null")); } @Override @@ -45,7 +49,13 @@ protected void configure() .annotatedWith( Names.named(NativeExecutionSystemConfig.NATIVE_EXECUTION_SYSTEM_CONFIG)) .toInstance(systemConfigs); + bind(new TypeLiteral>>() {}) + .annotatedWith( + Names.named(NativeExecutionCatalogConfig.NATIVE_EXECUTION_CATALOG_CONFIG)) + .toInstance(catalogConfigs); bind(NativeExecutionSystemConfig.class).in(Scopes.SINGLETON); + bind(NativeExecutionCatalogConfig.class).in(Scopes.SINGLETON); + bind(NativeExecutionNodeConfig.class).in(Scopes.SINGLETON); } } diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/PrestoSparkWorkerProperty.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/PrestoSparkWorkerProperty.java index becc2c4b47d71..05ceccc8755ad 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/PrestoSparkWorkerProperty.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/PrestoSparkWorkerProperty.java @@ -19,11 +19,11 @@ * A utility class that helps with properties and its materialization. */ public class PrestoSparkWorkerProperty - extends WorkerProperty + extends WorkerProperty { @Inject public PrestoSparkWorkerProperty( - NativeExecutionCatalogProperties catalogProperties, + NativeExecutionCatalogConfig catalogProperties, NativeExecutionNodeConfig nodeConfig, NativeExecutionSystemConfig systemConfig) { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/WorkerProperty.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/WorkerProperty.java index 67b5aa22986fb..d594b811da374 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/WorkerProperty.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/property/WorkerProperty.java @@ -64,25 +64,25 @@ * } * } */ -public class WorkerProperty +public class WorkerProperty { - private final T1 catalogProperties; + private final T1 catalogConfig; private final T2 nodeConfig; private final T3 systemConfig; public WorkerProperty( - T1 catalogProperties, + T1 catalogConfig, T2 nodeConfig, T3 systemConfig) { this.systemConfig = requireNonNull(systemConfig, "systemConfig is null"); this.nodeConfig = requireNonNull(nodeConfig, "nodeConfig is null"); - this.catalogProperties = requireNonNull(catalogProperties, "catalogProperties is null"); + this.catalogConfig = requireNonNull(catalogConfig, "catalogConfig is null"); } - public T1 getCatalogProperties() + public T1 getCatalogConfig() { - return catalogProperties; + return catalogConfig; } public T2 getNodeConfig() @@ -95,12 +95,12 @@ public T3 getSystemConfig() return systemConfig; } - public void populateAllProperties(Path systemConfigPath, Path nodeConfigPath, Path catalogConfigsPath) + public void populateAllProperties(Path systemConfigPath, Path nodeConfigPath, Path catalogConfigsDir) throws IOException { populateProperty(systemConfig.getAllProperties(), systemConfigPath); populateProperty(nodeConfig.getAllProperties(), nodeConfigPath); - populateCatalogProperties(catalogProperties.getAllCatalogProperties(), catalogConfigsPath); + populateCatalogConfig(catalogConfig.getAllProperties(), catalogConfigsDir); } private void populateProperty(Map properties, Path path) @@ -123,7 +123,7 @@ private void populateProperty(Map properties, Path path) } } - private void populateCatalogProperties(Map> catalogProperties, Path path) + private void populateCatalogConfig(Map> catalogProperties, Path path) throws IOException { File catalogDir = path.toFile(); diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index 0d5945fdb5a82..b279e11f825cc 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -58,6 +58,7 @@ import com.facebook.presto.spark.classloader_interface.PrestoSparkTaskExecutorFactoryProvider; import com.facebook.presto.spark.execution.AbstractPrestoSparkQueryExecution; import com.facebook.presto.spark.execution.nativeprocess.NativeExecutionModule; +import com.facebook.presto.spark.execution.property.NativeExecutionConfigModule; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.WarningCollector; @@ -247,8 +248,9 @@ public static PrestoSparkQueryRunner createHivePrestoSparkQueryRunner(Iterable> catalogProperties = ImmutableMap.of( "hive", ImmutableMap.of("hive.metastore.uri", "thrift://localhost:9083"), "tpch", ImmutableMap.of("tpch.splits-per-node", "4")); - NativeExecutionCatalogProperties configWithProps = new NativeExecutionCatalogProperties(catalogProperties); - assertEquals(configWithProps.getAllCatalogProperties(), catalogProperties); + NativeExecutionCatalogConfig configWithProps = new NativeExecutionCatalogConfig(catalogProperties); + assertEquals(configWithProps.getAllProperties(), catalogProperties); } @Test public void testFilePropertiesPopulator() { PrestoSparkWorkerProperty workerProperty = new PrestoSparkWorkerProperty( - new NativeExecutionCatalogProperties(ImmutableMap.of()), new NativeExecutionNodeConfig(), + new NativeExecutionCatalogConfig(ImmutableMap.of()), new NativeExecutionNodeConfig(), new NativeExecutionSystemConfig(ImmutableMap.of())); testPropertiesPopulate(workerProperty); } @@ -237,7 +237,7 @@ private void testPropertiesPopulate(PrestoSparkWorkerProperty workerProperty) verifyProperties(workerProperty.getSystemConfig().getAllProperties(), readPropertiesFromDisk(configPropertiesPath)); verifyProperties(workerProperty.getNodeConfig().getAllProperties(), readPropertiesFromDisk(nodePropertiesPath)); // Verify each catalog file was created properly - workerProperty.getCatalogProperties().getAllCatalogProperties().forEach( + workerProperty.getCatalogConfig().getAllProperties().forEach( (catalogName, catalogProperties) -> { Path catalogFilePath = catalogDirectory.resolve(catalogName + ".properties"); verifyProperties(catalogProperties, readPropertiesFromDisk(catalogFilePath)); diff --git a/presto-spark-classloader-interface/pom.xml b/presto-spark-classloader-interface/pom.xml index a95e45827c5f1..713b5b029c0b9 100644 --- a/presto-spark-classloader-interface/pom.xml +++ b/presto-spark-classloader-interface/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkConfiguration.java b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkConfiguration.java index 378373dd4e9b0..de2b31bd08198 100644 --- a/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkConfiguration.java +++ b/presto-spark-classloader-interface/src/main/java/com/facebook/presto/spark/classloader_interface/PrestoSparkConfiguration.java @@ -31,6 +31,7 @@ public class PrestoSparkConfiguration private final Map> catalogProperties; private final Map prestoSparkProperties; private final Optional> nativeWorkerConfigProperties; + private final Optional>> nativeWorkerCatalogProperties; private final Optional> eventListenerProperties; private final Optional> accessControlProperties; private final Optional> sessionPropertyConfigurationProperties; @@ -43,6 +44,7 @@ public PrestoSparkConfiguration( Map> catalogProperties, Map prestoSparkProperties, Optional> nativeWorkerConfigProperties, + Optional>> nativeWorkerCatalogProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -56,6 +58,7 @@ public PrestoSparkConfiguration( this.prestoSparkProperties = unmodifiableMap(new HashMap<>(requireNonNull(prestoSparkProperties, "prestoSparkProperties is null"))); requireNonNull(prestoSparkProperties.get(METADATA_STORAGE_TYPE_KEY), "prestoSparkProperties must contain " + METADATA_STORAGE_TYPE_KEY); this.nativeWorkerConfigProperties = requireNonNull(nativeWorkerConfigProperties, "nativeWorkerConfigProperties is null"); + this.nativeWorkerCatalogProperties = requireNonNull(nativeWorkerCatalogProperties, "nativeWorkerCatalogProperties is null"); this.eventListenerProperties = requireNonNull(eventListenerProperties, "eventListenerProperties is null") .map(properties -> unmodifiableMap(new HashMap<>(properties))); this.accessControlProperties = requireNonNull(accessControlProperties, "accessControlProperties is null") @@ -100,6 +103,11 @@ public Optional> getNativeWorkerConfigProperties() return nativeWorkerConfigProperties; } + public Optional>> getNativeWorkerCatalogProperties() + { + return nativeWorkerCatalogProperties; + } + public Optional> getEventListenerProperties() { return eventListenerProperties; diff --git a/presto-spark-classloader-spark2/pom.xml b/presto-spark-classloader-spark2/pom.xml index efbe208415200..d3e03d9ec548d 100644 --- a/presto-spark-classloader-spark2/pom.xml +++ b/presto-spark-classloader-spark2/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spark-common/pom.xml b/presto-spark-common/pom.xml index 7f66f0245cd89..ec22b3acda8c2 100644 --- a/presto-spark-common/pom.xml +++ b/presto-spark-common/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-spark-common diff --git a/presto-spark-launcher/pom.xml b/presto-spark-launcher/pom.xml index 0b0dd2bb59d92..c7fac2026bfc8 100644 --- a/presto-spark-launcher/pom.xml +++ b/presto-spark-launcher/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkClientOptions.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkClientOptions.java index 56dc51cfc1d70..7b35270ae0d96 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkClientOptions.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkClientOptions.java @@ -32,6 +32,9 @@ public class PrestoSparkClientOptions @Option(name = {"--catalogs"}, title = "directory", description = "catalog configuration directory path", required = true) public String catalogs; + @Option(name = {"-nca", "--native-worker-catalogs"}, title = "directory", description = "native worker catalog configuration directory path") + public String nativeWorkerCatalogs; + @Option(name = "--catalog", title = "catalog", description = "Default catalog") public String catalog; diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkDistribution.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkDistribution.java index 6d939be5be8e4..e0c76afded8b0 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkDistribution.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkDistribution.java @@ -34,6 +34,7 @@ public class PrestoSparkDistribution private final Map> catalogProperties; private final Map prestoSparkProperties; private final Optional> nativeWorkerConfigProperties; + private final Optional>> nativeWorkerCatalogConfigProperties; private final Optional> eventListenerProperties; private final Optional> accessControlProperties; private final Optional> sessionPropertyConfigurationProperties; @@ -48,6 +49,7 @@ public PrestoSparkDistribution( Map> catalogProperties, String metadataStorageType, Optional> nativeWorkerConfigProperties, + Optional>> nativeWorkerCatalogConfigProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -61,6 +63,7 @@ public PrestoSparkDistribution( catalogProperties, ImmutableMap.of(METADATA_STORAGE_TYPE_KEY, metadataStorageType), nativeWorkerConfigProperties, + nativeWorkerCatalogConfigProperties, eventListenerProperties, accessControlProperties, sessionPropertyConfigurationProperties, @@ -75,6 +78,7 @@ public PrestoSparkDistribution( Map> catalogProperties, Map prestoSparkProperties, Optional> nativeWorkerConfigProperties, + Optional>> nativeWorkerCatalogConfigProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -88,6 +92,7 @@ public PrestoSparkDistribution( .collect(toImmutableMap(Map.Entry::getKey, entry -> ImmutableMap.copyOf(entry.getValue()))); this.prestoSparkProperties = ImmutableMap.copyOf(requireNonNull(prestoSparkProperties, "prestoSparkProperties is null")); this.nativeWorkerConfigProperties = requireNonNull(nativeWorkerConfigProperties, "nativeWorkerConfigProperties is null"); + this.nativeWorkerCatalogConfigProperties = requireNonNull(nativeWorkerCatalogConfigProperties, "nativeWorkerCatalogConfigProperties is null"); this.eventListenerProperties = requireNonNull(eventListenerProperties, "eventListenerProperties is null") .map(properties -> unmodifiableMap(new HashMap<>(properties))); this.accessControlProperties = requireNonNull(accessControlProperties, "accessControlProperties is null") @@ -132,6 +137,11 @@ public Optional> getNativeWorkerConfigProperties() return nativeWorkerConfigProperties; } + public Optional>> getNativeWorkerCatalogConfigProperties() + { + return nativeWorkerCatalogConfigProperties; + } + public Optional> getEventListenerProperties() { return eventListenerProperties; diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java index db05ab40a187c..c3e8c08c8db45 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkLauncherCommand.java @@ -61,8 +61,10 @@ public void run() ImmutableMap.of(METADATA_STORAGE_TYPE_KEY, METADATA_STORAGE_TYPE_LOCAL), clientOptions.nativeWorkerConfig == null ? Optional.empty() : Optional.of( loadProperties(checkFile(new File(clientOptions.nativeWorkerConfig)))), - Optional.empty(), - Optional.empty(), + clientOptions.nativeWorkerCatalogs == null ? Optional.empty() : Optional.of( + loadCatalogProperties(new File(clientOptions.nativeWorkerCatalogs))), + Optional.empty(), + Optional.empty(), clientOptions.sessionPropertyConfig == null ? Optional.empty() : Optional.of( loadProperties(checkFile(new File(clientOptions.sessionPropertyConfig)))), Optional.empty(), diff --git a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java index 134719d5b650d..4c8af446e8b40 100644 --- a/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java +++ b/presto-spark-launcher/src/main/java/com/facebook/presto/spark/launcher/PrestoSparkRunner.java @@ -71,6 +71,7 @@ public PrestoSparkRunner(PrestoSparkDistribution distribution) distribution.getCatalogProperties(), distribution.getPrestoSparkProperties(), distribution.getNativeWorkerConfigProperties(), + distribution.getNativeWorkerCatalogConfigProperties(), distribution.getEventListenerProperties(), distribution.getAccessControlProperties(), distribution.getSessionPropertyConfigurationProperties(), @@ -234,6 +235,7 @@ private static IPrestoSparkService createService( Map> catalogProperties, Map prestoSparkProperties, Optional> nativeWorkerConfigProperties, + Optional>> nativeWorkerCatalogConfigProperties, Optional> eventListenerProperties, Optional> accessControlProperties, Optional> sessionPropertyConfigurationProperties, @@ -252,6 +254,7 @@ private static IPrestoSparkService createService( catalogProperties, prestoSparkProperties, nativeWorkerConfigProperties, + nativeWorkerCatalogConfigProperties, eventListenerProperties, accessControlProperties, sessionPropertyConfigurationProperties, @@ -278,6 +281,7 @@ private static class DistributionBasedPrestoSparkTaskExecutorFactoryProvider private final Map configProperties; private final Map nativeWorkerConfigProperties; private final Map> catalogProperties; + private final Map> nativeWorkerCatalogConfigProperties; private final Map prestoSparkProperties; private final Map eventListenerProperties; private final Map accessControlProperties; @@ -296,6 +300,7 @@ public DistributionBasedPrestoSparkTaskExecutorFactoryProvider( this.configProperties = distribution.getConfigProperties(); this.nativeWorkerConfigProperties = distribution.getNativeWorkerConfigProperties().orElse(null); this.catalogProperties = distribution.getCatalogProperties(); + this.nativeWorkerCatalogConfigProperties = distribution.getNativeWorkerCatalogConfigProperties().orElse(null); this.prestoSparkProperties = distribution.getPrestoSparkProperties(); this.bootstrapMetricsCollector = requireNonNull(bootstrapMetricsCollector); // Optional is not Serializable @@ -328,6 +333,7 @@ public IPrestoSparkTaskExecutorFactory getNative() private static Map currentConfigProperties; private static Map currentNativeWorkerConfigProperties; private static Map> currentCatalogProperties; + private static Map> currentNativeWorkerCatalogConfigProperties; private static Map currentPrestoSparkProperties; private static Map currentEventListenerProperties; private static Map currentAccessControlProperties; @@ -346,6 +352,7 @@ private IPrestoSparkService getOrCreatePrestoSparkService() catalogProperties, prestoSparkProperties, Optional.ofNullable(nativeWorkerConfigProperties), + Optional.ofNullable(nativeWorkerCatalogConfigProperties), Optional.ofNullable(eventListenerProperties), Optional.ofNullable(accessControlProperties), Optional.ofNullable(sessionPropertyConfigurationProperties), @@ -357,6 +364,7 @@ private IPrestoSparkService getOrCreatePrestoSparkService() currentConfigProperties = configProperties; currentNativeWorkerConfigProperties = nativeWorkerConfigProperties; currentCatalogProperties = catalogProperties; + currentNativeWorkerCatalogConfigProperties = nativeWorkerCatalogConfigProperties; currentPrestoSparkProperties = prestoSparkProperties; currentEventListenerProperties = eventListenerProperties; currentAccessControlProperties = accessControlProperties; @@ -369,6 +377,7 @@ private IPrestoSparkService getOrCreatePrestoSparkService() checkEquals("configProperties", currentConfigProperties, configProperties); checkEquals("nativeWorkerConfigProperties", currentNativeWorkerConfigProperties, nativeWorkerConfigProperties); checkEquals("catalogProperties", currentCatalogProperties, catalogProperties); + checkEquals("nativeWorkerCatalogConfigProperties", currentNativeWorkerCatalogConfigProperties, nativeWorkerCatalogConfigProperties); checkEquals("prestoSparkProperties", currentPrestoSparkProperties, prestoSparkProperties); checkEquals("eventListenerProperties", currentEventListenerProperties, eventListenerProperties); checkEquals("accessControlProperties", currentAccessControlProperties, accessControlProperties); diff --git a/presto-spark-package/pom.xml b/presto-spark-package/pom.xml index 44222dece8b37..a9fb149b453b3 100644 --- a/presto-spark-package/pom.xml +++ b/presto-spark-package/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-spark-package diff --git a/presto-spark-testing/pom.xml b/presto-spark-testing/pom.xml index 43ec80cad2aaa..c759233a63b4d 100644 --- a/presto-spark-testing/pom.xml +++ b/presto-spark-testing/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spark/pom.xml b/presto-spark/pom.xml index 070b5f9a99740..1b7aa306824e3 100644 --- a/presto-spark/pom.xml +++ b/presto-spark/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index c01a6c5c56885..7a74b39712b15 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-spi diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/TableHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/TableHandle.java index 51df365849ee0..3dd4c8abfad8e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/TableHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/TableHandle.java @@ -33,9 +33,6 @@ public final class TableHandle private final ConnectorId connectorId; private final ConnectorTableHandle connectorHandle; private final ConnectorTransactionHandle transaction; - - // ConnectorTableHandle will represent the engine's view of data set on a table, we will deprecate ConnectorTableLayoutHandle later. - // TODO remove table layout once it is fully deprecated. private final Optional layout; // This is not serializable; for local execution only diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java index 006495a66a5d0..e8a888bbc8ab9 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/ConnectorMetadata.java @@ -573,12 +573,23 @@ default ConnectorDeleteTableHandle beginDelete(ConnectorSession session, Connect * Finish delete query * * @param fragments all fragments returned by {@link com.facebook.presto.spi.UpdatablePageSource#finish()} + * + * @deprecated Implementors should override {@link #finishDeleteWithOutput(ConnectorSession, ConnectorDeleteTableHandle, Collection)} instead. */ default void finishDelete(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support deletes"); } + /** + * Finish delete query + */ + default Optional finishDeleteWithOutput(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) + { + finishDelete(session, tableHandle, fragments); + return Optional.empty(); + } + default ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTableHandle tableHandle, List updatedColumns) { throw new PrestoException(NOT_SUPPORTED, "This connector does not support update"); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java index 307bbc4d9607b..4d8d3b8bbe3d5 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/classloader/ClassLoaderSafeConnectorMetadata.java @@ -628,6 +628,14 @@ public void finishDelete(ConnectorSession session, ConnectorDeleteTableHandle ta } } + @Override + public Optional finishDeleteWithOutput(ConnectorSession session, ConnectorDeleteTableHandle tableHandle, Collection fragments) + { + try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) { + return delegate.finishDeleteWithOutput(session, tableHandle, fragments); + } + } + @Override public ConnectorTableHandle beginUpdate(ConnectorSession session, ConnectorTableHandle tableHandle, List updatedColumns) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/table/TableArgument.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/table/TableArgument.java index 1fdbdc6c3a0ea..4526f80fd5b76 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/table/TableArgument.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/table/TableArgument.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; import static java.util.Objects.requireNonNull; @@ -69,8 +69,8 @@ public static Builder builder() public static final class Builder { private RowType rowType; - private List partitionBy = Collections.emptyList(); - private List orderBy = Collections.emptyList(); + private List partitionBy = new ArrayList<>(); + private List orderBy = new ArrayList<>(); private Builder() {} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/ConnectorIdentity.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/ConnectorIdentity.java index 839b6aaeede7e..5bf5c44a8eb1e 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/ConnectorIdentity.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/ConnectorIdentity.java @@ -14,11 +14,16 @@ package com.facebook.presto.spi.security; import java.security.Principal; +import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -31,10 +36,11 @@ public class ConnectorIdentity private final Map extraAuthenticators; private final Optional selectedUser; private final Optional reasonForSelect; + private final List certificates; public ConnectorIdentity(String user, Optional principal, Optional role) { - this(user, principal, role, emptyMap(), emptyMap(), Optional.empty(), Optional.empty()); + this(user, principal, role, emptyMap(), emptyMap(), Optional.empty(), Optional.empty(), emptyList()); } public ConnectorIdentity( @@ -45,6 +51,19 @@ public ConnectorIdentity( Map extraAuthenticators, Optional selectedUser, Optional reasonForSelect) + { + this(user, principal, role, extraCredentials, extraAuthenticators, selectedUser, reasonForSelect, emptyList()); + } + + public ConnectorIdentity( + String user, + Optional principal, + Optional role, + Map extraCredentials, + Map extraAuthenticators, + Optional selectedUser, + Optional reasonForSelect, + List certificates) { this.user = requireNonNull(user, "user is null"); this.principal = requireNonNull(principal, "principal is null"); @@ -53,6 +72,7 @@ public ConnectorIdentity( this.extraAuthenticators = unmodifiableMap(new HashMap<>(requireNonNull(extraAuthenticators, "extraAuthenticators is null"))); this.selectedUser = requireNonNull(selectedUser, "selectedUser is null"); this.reasonForSelect = requireNonNull(reasonForSelect, "reasonForSelect is null"); + this.certificates = unmodifiableList(new ArrayList<>(requireNonNull(certificates, "certificates is null"))); } public String getUser() @@ -90,6 +110,11 @@ public Optional getReasonForSelect() return reasonForSelect; } + public List getCertificates() + { + return certificates; + } + @Override public String toString() { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java b/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java index be68b06623cc9..b49202c42748f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/security/Identity.java @@ -14,12 +14,17 @@ package com.facebook.presto.spi.security; import java.security.Principal; +import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -31,6 +36,7 @@ public class Identity private final Map extraCredentials; private final Optional selectedUser; private final Optional reasonForSelect; + private final List certificates; /** * extraAuthenticators is used when short-lived access token has to be refreshed periodically. @@ -49,16 +55,17 @@ public Identity(Identity other) this.extraAuthenticators = other.extraAuthenticators; this.selectedUser = other.selectedUser; this.reasonForSelect = other.reasonForSelect; + this.certificates = other.certificates; } public Identity(String user, Optional principal) { - this(user, principal, emptyMap(), emptyMap(), emptyMap(), Optional.empty(), Optional.empty()); + this(user, principal, emptyMap(), emptyMap(), emptyMap(), Optional.empty(), Optional.empty(), emptyList()); } public Identity(String user, Optional principal, Map extraCredentials) { - this(user, principal, emptyMap(), extraCredentials, emptyMap(), Optional.empty(), Optional.empty()); + this(user, principal, emptyMap(), extraCredentials, emptyMap(), Optional.empty(), Optional.empty(), emptyList()); } public Identity( @@ -69,6 +76,19 @@ public Identity( Map extraAuthenticators, Optional selectedUser, Optional reasonForSelect) + { + this(user, principal, roles, extraCredentials, extraAuthenticators, selectedUser, reasonForSelect, emptyList()); + } + + public Identity( + String user, + Optional principal, + Map roles, + Map extraCredentials, + Map extraAuthenticators, + Optional selectedUser, + Optional reasonForSelect, + List certificates) { this.user = requireNonNull(user, "user is null"); this.principal = requireNonNull(principal, "principal is null"); @@ -77,6 +97,7 @@ public Identity( this.extraAuthenticators = unmodifiableMap(new HashMap<>(requireNonNull(extraAuthenticators, "extraAuthenticators is null"))); this.selectedUser = requireNonNull(selectedUser, "selectedUser is null"); this.reasonForSelect = requireNonNull(reasonForSelect, "reasonForSelect is null"); + this.certificates = unmodifiableList(new ArrayList<>(requireNonNull(certificates, "certificates is null"))); } public String getUser() @@ -114,6 +135,11 @@ public Optional getReasonForSelect() return reasonForSelect; } + public List getCertificates() + { + return certificates; + } + public ConnectorIdentity toConnectorIdentity() { return new ConnectorIdentity( @@ -123,7 +149,8 @@ public ConnectorIdentity toConnectorIdentity() extraCredentials, extraAuthenticators, selectedUser, - reasonForSelect); + reasonForSelect, + certificates); } public ConnectorIdentity toConnectorIdentity(String catalog) @@ -136,7 +163,8 @@ public ConnectorIdentity toConnectorIdentity(String catalog) extraCredentials, extraAuthenticators, selectedUser, - reasonForSelect); + reasonForSelect, + certificates); } @Override diff --git a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/pom.xml b/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/pom.xml index 236864962a0f4..1b6cd432e2e9b 100644 --- a/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/pom.xml +++ b/presto-sql-helpers/presto-native-sql-invoked-functions-plugin/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT ../../pom.xml diff --git a/presto-sql-helpers/presto-sql-invoked-functions-plugin/pom.xml b/presto-sql-helpers/presto-sql-invoked-functions-plugin/pom.xml index dd259ca93855d..239ee134013a4 100644 --- a/presto-sql-helpers/presto-sql-invoked-functions-plugin/pom.xml +++ b/presto-sql-helpers/presto-sql-invoked-functions-plugin/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT ../../pom.xml diff --git a/presto-sqlserver/pom.xml b/presto-sqlserver/pom.xml index 4b6fa6975911b..d2ad74ebf1151 100644 --- a/presto-sqlserver/pom.xml +++ b/presto-sqlserver/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT 4.0.0 diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index e1fd134d18169..402f476635eaa 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-teradata-functions diff --git a/presto-test-coverage/pom.xml b/presto-test-coverage/pom.xml index 849974018f171..3dd90e8e9b737 100644 --- a/presto-test-coverage/pom.xml +++ b/presto-test-coverage/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-test-coverage diff --git a/presto-testing-docker/pom.xml b/presto-testing-docker/pom.xml index 7b982921a12b2..45096037b33d7 100644 --- a/presto-testing-docker/pom.xml +++ b/presto-testing-docker/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-testing-docker diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index dc51f0626e6a0..22f75f0afde58 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-testing-server-launcher diff --git a/presto-testng-services/pom.xml b/presto-testng-services/pom.xml index 426de228c16f0..a1fef3a85e139 100644 --- a/presto-testng-services/pom.xml +++ b/presto-testng-services/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-testng-services diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 963a82a22adad..232db53255b71 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-tests @@ -16,6 +16,7 @@ 5g 17 true + 1.81 @@ -385,6 +386,18 @@ ${project.version} test + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java index 16cbc24331c8b..fe208883a221e 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestEngineOnlyQueries.java @@ -27,12 +27,6 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED; -import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_FUNCTION; -import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_PERCENTAGE; -import static com.facebook.presto.SystemSessionProperties.PUSH_DOWN_FILTER_EXPRESSION_EVALUATION_THROUGH_CROSS_JOIN; -import static com.facebook.presto.SystemSessionProperties.REWRITE_CROSS_JOIN_ARRAY_NOT_CONTAINS_TO_ANTI_JOIN; -import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN; import static com.google.common.base.Preconditions.checkState; import static org.testng.Assert.assertEquals; @@ -113,267 +107,4 @@ public void testLocallyUnrepresentableTimestampLiterals() assertEquals(computeScalar(sql), localTimeThatDidNotExist); // this tests Presto and the QueryRunner assertQuery(sql); // this tests H2QueryRunner } - - @Test - public void testArraySplitIntoChunks() - { - @Language("SQL") String sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5, 6], 2)"; - assertQuery(sql, "values array[array[1, 2], array[3, 4], array[5, 6]]"); - - sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5], 3)"; - assertQuery(sql, "values array[array[1, 2, 3], array[4, 5]]"); - - sql = "select array_split_into_chunks(array[1, 2, 3], 5)"; - assertQuery(sql, "values array[array[1, 2, 3]]"); - - sql = "select array_split_into_chunks(null, 2)"; - assertQuery(sql, "values null"); - - sql = "select array_split_into_chunks(array[1, 2, 3], 0)"; - assertQueryFails(sql, "Invalid slice size: 0. Size must be greater than zero."); - - sql = "select array_split_into_chunks(array[1, 2, 3], -1)"; - assertQueryFails(sql, "Invalid slice size: -1. Size must be greater than zero."); - - sql = "select array_split_into_chunks(array[1, null, 3, null, 5], 2)"; - assertQuery(sql, "values array[array[1, null], array[3, null], array[5]]"); - - sql = "select array_split_into_chunks(array['a', 'b', 'c', 'd'], 2)"; - assertQuery(sql, "values array[array['a', 'b'], array['c', 'd']]"); - - sql = "select array_split_into_chunks(array[1.1, 2.2, 3.3, 4.4, 5.5], 2)"; - assertQuery(sql, "values array[array[1.1, 2.2], array[3.3, 4.4], array[5.5]]"); - - sql = "select array_split_into_chunks(array[null, null, null], 0)"; - assertQueryFails(sql, "Invalid slice size: 0. Size must be greater than zero."); - - sql = "select array_split_into_chunks(array[null, null, null], 2)"; - assertQuery(sql, "values array[array[null, null], array[null]]"); - - sql = "select array_split_into_chunks(array[null, 1, 2], 5)"; - assertQuery(sql, "values array[array[null, 1, 2]]"); - - sql = "select array_split_into_chunks(array[], 0)"; - assertQueryFails(sql, "Invalid slice size: 0. Size must be greater than zero."); - } - - @Test - public void testCrossJoinWithArrayNotContainsCondition() - { - Session enableOptimization = Session.builder(getSession()) - .setSystemProperty(PUSH_DOWN_FILTER_EXPRESSION_EVALUATION_THROUGH_CROSS_JOIN, "REWRITTEN_TO_INNER_JOIN") - .setSystemProperty(REWRITE_CROSS_JOIN_ARRAY_NOT_CONTAINS_TO_ANTI_JOIN, "true") - .build(); - - @Language("SQL") String sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t2.k, t2.v from t2 where not contains((select t1.arr from t1), t2.k)"; - assertQuery(enableOptimization, sql, "values (4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3, 3, null])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t2.k, t2.v from t2 where not contains((select t1.arr from t1), t2.k)"; - assertQuery(enableOptimization, sql, "values (4, 'b')"); - - sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + - "select t1.k, t1.nation from t1 where not contains((select array_agg(name) from nation), t1.nation)"; - assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); - - // array is an expression that needs to be pushed down - sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + - "select t1.k, t1.nation from t1 where not contains(array_distinct((select array_agg(name) from nation)), t1.nation)"; - assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); - - // check not applicable cases for optimization - - // optimization doesn't apply when there are additional columns on array side - sql = "with t1 as (select * from (values (array[1, 1, 3], 10)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 4, 'b')"); - - // optimization doesn't apply for multi-row array tables - sql = "with t1 as (select * from (values (array[1, 2, 3]), (array[4, 5, 6])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.arr, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (array[1,2,3], 4, 'b'), (array[4,5,6], 1, 'a')"); - - // we currently don't support the optimization for cases that didn't come from a subquery - sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.arr, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (array[1,2,3], 4, 'b')"); - - // transform function considered non-deterministic and doesn't get pushed down - sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + - "select t1.k, t1.nation from t1 where not contains(transform((select array_agg(name) from nation), (x) ->lower(x)), lower(t1.nation))"; - assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); - } - - @Test - public void testDefaultSamplingPercent() - { - assertQuery("select key_sampling_percent('abc')", "select 0.56"); - } - - @Test - public void testKeyBasedSampling() - { - String[] queries = { - "select count(1) from orders join lineitem using(orderkey)", - "select count(1) from (select custkey, max(orderkey) from orders group by custkey)", - "select count_if(m >= 1) from (select max(orderkey) over(partition by custkey) m from orders)", - "select cast(m as bigint) from (select sum(totalprice) over(partition by custkey order by comment) m from orders order by 1 desc limit 1)", - "select count(1) from lineitem where orderkey in (select orderkey from orders where length(comment) > 7)", - "select count(1) from lineitem where orderkey not in (select orderkey from orders where length(comment) > 27)", - "select count(1) from (select distinct orderkey, custkey from orders)", - }; - - int[] unsampledResults = {60175, 1000, 15000, 5408941, 60175, 9256, 15000}; - for (int i = 0; i < queries.length; i++) { - assertQuery(queries[i], "select " + unsampledResults[i]); - } - - Session sessionWithKeyBasedSampling = Session.builder(getSession()) - .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") - .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.2") - .build(); - - int[] sampled20PercentResults = {37170, 616, 9189, 5408941, 37170, 5721, 9278}; - for (int i = 0; i < queries.length; i++) { - assertQuery(sessionWithKeyBasedSampling, queries[i], "select " + sampled20PercentResults[i]); - } - - sessionWithKeyBasedSampling = Session.builder(getSession()) - .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") - .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.1") - .build(); - - int[] sampled10PercentResults = {33649, 557, 8377, 4644937, 33649, 5098, 8397}; - for (int i = 0; i < queries.length; i++) { - assertQuery(sessionWithKeyBasedSampling, queries[i], "select " + sampled10PercentResults[i]); - } - } - - @Test - public void testLeftJoinWithArrayContainsCondition() - { - Session enableOptimization = Session.builder(getSession()) - .setSystemProperty(REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN, "ALWAYS_ENABLED") - .build(); - - @Language("SQL") String sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3, null], 10), (array[4, 5, 6, null, null], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11), (array[null, 9], 12)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b'), (null, 'c'), (9, 'd'), (8, 'd')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b'), (null, null, 'c'), (12, 9, 'd'), (null, 8, 'd')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3, null, null], 10), (array[4, 5, 6, null, null], 11), (array[null, 9], 12)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b'), (null, 'c'), (9, 'd'), (8, 'd')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b'), (null, null, 'c'), (12, 9, 'd'), (null, 8, 'd')"); - - sql = "with t1 as (select * from (values (array[1, 1, 3], 10), (array[4, 4, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 1, 3, null, null], 10), (array[4, 4, 6, null, null], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, null, 3], 10), (array[4, null, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (null, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (NULL, NULL, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k) and t1.k > 10"; - assertQuery(enableOptimization, sql, "values (NULL, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select * from (values (array[1, 2, 3], 1), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k) or t1.k = t2.k"; - assertQuery(enableOptimization, sql, "values (1, 1, 'a'), (11, 4, 'b')"); - - sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + - "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on contains(t1.orderkey, o.orderkey) where o.totalprice < 2000"; - // Because the UDF has different names in H2, which is `array_contains` - String h2Sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + - "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on array_contains(t1.orderkey, o.orderkey) where o.totalprice < 2000"; - assertQuery(enableOptimization, sql, h2Sql); - - sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + - "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on contains(t1.orderkey, o.orderkey) and t1.partkey < o.orderkey where o.totalprice < 2000"; - h2Sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + - "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on array_contains(t1.orderkey, o.orderkey) and t1.partkey < o.orderkey where o.totalprice < 2000"; - assertQuery(enableOptimization, sql, h2Sql); - - // Element type and array type does not match - sql = "with t1 as (select * from (values (array[cast(1 as bigint), 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (cast(1 as integer), 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (11, 4, 'b'), (10, 1, 'a')"); - - sql = "with t1 as (select * from (values (array[cast(1 as integer), 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (cast(1 as bigint), 'a'), (4, 'b')) t(k, v)) " + - "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; - assertQuery(enableOptimization, sql, "values (11, 4, 'b'), (10, 1, 'a')"); - } - - @Test - public void testKeyBasedSamplingFunctionError() - { - Session sessionWithKeyBasedSampling = Session.builder(getSession()) - .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") - .setSystemProperty(KEY_BASED_SAMPLING_FUNCTION, "blah") - .build(); - - assertQueryFails(sessionWithKeyBasedSampling, "select count(1) from orders join lineitem using(orderkey)", "Sampling function: blah not cannot be resolved"); - } - - @Test - public void testSamplingJoinChain() - { - Session sessionWithKeyBasedSampling = Session.builder(getSession()) - .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") - .build(); - @Language("SQL") String sql = "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey"; - - assertQuery(sql, "select 60175"); - assertQuery(sessionWithKeyBasedSampling, sql, "select 16185"); - } - - @Test - public void testTry() - { - // Test try with map method and value parameter is optional and argument is an array with null, - // the error should be suppressed and just return null. - assertQuery("SELECT\n" + - " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + - "FROM (\n" + - " VALUES\n" + - " MAP(\n" + - " ARRAY[1, 2], ARRAY[\n" + - " ARRAY[1, null],\n" + - " ARRAY[1, null]\n" + - " ]\n" + - " )\n" + - ") t(c0)", "SELECT NULL"); - - assertQuery("SELECT\n" + - " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + - "FROM (\n" + - " VALUES\n" + - " MAP(\n" + - " ARRAY[1, 2], ARRAY[\n" + - " ARRAY[null, null],\n" + - " ARRAY[1, 2]\n" + - " ]\n" + - " )\n" + - ") t(c0)", "SELECT NULL"); - - // Test try with array method with an input array containing null values. - // the error should be suppressed and just return null. - assertQuery("SELECT TRY(ARRAY_MAX(ARRAY [ARRAY[1, NULL], ARRAY[1, 2]]))", "SELECT NULL"); - } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestSqlInvokedFunctions.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestSqlInvokedFunctions.java new file mode 100644 index 0000000000000..090a6621b81a2 --- /dev/null +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestSqlInvokedFunctions.java @@ -0,0 +1,292 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.Session; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_ENABLED; +import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_FUNCTION; +import static com.facebook.presto.SystemSessionProperties.KEY_BASED_SAMPLING_PERCENTAGE; +import static com.facebook.presto.SystemSessionProperties.PUSH_DOWN_FILTER_EXPRESSION_EVALUATION_THROUGH_CROSS_JOIN; +import static com.facebook.presto.SystemSessionProperties.REWRITE_CROSS_JOIN_ARRAY_NOT_CONTAINS_TO_ANTI_JOIN; +import static com.facebook.presto.SystemSessionProperties.REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN; + +public abstract class AbstractTestSqlInvokedFunctions + extends AbstractTestQueryFramework +{ + @Test + public void testArraySplitIntoChunks() + { + @Language("SQL") String sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5, 6], 2)"; + assertQuery(sql, "values array[array[1, 2], array[3, 4], array[5, 6]]"); + + sql = "select array_split_into_chunks(array[1, 2, 3, 4, 5], 3)"; + assertQuery(sql, "values array[array[1, 2, 3], array[4, 5]]"); + + sql = "select array_split_into_chunks(array[1, 2, 3], 5)"; + assertQuery(sql, "values array[array[1, 2, 3]]"); + + sql = "select array_split_into_chunks(null, 2)"; + assertQuery(sql, "values null"); + + sql = "select array_split_into_chunks(array[1, 2, 3], 0)"; + assertQueryFails(sql, ".*Invalid slice size: 0. Size must be greater than zero.*"); + + sql = "select array_split_into_chunks(array[1, 2, 3], -1)"; + assertQueryFails(sql, ".*Invalid slice size: -1. Size must be greater than zero.*"); + + sql = "select array_split_into_chunks(array[1, null, 3, null, 5], 2)"; + assertQuery(sql, "values array[array[1, null], array[3, null], array[5]]"); + + sql = "select array_split_into_chunks(array['a', 'b', 'c', 'd'], 2)"; + assertQuery(sql, "values array[array['a', 'b'], array['c', 'd']]"); + + sql = "select array_split_into_chunks(array[1.1, 2.2, 3.3, 4.4, 5.5], 2)"; + assertQuery(sql, "values array[array[1.1, 2.2], array[3.3, 4.4], array[5.5]]"); + + sql = "select array_split_into_chunks(array[null, null, null], 0)"; + assertQueryFails(sql, ".*Invalid slice size: 0. Size must be greater than zero.*"); + + sql = "select array_split_into_chunks(array[null, null, null], 2)"; + assertQuery(sql, "values array[array[null, null], array[null]]"); + + sql = "select array_split_into_chunks(array[null, 1, 2], 5)"; + assertQuery(sql, "values array[array[null, 1, 2]]"); + + sql = "select array_split_into_chunks(array[], 0)"; + assertQueryFails(sql, ".*Invalid slice size: 0. Size must be greater than zero.*"); + } + + @Test + public void testCrossJoinWithArrayNotContainsCondition() + { + Session enableOptimization = Session.builder(getSession()) + .setSystemProperty(PUSH_DOWN_FILTER_EXPRESSION_EVALUATION_THROUGH_CROSS_JOIN, "REWRITTEN_TO_INNER_JOIN") + .setSystemProperty(REWRITE_CROSS_JOIN_ARRAY_NOT_CONTAINS_TO_ANTI_JOIN, "true") + .build(); + + @Language("SQL") String sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t2.k, t2.v from t2 where not contains((select t1.arr from t1), t2.k)"; + assertQuery(enableOptimization, sql, "values (4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3, 3, null])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t2.k, t2.v from t2 where not contains((select t1.arr from t1), t2.k)"; + assertQuery(enableOptimization, sql, "values (4, 'b')"); + + sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + + "select t1.k, t1.nation from t1 where not contains((select array_agg(name) from nation), t1.nation)"; + assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); + + // array is an expression that needs to be pushed down + sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + + "select t1.k, t1.nation from t1 where not contains(array_distinct((select array_agg(name) from nation)), t1.nation)"; + assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); + + // check not applicable cases for optimization + + // optimization doesn't apply when there are additional columns on array side + sql = "with t1 as (select * from (values (array[1, 1, 3], 10)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 4, 'b')"); + + // optimization doesn't apply for multi-row array tables + sql = "with t1 as (select * from (values (array[1, 2, 3]), (array[4, 5, 6])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.arr, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (array[1,2,3], 4, 'b'), (array[4,5,6], 1, 'a')"); + + // we currently don't support the optimization for cases that didn't come from a subquery + sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3])) t(arr)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.arr, t2.k, t2.v from t1 join t2 on not contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (array[1,2,3], 4, 'b')"); + + // transform function considered non-deterministic and doesn't get pushed down + sql = "with t1 as (select * from (values (1, 'JAPAN'), (2, 'invalid_nation')) t(k, nation)) " + + "select t1.k, t1.nation from t1 where not contains(transform((select array_agg(name) from nation), (x) ->lower(x)), lower(t1.nation))"; + assertQuery(enableOptimization, sql, "values (2, 'invalid_nation')"); + } + + @Test + public void testDefaultSamplingPercent() + { + assertQuery("select key_sampling_percent('abc')", "select 0.56"); + } + + @Test + public void testKeyBasedSampling() + { + String[] queries = { + "select count(1) from orders join lineitem using(orderkey)", + "select count(1) from (select custkey, max(orderkey) from orders group by custkey)", + "select count_if(m >= 1) from (select max(orderkey) over(partition by custkey) m from orders)", + "select cast(m as bigint) from (select sum(totalprice) over(partition by custkey order by comment) m from orders order by 1 desc limit 1)", + "select count(1) from lineitem where orderkey in (select orderkey from orders where length(comment) > 7)", + "select count(1) from lineitem where orderkey not in (select orderkey from orders where length(comment) > 27)", + "select count(1) from (select distinct orderkey, custkey from orders)", + }; + + int[] unsampledResults = {60175, 1000, 15000, 5408941, 60175, 9256, 15000}; + for (int i = 0; i < queries.length; i++) { + assertQuery(queries[i], "select " + unsampledResults[i]); + } + + Session sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.2") + .build(); + + int[] sampled20PercentResults = {37170, 616, 9189, 5408941, 37170, 5721, 9278}; + for (int i = 0; i < queries.length; i++) { + assertQuery(sessionWithKeyBasedSampling, queries[i], "select " + sampled20PercentResults[i]); + } + + sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .setSystemProperty(KEY_BASED_SAMPLING_PERCENTAGE, "0.1") + .build(); + + int[] sampled10PercentResults = {33649, 557, 8377, 4644937, 33649, 5098, 8397}; + for (int i = 0; i < queries.length; i++) { + assertQuery(sessionWithKeyBasedSampling, queries[i], "select " + sampled10PercentResults[i]); + } + } + + @Test + public void testLeftJoinWithArrayContainsCondition() + { + Session enableOptimization = Session.builder(getSession()) + .setSystemProperty(REWRITE_LEFT_JOIN_ARRAY_CONTAINS_TO_EQUI_JOIN, "ALWAYS_ENABLED") + .build(); + + @Language("SQL") String sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3, null], 10), (array[4, 5, 6, null, null], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11), (array[null, 9], 12)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b'), (null, 'c'), (9, 'd'), (8, 'd')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b'), (null, null, 'c'), (12, 9, 'd'), (null, 8, 'd')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3, null, null], 10), (array[4, 5, 6, null, null], 11), (array[null, 9], 12)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b'), (null, 'c'), (9, 'd'), (8, 'd')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b'), (null, null, 'c'), (12, 9, 'd'), (null, 8, 'd')"); + + sql = "with t1 as (select * from (values (array[1, 1, 3], 10), (array[4, 4, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 1, 3, null, null], 10), (array[4, 4, 6, null, null], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, null, 3], 10), (array[4, null, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (null, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (10, 1, 'a'), (NULL, NULL, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k) and t1.k > 10"; + assertQuery(enableOptimization, sql, "values (NULL, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select * from (values (array[1, 2, 3], 1), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (1, 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k) or t1.k = t2.k"; + assertQuery(enableOptimization, sql, "values (1, 1, 'a'), (11, 4, 'b')"); + + sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + + "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on contains(t1.orderkey, o.orderkey) where o.totalprice < 2000"; + // Because the UDF has different names in H2, which is `array_contains` + String h2Sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + + "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on array_contains(t1.orderkey, o.orderkey) where o.totalprice < 2000"; + assertQuery(enableOptimization, sql, h2Sql); + + sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + + "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on contains(t1.orderkey, o.orderkey) and t1.partkey < o.orderkey where o.totalprice < 2000"; + h2Sql = "with t1 as (select array_agg(orderkey) orderkey, partkey from lineitem l where l.quantity < 5 group by partkey) " + + "select t1.partkey, o.orderkey, o.totalprice from orders o left join t1 on array_contains(t1.orderkey, o.orderkey) and t1.partkey < o.orderkey where o.totalprice < 2000"; + assertQuery(enableOptimization, sql, h2Sql); + + // Element type and array type does not match + sql = "with t1 as (select * from (values (array[cast(1 as bigint), 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (cast(1 as integer), 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (11, 4, 'b'), (10, 1, 'a')"); + + sql = "with t1 as (select * from (values (array[cast(1 as integer), 2, 3], 10), (array[4, 5, 6], 11)) t(arr, k)), t2 as (select * from (values (cast(1 as bigint), 'a'), (4, 'b')) t(k, v)) " + + "select t1.k, t2.k, t2.v from t2 left join t1 on contains(t1.arr, t2.k)"; + assertQuery(enableOptimization, sql, "values (11, 4, 'b'), (10, 1, 'a')"); + } + + @Test + public void testKeyBasedSamplingFunctionError() + { + Session sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .setSystemProperty(KEY_BASED_SAMPLING_FUNCTION, "blah") + .build(); + + assertQueryFails(sessionWithKeyBasedSampling, "select count(1) from orders join lineitem using(orderkey)", "Sampling function: blah not cannot be resolved"); + } + + @Test + public void testSamplingJoinChain() + { + Session sessionWithKeyBasedSampling = Session.builder(getSession()) + .setSystemProperty(KEY_BASED_SAMPLING_ENABLED, "true") + .build(); + @Language("SQL") String sql = "select count(1) FROM lineitem l left JOIN orders o ON l.orderkey = o.orderkey JOIN customer c ON o.custkey = c.custkey"; + + assertQuery(sql, "select 60175"); + assertQuery(sessionWithKeyBasedSampling, sql, "select 16185"); + } + + @Test + public void testTry() + { + // Test try with map method and value parameter is optional and argument is an array with null, + // the error should be suppressed and just return null. + assertQuery("SELECT\n" + + " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + + "FROM (\n" + + " VALUES\n" + + " MAP(\n" + + " ARRAY[1, 2], ARRAY[\n" + + " ARRAY[1, null],\n" + + " ARRAY[1, null]\n" + + " ]\n" + + " )\n" + + ") t(c0)", "SELECT NULL"); + + assertQuery("SELECT\n" + + " TRY(map_keys_by_top_n_values(c0, BIGINT '6455219767830808341'))\n" + + "FROM (\n" + + " VALUES\n" + + " MAP(\n" + + " ARRAY[1, 2], ARRAY[\n" + + " ARRAY[null, null],\n" + + " ARRAY[1, 2]\n" + + " ]\n" + + " )\n" + + ") t(c0)", "SELECT NULL"); + + // Test try with array method with an input array containing null values. + // the error should be suppressed and just return null. + assertQuery("SELECT TRY(ARRAY_MAX(ARRAY [ARRAY[1, NULL], ARRAY[1, 2]]))", "SELECT NULL"); + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 793247eb7a3c9..b9f88018332f8 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -31,6 +31,7 @@ import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.security.AllowAllSystemAccessControl; import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.server.testing.TestingPrestoServer; import com.facebook.presto.spi.ConnectorId; @@ -134,6 +135,8 @@ public class DistributedQueryRunner private final int resourceManagerCount; private final AtomicReference testFunctionNamespacesHandle = new AtomicReference<>(); + private final Map accessControlProperties; + @Deprecated public DistributedQueryRunner(Session defaultSession, int nodeCount) throws Exception @@ -163,7 +166,8 @@ public DistributedQueryRunner(Session defaultSession, int nodeCount, Map dataDirectory, Optional> externalWorkerLauncher, - List extraModules) + List extraModules, + Map accessControlProperties) throws Exception { requireNonNull(defaultSession, "defaultSession is null"); this.extraModules = requireNonNull(extraModules, "extraModules is null"); + this.accessControlProperties = requireNonNull(accessControlProperties, "accessControlProperties is null"); try { long start = nanoTime(); @@ -243,6 +249,7 @@ private DistributedQueryRunner( coordinatorSidecarEnabled, false, skipLoadingResourceGroupConfigurationManager, + false, workerProperties, parserOptions, environment, @@ -273,6 +280,7 @@ private DistributedQueryRunner( false, false, skipLoadingResourceGroupConfigurationManager, + false, rmProperties, parserOptions, environment, @@ -294,6 +302,7 @@ private DistributedQueryRunner( false, false, skipLoadingResourceGroupConfigurationManager, + false, catalogServerProperties, parserOptions, environment, @@ -313,6 +322,7 @@ private DistributedQueryRunner( true, false, skipLoadingResourceGroupConfigurationManager, + false, coordinatorSidecarProperties, parserOptions, environment, @@ -321,6 +331,8 @@ private DistributedQueryRunner( servers.add(coordinatorSidecar.get()); } + final boolean loadDefaultSystemAccessControl = !accessControlProperties.containsKey("access-control.name") || + accessControlProperties.get("access-control.name").equals("allow-all"); for (int i = 0; i < coordinatorCount; i++) { TestingPrestoServer coordinator = closer.register(createTestingPrestoServer( discoveryUrl, @@ -332,6 +344,7 @@ private DistributedQueryRunner( false, true, skipLoadingResourceGroupConfigurationManager, + loadDefaultSystemAccessControl, extraCoordinatorProperties, parserOptions, environment, @@ -464,6 +477,7 @@ private static TestingPrestoServer createTestingPrestoServer( boolean coordinatorSidecarEnabled, boolean coordinator, boolean skipLoadingResourceGroupConfigurationManager, + boolean loadDefaultSystemAccessControl, Map extraProperties, SqlParserOptions parserOptions, String environment, @@ -495,6 +509,7 @@ private static TestingPrestoServer createTestingPrestoServer( coordinatorSidecarEnabled, coordinator, skipLoadingResourceGroupConfigurationManager, + loadDefaultSystemAccessControl, properties, environment, discoveryUri, @@ -798,6 +813,15 @@ public void createTestFunctionNamespace(String catalogName, String schemaName) testFunctionNamespacesHandle.get().execute("INSERT INTO function_namespaces SELECT ?, ?", catalogName, schemaName); } + public void loadSystemAccessControl() + { + for (TestingPrestoServer server : servers) { + if (server.isCoordinator()) { + server.getAccessControl().loadSystemAccessControl(accessControlProperties); + } + } + } + private boolean isConnectorVisibleToAllNodes(ConnectorId connectorId) { if (!externalWorkers.isEmpty()) { @@ -1097,6 +1121,7 @@ public static class Builder private boolean skipLoadingResourceGroupConfigurationManager; private List extraModules = ImmutableList.of(); private int resourceManagerCount = 1; + private Map accessControlProperties = ImmutableMap.of("access-control.name", AllowAllSystemAccessControl.NAME); protected Builder(Session defaultSession) { @@ -1232,6 +1257,12 @@ public Builder setSkipLoadingResourceGroupConfigurationManager(boolean skipLoadi return this; } + public Builder setAccessControlProperties(Map accessControlProperties) + { + this.accessControlProperties = accessControlProperties; + return this; + } + public DistributedQueryRunner build() throws Exception { @@ -1253,7 +1284,8 @@ public DistributedQueryRunner build() environment, dataDirectory, externalWorkerLauncher, - extraModules); + extraModules, + accessControlProperties); } } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/SslKeystoreManager.java b/presto-tests/src/main/java/com/facebook/presto/tests/SslKeystoreManager.java new file mode 100644 index 0000000000000..74bc2de9f579f --- /dev/null +++ b/presto-tests/src/main/java/com/facebook/presto/tests/SslKeystoreManager.java @@ -0,0 +1,177 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.airlift.log.Logger; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Date; + +public class SslKeystoreManager +{ + private static final Logger log = Logger.get(SslKeystoreManager.class); + private static boolean initialized; + private static Path jksFilesPath; + private static File keyStoreFile; + private static File trustStoreFile; + public static final String SSL_STORE_PASSWORD = "123456"; + + private SslKeystoreManager() + { + } + + public static synchronized void initializeKeystoreAndTruststore() + { + try { + if (initialized) { + return; + } + jksFilesPath = Paths.get("src", "test", "resources", "ssl_enable"); + + if (Files.notExists(jksFilesPath)) { + Files.createDirectories(jksFilesPath); + } + + keyStoreFile = jksFilesPath.resolve("keystore.jks").toFile(); + trustStoreFile = jksFilesPath.resolve("truststore.jks").toFile(); + + if (keyStoreFile.exists() && trustStoreFile.exists()) { + initialized = true; + return; + } + generateKeyStoreFiles(); + } + catch (Exception e) { + throw new RuntimeException("Failed to generate keystore files at path: " + jksFilesPath, e); + } + } + + static void generateKeyStoreFiles() throws Exception + { + Security.addProvider(new BouncyCastleProvider()); + + String alias = "tls"; + char[] password = SSL_STORE_PASSWORD.toCharArray(); + String certFile = "server.cer"; + int validityDays = 100000; + + // 1. Generate RSA KeyPair + KeyPair keyPair = generateRSAKeyPair(); + + // 2. Generate self-signed certificate + X509Certificate cert = generateSelfSignedCertificate(keyPair, alias, validityDays); + + // 3. Create Keystore and save key + cert + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null, null); + keyStore.setKeyEntry(alias, keyPair.getPrivate(), password, new Certificate[]{cert}); + + try (FileOutputStream fos = new FileOutputStream(keyStoreFile)) { + keyStore.store(fos, password); + } + + File certFilePath = jksFilesPath.resolve(certFile).toFile(); + + // 4. Export certificate to file (DER encoded) + try (FileOutputStream fos = new FileOutputStream(certFilePath)) { + fos.write(cert.getEncoded()); + } + log.info("Certificate exported to: " + certFilePath); + + // 5. Create truststore and import certificate + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null, null); + trustStore.setCertificateEntry(alias, cert); + + try (FileOutputStream fos = new FileOutputStream(trustStoreFile)) { + trustStore.store(fos, password); + } + } + + private static KeyPair generateRSAKeyPair() throws NoSuchAlgorithmException + { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048); + return keyGen.generateKeyPair(); + } + + private static X509Certificate generateSelfSignedCertificate(KeyPair keyPair, String dn, int validityDays) + throws OperatorCreationException, CertificateException, IOException + { + long now = System.currentTimeMillis(); + Date startDate = new Date(now); + + X500Name issuer = new X500Name("CN=" + dn + ", OU=, O=, L=, ST=, C="); + BigInteger serialNumber = new BigInteger(64, new SecureRandom()); + Date endDate = new Date(now + validityDays * 24L * 60L * 60L * 1000L); + + // Use SHA256withRSA + ContentSigner contentSigner = new JcaContentSignerBuilder("SHA256withRSA") + .build(keyPair.getPrivate()); + + JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder( + issuer, + serialNumber, + startDate, + endDate, + issuer, + keyPair.getPublic()); + + X509CertificateHolder certHolder = certBuilder.build(contentSigner); + return new JcaX509CertificateConverter() + .setProvider("BC") + .getCertificate(certHolder); + } + + public static String getKeystorePath() + { + initializeKeystoreAndTruststore(); + if (keyStoreFile == null || !keyStoreFile.exists()) { + throw new IllegalStateException("Keystore file is not initialized or missing"); + } + return keyStoreFile.getAbsolutePath(); + } + + public static String getTruststorePath() + { + initializeKeystoreAndTruststore(); + if (trustStoreFile == null || !trustStoreFile.exists()) { + throw new IllegalStateException("Truststore file is not initialized or missing"); + } + return trustStoreFile.getAbsolutePath(); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java new file mode 100644 index 0000000000000..a1395ae886ca5 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestCheckAccessPermissionsForQueryTypes.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.tests; + +import com.facebook.presto.security.DenyQueryIntegrityCheckSystemAccessControl; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +public class TestCheckAccessPermissionsForQueryTypes + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + DistributedQueryRunner queryRunner = TpchQueryRunnerBuilder.builder() + .setAccessControlProperties(ImmutableMap.of("access-control.name", DenyQueryIntegrityCheckSystemAccessControl.NAME)).build(); + + queryRunner.loadSystemAccessControl(); + + return queryRunner; + } + + @Override + protected QueryRunner createExpectedQueryRunner() + throws Exception + { + QueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build(); + return queryRunner; + } + + @Test + public void testCheckQueryIntegrityCalls() + { + assertAccessDenied("select * from orders", ".*Query integrity check failed.*"); + assertAccessDenied("analyze orders", ".*Query integrity check failed.*"); + assertAccessDenied("explain analyze select * from orders", ".*Query integrity check failed.*"); + assertAccessDenied("explain select * from orders", ".*Query integrity check failed.*"); + assertAccessDenied("explain (type validate) select * from orders", ".*Query integrity check failed.*"); + assertAccessDenied("CREATE TABLE test_empty (a BIGINT)", ".*Query integrity check failed.*"); + assertAccessDenied("use tpch.tiny", ".*Query integrity check failed.*"); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java index 8d6910ccaa59d..b18173ff0bf20 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestDistributedEngineOnlyQueries.java @@ -13,7 +13,6 @@ */ package com.facebook.presto.tests; -import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; import com.facebook.presto.testing.QueryRunner; import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; @@ -24,8 +23,6 @@ public class TestDistributedEngineOnlyQueries protected QueryRunner createQueryRunner() throws Exception { - QueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build(); - queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); - return queryRunner; + return TpchQueryRunnerBuilder.builder().build(); } } diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlInvokedFunctions.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlInvokedFunctions.java new file mode 100644 index 0000000000000..24b9590ff6c5f --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestSqlInvokedFunctions.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.tpch.TpchQueryRunnerBuilder; + +public class TestSqlInvokedFunctions + extends AbstractTestSqlInvokedFunctions +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + QueryRunner queryRunner = TpchQueryRunnerBuilder.builder().build(); + queryRunner.installPlugin(new SqlInvokedFunctionsPlugin()); + return queryRunner; + } +} diff --git a/presto-thrift-api/pom.xml b/presto-thrift-api/pom.xml index 7392a92950ecc..3608f6e0f24de 100644 --- a/presto-thrift-api/pom.xml +++ b/presto-thrift-api/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-thrift-api diff --git a/presto-thrift-connector/pom.xml b/presto-thrift-connector/pom.xml index 565f3b5fc7183..a8391b11ca11b 100644 --- a/presto-thrift-connector/pom.xml +++ b/presto-thrift-connector/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-thrift-connector diff --git a/presto-thrift-spec/pom.xml b/presto-thrift-spec/pom.xml index d18e524dafd81..e6079576f1b89 100644 --- a/presto-thrift-spec/pom.xml +++ b/presto-thrift-spec/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-thrift-spec diff --git a/presto-thrift-testing-server/pom.xml b/presto-thrift-testing-server/pom.xml index 004555bed0e4f..cdb0339c53470 100644 --- a/presto-thrift-testing-server/pom.xml +++ b/presto-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-thrift-testing-server diff --git a/presto-thrift-testing-udf-server/pom.xml b/presto-thrift-testing-udf-server/pom.xml index a8785e585f9de..d59c6929a456c 100644 --- a/presto-thrift-testing-udf-server/pom.xml +++ b/presto-thrift-testing-udf-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-thrift-testing-udf-server diff --git a/presto-tpcds/pom.xml b/presto-tpcds/pom.xml index 5afffcbc8ac7a..26d2ed201032f 100644 --- a/presto-tpcds/pom.xml +++ b/presto-tpcds/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-tpcds diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 6261c9b55f22e..65e3cfbf19853 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-tpch diff --git a/presto-ui/pom.xml b/presto-ui/pom.xml index 48795de494f40..34b27fef51a57 100644 --- a/presto-ui/pom.xml +++ b/presto-ui/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-ui diff --git a/presto-ui/src/components/QueryResults.jsx b/presto-ui/src/components/QueryResults.jsx index 5c1b3582ba337..071f871858794 100644 --- a/presto-ui/src/components/QueryResults.jsx +++ b/presto-ui/src/components/QueryResults.jsx @@ -38,7 +38,7 @@ export function QueryResults({ results }) { let column = { name: row.name, }; - column.selector = row.type === 'bigint' ? row => row[index].toString() : row => row[index]; + column.selector = row.type === 'bigint' ? row => row[index]?.toString() ?? 'NULL' : row => row[index]; return column; }); }; diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index fa1a823018c7a..bb4b02728e29a 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.295-SNAPSHOT + 0.296-SNAPSHOT presto-verifier diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java index 6693b5f198c61..ed91f2d3323c3 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/rewrite/QueryRewriter.java @@ -35,6 +35,7 @@ import com.facebook.presto.sql.tree.DropTable; import com.facebook.presto.sql.tree.DropView; import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.Insert; import com.facebook.presto.sql.tree.IsNullPredicate; @@ -84,6 +85,7 @@ import static com.facebook.presto.common.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.geospatial.type.GeometryType.GEOMETRY; import static com.facebook.presto.hive.HiveUtil.parsePartitionValue; import static com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionNamesAndValues; import static com.facebook.presto.sql.tree.LikeClause.PropertiesOption.INCLUDING; @@ -424,9 +426,19 @@ private Query rewriteNonStorableColumns(Query query, ResultSetMetaData metadata) checkState(selectItems.size() == columnTypes.size(), "SelectItem count (%s) mismatches column count (%s)", selectItems.size(), columnTypes.size()); for (int i = 0; i < selectItems.size(); i++) { SingleColumn singleColumn = (SingleColumn) selectItems.get(i); - Optional columnTypeRewrite = getColumnTypeRewrite(columnTypes.get(i)); + Type columnType = columnTypes.get(i); + Optional columnTypeRewrite = getColumnTypeRewrite(columnType); if (columnTypeRewrite.isPresent()) { - newItems.add(new SingleColumn(new Cast(singleColumn.getExpression(), columnTypeRewrite.get().getTypeSignature().toString()), singleColumn.getAlias())); + Expression expression = singleColumn.getExpression(); + if (columnType.equals(GEOMETRY)) { + // Geometry type not a Hive writable type, thus it should be converted to VARCHAR type in WKT + // format using the ST_AsText function. + expression = new FunctionCall(QualifiedName.of("ST_AsText"), ImmutableList.of(expression)); + } + else { + expression = new Cast(expression, columnTypeRewrite.get().getTypeSignature().toString()); + } + newItems.add(new SingleColumn(expression, singleColumn.getAlias())); } else { newItems.add(singleColumn); @@ -460,6 +472,9 @@ private Optional getColumnTypeRewrite(Type type) if (type.equals(UNKNOWN)) { return Optional.of(BIGINT); } + if (type.equals(GEOMETRY)) { + return Optional.of(VARCHAR); + } if (type instanceof DecimalType) { return Optional.of(DOUBLE); } diff --git a/redis-hbo-provider/pom.xml b/redis-hbo-provider/pom.xml index fe9d1993878f2..f21c6dac23fc9 100644 --- a/redis-hbo-provider/pom.xml +++ b/redis-hbo-provider/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.295-SNAPSHOT + 0.296-SNAPSHOT