diff --git a/presto-analyzer/pom.xml b/presto-analyzer/pom.xml index 77893fbc54f6d..0a18733f606cf 100644 --- a/presto-analyzer/pom.xml +++ b/presto-analyzer/pom.xml @@ -31,6 +31,11 @@ presto-parser + + com.google.guava + guava + + com.google.code.findbugs jsr305 @@ -38,13 +43,13 @@ - javax.inject - javax.inject + com.google.inject + guice - com.google.guava - guava + javax.inject + javax.inject diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerModule.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerModule.java new file mode 100644 index 0000000000000..16c6cff736144 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerModule.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.multibindings.MapBinder; + +import static com.facebook.presto.sql.analyzer.AnalyzerType.BUILTIN; +import static com.facebook.presto.sql.analyzer.AnalyzerType.NATIVE; +import static com.google.inject.Scopes.SINGLETON; + +public class AnalyzerModule + implements Module +{ + @Override + public void configure(Binder binder) + { + MapBinder queryPreparersByType = MapBinder.newMapBinder(binder, AnalyzerType.class, QueryPreparer.class); + queryPreparersByType.addBinding(BUILTIN).to(BuiltInQueryPreparer.class).in(SINGLETON); + queryPreparersByType.addBinding(NATIVE).to(NativeQueryPreparer.class).in(SINGLETON); + + binder.bind(AnalyzerProvider.class).in(SINGLETON); + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerProvider.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerProvider.java new file mode 100644 index 0000000000000..d15f1436661e4 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerProvider.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.sql.analyzer; + +import com.facebook.presto.spi.PrestoException; + +import javax.inject.Inject; + +import java.util.Map; + +import static com.facebook.presto.spi.StandardErrorCode.UNSUPPORTED_ANALYZER_TYPE; +import static java.util.Objects.requireNonNull; + +/** + * This class provides various interfaces for various functionalities in the analyzer. + * This class can be used to get a specific analyzer implementation for a given analyzer type. + */ +public class AnalyzerProvider +{ + private final Map queryPreparersByType; + + @Inject + public AnalyzerProvider(Map queryPreparersByType) + { + this.queryPreparersByType = requireNonNull(queryPreparersByType, "queryPreparersByType is null"); + } + + public QueryPreparer getQueryPreparer(AnalyzerType analyzerType) + { + requireNonNull(analyzerType, "AnalyzerType is null"); + if (queryPreparersByType.containsKey(analyzerType)) { + return queryPreparersByType.get(analyzerType); + } + + throw new PrestoException(UNSUPPORTED_ANALYZER_TYPE, "Unsupported analyzer type: " + analyzerType); + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerType.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerType.java new file mode 100644 index 0000000000000..d3db09907a19d --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/AnalyzerType.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +/** + * Presto supports various analyzers to support various analyzer functionality. + */ +public enum AnalyzerType +{ + /** + * This is for builtin analyzer. This provides default antlr based parser and inbuilt analyzer to produce Analysis object. + */ + BUILTIN, + /** + * This is for C++ based parser and analyzer. This analyzer is currently under development. + * Please avoid using it, as it may corrupt the session. + */ + NATIVE +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryPreparer.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryPreparer.java new file mode 100644 index 0000000000000..5aa0b465e17c1 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/BuiltInQueryPreparer.java @@ -0,0 +1,166 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import com.facebook.presto.common.resourceGroups.QueryType; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.PrestoWarning; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.analyzer.utils.StatementUtils; +import com.facebook.presto.sql.parser.ParsingException; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.Execute; +import com.facebook.presto.sql.tree.Explain; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.Statement; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.common.WarningHandlingLevel.AS_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.StandardErrorCode.WARNING_AS_ERROR; +import static com.facebook.presto.sql.SqlFormatter.formatSql; +import static com.facebook.presto.sql.analyzer.ConstantExpressionVerifier.verifyExpressionIsConstant; +import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_PARAMETER_USAGE; +import static com.facebook.presto.sql.analyzer.utils.ParameterExtractor.getParameterCount; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +/** + * This query preparer provides builtin functionality. It leverages builtin parser and analyzer. + */ +public class BuiltInQueryPreparer + implements QueryPreparer +{ + private final SqlParser sqlParser; + + @Inject + public BuiltInQueryPreparer(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + @Override + public BuiltInPreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, String query, Map preparedStatements, WarningCollector warningCollector) + throws ParsingException, PrestoException, SemanticException + { + Statement wrappedStatement = sqlParser.createStatement(query, analyzerOptions.getParsingOptions()); + if (warningCollector.hasWarnings() && analyzerOptions.getWarningHandlingLevel() == AS_ERROR) { + throw new PrestoException(WARNING_AS_ERROR, format("Warning handling level set to AS_ERROR. Warnings: %n %s", + warningCollector.getWarnings().stream() + .map(PrestoWarning::getMessage) + .collect(joining(System.lineSeparator())))); + } + return prepareQuery(analyzerOptions, wrappedStatement, preparedStatements); + } + + public BuiltInPreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, Statement wrappedStatement, Map preparedStatements) + throws ParsingException, PrestoException, SemanticException + { + Statement statement = wrappedStatement; + Optional prepareSql = Optional.empty(); + if (statement instanceof Execute) { + String preparedStatementName = ((Execute) statement).getName().getValue(); + prepareSql = Optional.ofNullable(preparedStatements.get(preparedStatementName)); + String query = prepareSql.orElseThrow(() -> new PrestoException(NOT_FOUND, "Prepared statement not found: " + preparedStatementName)); + statement = sqlParser.createStatement(query, analyzerOptions.getParsingOptions()); + } + + if (statement instanceof Explain && ((Explain) statement).isAnalyze()) { + Statement innerStatement = ((Explain) statement).getStatement(); + Optional innerQueryType = StatementUtils.getQueryType(innerStatement.getClass()); + if (!innerQueryType.isPresent() || innerQueryType.get() == QueryType.DATA_DEFINITION || innerQueryType.get() == QueryType.CONTROL) { + throw new PrestoException(NOT_SUPPORTED, "EXPLAIN ANALYZE doesn't support statement type: " + innerStatement.getClass().getSimpleName()); + } + } + List parameters = ImmutableList.of(); + if (wrappedStatement instanceof Execute) { + parameters = ((Execute) wrappedStatement).getParameters(); + } + validateParameters(statement, parameters); + Optional formattedQuery = Optional.empty(); + if (analyzerOptions.isLogFormattedQueryEnabled()) { + formattedQuery = Optional.of(getFormattedQuery(statement, parameters)); + } + return new BuiltInPreparedQuery(wrappedStatement, statement, parameters, formattedQuery, prepareSql); + } + + private static String getFormattedQuery(Statement statement, List parameters) + { + String formattedQuery = formatSql( + statement, + parameters.isEmpty() ? Optional.empty() : Optional.of(parameters)); + return format("-- Formatted Query:%n%s", formattedQuery); + } + + private static void validateParameters(Statement node, List parameterValues) + { + int parameterCount = getParameterCount(node); + if (parameterValues.size() != parameterCount) { + throw new SemanticException(INVALID_PARAMETER_USAGE, node, "Incorrect number of parameters: expected %s but found %s", parameterCount, parameterValues.size()); + } + for (Expression expression : parameterValues) { + verifyExpressionIsConstant(ImmutableSet.of(), expression); + } + } + + public static class BuiltInPreparedQuery + extends PreparedQuery + { + private final Statement statement; + private final Statement wrappedStatement; + private final List parameters; + + public BuiltInPreparedQuery(Statement wrappedStatement, Statement statement, List parameters, Optional formattedQuery, Optional prepareSql) + { + super(formattedQuery, prepareSql); + this.wrappedStatement = requireNonNull(wrappedStatement, "wrappedStatement is null"); + this.statement = requireNonNull(statement, "statement is null"); + this.parameters = ImmutableList.copyOf(requireNonNull(parameters, "parameters is null")); + } + + public Statement getStatement() + { + return statement; + } + + public Statement getWrappedStatement() + { + return wrappedStatement; + } + + public List getParameters() + { + return parameters; + } + + public Optional getQueryType() + { + return StatementUtils.getQueryType(statement.getClass()); + } + + public boolean isTransactionControlStatement() + { + return StatementUtils.isTransactionControlStatement(getStatement()); + } + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativePreparedQuery.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativePreparedQuery.java new file mode 100644 index 0000000000000..771d44ffbbd61 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativePreparedQuery.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import java.util.Optional; + +public class NativePreparedQuery + extends PreparedQuery +{ + // TODO: Dummy implementation. This should be replaced with native implementation. + public NativePreparedQuery(Optional formattedQuery, Optional prepareSql) + { + super(formattedQuery, prepareSql); + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativeQueryPreparer.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativeQueryPreparer.java new file mode 100644 index 0000000000000..b3713811af4e3 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/NativeQueryPreparer.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.parser.ParsingException; + +import java.util.Map; +import java.util.Optional; + +public class NativeQueryPreparer + implements QueryPreparer +{ + @Override + public PreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, String query, Map preparedStatements, WarningCollector warningCollector) + throws ParsingException, PrestoException, SemanticException + { + return new NativePreparedQuery(Optional.of(query), Optional.of(query)); + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/PreparedQuery.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/PreparedQuery.java new file mode 100644 index 0000000000000..3a03d9e2c2cb2 --- /dev/null +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/PreparedQuery.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.analyzer; + +import com.facebook.presto.common.resourceGroups.QueryType; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public abstract class PreparedQuery +{ + private final Optional formattedQuery; + private final Optional prepareSql; + + public PreparedQuery(Optional formattedQuery, Optional prepareSql) + { + this.formattedQuery = requireNonNull(formattedQuery, "formattedQuery is null"); + this.prepareSql = requireNonNull(prepareSql, "prepareSql is null"); + } + + public Optional getFormattedQuery() + { + return formattedQuery; + } + + public Optional getPrepareSql() + { + return prepareSql; + } + + public Optional getQueryType() + { + throw new UnsupportedOperationException("getQueryType method is not supported!"); + } + + public boolean isTransactionControlStatement() + { + throw new UnsupportedOperationException("isTransactionControlStatement method is not supported!"); + } + + public Class getStatementClass() + { + throw new UnsupportedOperationException("getStatementClass method is not supported!"); + } +} diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/QueryPreparer.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/QueryPreparer.java index 1d3f62ded10d6..e530715d5c1eb 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/QueryPreparer.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/QueryPreparer.java @@ -13,151 +13,28 @@ */ package com.facebook.presto.sql.analyzer; -import com.facebook.presto.common.resourceGroups.QueryType; import com.facebook.presto.spi.PrestoException; -import com.facebook.presto.spi.PrestoWarning; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.analyzer.utils.StatementUtils; import com.facebook.presto.sql.parser.ParsingException; -import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.tree.Execute; -import com.facebook.presto.sql.tree.Explain; -import com.facebook.presto.sql.tree.Expression; -import com.facebook.presto.sql.tree.Statement; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import javax.inject.Inject; - -import java.util.List; import java.util.Map; -import java.util.Optional; - -import static com.facebook.presto.common.WarningHandlingLevel.AS_ERROR; -import static com.facebook.presto.spi.StandardErrorCode.NOT_FOUND; -import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.spi.StandardErrorCode.WARNING_AS_ERROR; -import static com.facebook.presto.sql.SqlFormatter.formatSql; -import static com.facebook.presto.sql.analyzer.ConstantExpressionVerifier.verifyExpressionIsConstant; -import static com.facebook.presto.sql.analyzer.SemanticErrorCode.INVALID_PARAMETER_USAGE; -import static com.facebook.presto.sql.analyzer.utils.ParameterExtractor.getParameterCount; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.joining; -public class QueryPreparer +/** + * QueryPreparer interface should be implemented by the analyzer to provide prepare query related functionality. + */ +public interface QueryPreparer { - private final SqlParser sqlParser; - - @Inject - public QueryPreparer(SqlParser sqlParser) - { - this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); - } - - public PreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, String query, Map preparedStatements, WarningCollector warningCollector) - throws ParsingException, PrestoException, SemanticException - { - Statement wrappedStatement = sqlParser.createStatement(query, analyzerOptions.getParsingOptions()); - if (warningCollector.hasWarnings() && analyzerOptions.getWarningHandlingLevel() == AS_ERROR) { - throw new PrestoException(WARNING_AS_ERROR, format("Warning handling level set to AS_ERROR. Warnings: %n %s", - warningCollector.getWarnings().stream() - .map(PrestoWarning::getMessage) - .collect(joining(System.lineSeparator())))); - } - return prepareQuery(analyzerOptions, wrappedStatement, preparedStatements); - } - - public PreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, Statement wrappedStatement, Map preparedStatements) - throws ParsingException, PrestoException, SemanticException - { - Statement statement = wrappedStatement; - Optional prepareSql = Optional.empty(); - if (statement instanceof Execute) { - String preparedStatementName = ((Execute) statement).getName().getValue(); - prepareSql = Optional.ofNullable(preparedStatements.get(preparedStatementName)); - String query = prepareSql.orElseThrow(() -> new PrestoException(NOT_FOUND, "Prepared statement not found: " + preparedStatementName)); - statement = sqlParser.createStatement(query, analyzerOptions.getParsingOptions()); - } - - if (statement instanceof Explain && ((Explain) statement).isAnalyze()) { - Statement innerStatement = ((Explain) statement).getStatement(); - Optional innerQueryType = StatementUtils.getQueryType(innerStatement.getClass()); - if (!innerQueryType.isPresent() || innerQueryType.get() == QueryType.DATA_DEFINITION) { - throw new PrestoException(NOT_SUPPORTED, "EXPLAIN ANALYZE doesn't support statement type: " + innerStatement.getClass().getSimpleName()); - } - } - List parameters = ImmutableList.of(); - if (wrappedStatement instanceof Execute) { - parameters = ((Execute) wrappedStatement).getParameters(); - } - validateParameters(statement, parameters); - Optional formattedQuery = Optional.empty(); - if (analyzerOptions.isLogFormattedQueryEnabled()) { - formattedQuery = Optional.of(getFormattedQuery(statement, parameters)); - } - return new PreparedQuery(wrappedStatement, statement, parameters, formattedQuery, prepareSql); - } - - private static String getFormattedQuery(Statement statement, List parameters) - { - String formattedQuery = formatSql( - statement, - parameters.isEmpty() ? Optional.empty() : Optional.of(parameters)); - return format("-- Formatted Query:%n%s", formattedQuery); - } - - private static void validateParameters(Statement node, List parameterValues) - { - int parameterCount = getParameterCount(node); - if (parameterValues.size() != parameterCount) { - throw new SemanticException(INVALID_PARAMETER_USAGE, node, "Incorrect number of parameters: expected %s but found %s", parameterCount, parameterValues.size()); - } - for (Expression expression : parameterValues) { - verifyExpressionIsConstant(ImmutableSet.of(), expression); - } - } - - public static class PreparedQuery - { - private final Statement statement; - private final Statement wrappedStatement; - private final List parameters; - private final Optional formattedQuery; - private final Optional prepareSql; - - public PreparedQuery(Statement wrappedStatement, Statement statement, List parameters, Optional formattedQuery, Optional prepareSql) - { - this.wrappedStatement = requireNonNull(wrappedStatement, "wrappedStatement is null"); - this.statement = requireNonNull(statement, "statement is null"); - this.parameters = ImmutableList.copyOf(requireNonNull(parameters, "parameters is null")); - this.formattedQuery = requireNonNull(formattedQuery, "formattedQuery is null"); - this.prepareSql = requireNonNull(prepareSql, "prepareSql is null"); - } - - public Statement getStatement() - { - return statement; - } - - public Statement getWrappedStatement() - { - return wrappedStatement; - } - - public List getParameters() - { - return parameters; - } - - public Optional getFormattedQuery() - { - return formattedQuery; - } - - public Optional getPrepareSql() - { - return prepareSql; - } - } + /** + * This method should do the necessary work required to prepare the query and return the preparedQuery object. + * @param analyzerOptions various analyzer options required to parse and analyze the query + * @param query query string which needs to be prepared + * @param preparedStatements existing prepared query statements + * @param warningCollector Warning collector to collect various warnings + * @return preared query object + * @throws ParsingException + * @throws PrestoException + * @throws SemanticException + */ + PreparedQuery prepareQuery(AnalyzerOptions analyzerOptions, String query, Map preparedStatements, WarningCollector warningCollector) + throws ParsingException, PrestoException, SemanticException; } diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java index bfa22909facb1..8a4c37c35e52b 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/utils/StatementUtils.java @@ -70,23 +70,23 @@ import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.sql.tree.TruncateTable; import com.facebook.presto.sql.tree.Use; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.List; +import java.util.HashSet; import java.util.Map; import java.util.Optional; +import java.util.Set; + +import static java.util.Collections.unmodifiableSet; public final class StatementUtils { private StatementUtils() {} private static final Map, QueryType> STATEMENT_QUERY_TYPES; - private static final List> SESSION_TRANSACTION_CONTROL_TYPES; static { ImmutableMap.Builder, QueryType> builder = ImmutableMap.builder(); - ImmutableList.Builder> sessionTransactionBuilder = ImmutableList.builder(); builder.put(Query.class, QueryType.SELECT); @@ -132,40 +132,29 @@ private StatementUtils() {} builder.put(CreateFunction.class, QueryType.DATA_DEFINITION); builder.put(AlterFunction.class, QueryType.DATA_DEFINITION); builder.put(DropFunction.class, QueryType.DATA_DEFINITION); - builder.put(Use.class, QueryType.DATA_DEFINITION); - builder.put(SetSession.class, QueryType.DATA_DEFINITION); - builder.put(ResetSession.class, QueryType.DATA_DEFINITION); - builder.put(StartTransaction.class, QueryType.DATA_DEFINITION); - builder.put(Commit.class, QueryType.DATA_DEFINITION); - builder.put(Rollback.class, QueryType.DATA_DEFINITION); + builder.put(Use.class, QueryType.CONTROL); + builder.put(SetSession.class, QueryType.CONTROL); + builder.put(ResetSession.class, QueryType.CONTROL); + builder.put(StartTransaction.class, QueryType.CONTROL); + builder.put(Commit.class, QueryType.CONTROL); + builder.put(Rollback.class, QueryType.CONTROL); builder.put(Call.class, QueryType.DATA_DEFINITION); builder.put(CreateRole.class, QueryType.DATA_DEFINITION); builder.put(DropRole.class, QueryType.DATA_DEFINITION); builder.put(GrantRoles.class, QueryType.DATA_DEFINITION); builder.put(RevokeRoles.class, QueryType.DATA_DEFINITION); - builder.put(SetRole.class, QueryType.DATA_DEFINITION); + builder.put(SetRole.class, QueryType.CONTROL); builder.put(Grant.class, QueryType.DATA_DEFINITION); builder.put(Revoke.class, QueryType.DATA_DEFINITION); - builder.put(Prepare.class, QueryType.DATA_DEFINITION); - builder.put(Deallocate.class, QueryType.DATA_DEFINITION); - - sessionTransactionBuilder.add(Use.class); - sessionTransactionBuilder.add(SetSession.class); - sessionTransactionBuilder.add(ResetSession.class); - sessionTransactionBuilder.add(SetRole.class); - sessionTransactionBuilder.add(StartTransaction.class); - sessionTransactionBuilder.add(Commit.class); - sessionTransactionBuilder.add(Rollback.class); - sessionTransactionBuilder.add(Prepare.class); - sessionTransactionBuilder.add(Deallocate.class); + builder.put(Prepare.class, QueryType.CONTROL); + builder.put(Deallocate.class, QueryType.CONTROL); STATEMENT_QUERY_TYPES = builder.build(); - SESSION_TRANSACTION_CONTROL_TYPES = sessionTransactionBuilder.build(); } - public static Map, QueryType> getAllQueryTypes() + public static Set getAllQueryTypes() { - return STATEMENT_QUERY_TYPES; + return unmodifiableSet(new HashSet<>(STATEMENT_QUERY_TYPES.values())); } public static Optional getQueryType(Class statement) @@ -177,9 +166,4 @@ public static boolean isTransactionControlStatement(Statement statement) { return statement instanceof StartTransaction || statement instanceof Commit || statement instanceof Rollback; } - - public static boolean isSessionTransactionControlStatement(Class statement) - { - return SESSION_TRANSACTION_CONTROL_TYPES.contains(statement); - } } diff --git a/presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestQueryPreparer.java b/presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestBuiltInQueryPreparer.java similarity index 88% rename from presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestQueryPreparer.java rename to presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestBuiltInQueryPreparer.java index 5cf5030d6d502..e7de6d2cc1dad 100644 --- a/presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestQueryPreparer.java +++ b/presto-analyzer/src/test/java/com/facebook/presto/sql/analyzer/TestBuiltInQueryPreparer.java @@ -15,7 +15,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.AllColumns; import com.facebook.presto.sql.tree.QualifiedName; @@ -33,17 +33,17 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.fail; -public class TestQueryPreparer +public class TestBuiltInQueryPreparer { private static final SqlParser SQL_PARSER = new SqlParser(); - private static final QueryPreparer QUERY_PREPARER = new QueryPreparer(SQL_PARSER); + private static final BuiltInQueryPreparer QUERY_PREPARER = new BuiltInQueryPreparer(SQL_PARSER); private static final Map emptyPreparedStatements = ImmutableMap.of(); private static final AnalyzerOptions testAnalyzerOptions = AnalyzerOptions.builder().build(); @Test public void testSelectStatement() { - PreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(testAnalyzerOptions, "SELECT * FROM foo", emptyPreparedStatements, WarningCollector.NOOP); + BuiltInPreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(testAnalyzerOptions, "SELECT * FROM foo", emptyPreparedStatements, WarningCollector.NOOP); assertEquals(preparedQuery.getStatement(), simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); } @@ -52,7 +52,7 @@ public void testSelectStatement() public void testExecuteStatement() { Map preparedStatements = ImmutableMap.of("my_query", "SELECT * FROM foo"); - PreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(testAnalyzerOptions, "EXECUTE my_query", preparedStatements, WarningCollector.NOOP); + BuiltInPreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(testAnalyzerOptions, "EXECUTE my_query", preparedStatements, WarningCollector.NOOP); assertEquals(preparedQuery.getStatement(), simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); } @@ -99,7 +99,7 @@ public void testTooFewParameters() public void testFormattedQuery() { AnalyzerOptions analyzerOptions = AnalyzerOptions.builder().setLogFormattedQueryEnabled(true).build(); - PreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery( + BuiltInPreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery( analyzerOptions, "PREPARE test FROM SELECT * FROM foo where col1 = ?", emptyPreparedStatements, diff --git a/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java b/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java index 93d4afb84db13..e0353c6989223 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/resourceGroups/QueryType.java @@ -26,6 +26,7 @@ public enum QueryType ANALYZE(5), INSERT(6), SELECT(7), + CONTROL(8), /**/; private final int value; diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 46400db12ae95..f8eea8c97a20e 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -26,10 +26,10 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.session.PropertyMetadata; import com.facebook.presto.spiller.NodeSpillConfig; +import com.facebook.presto.sql.analyzer.AnalyzerType; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationIfToFilterRewriteStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy; -import com.facebook.presto.sql.analyzer.FeaturesConfig.AnalyzerType; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy; diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchManager.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchManager.java index ca02cd98bb1c2..ab06d4f142da0 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchManager.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchManager.java @@ -39,8 +39,8 @@ import com.facebook.presto.spi.security.AuthorizedIdentity; import com.facebook.presto.spi.security.Identity; import com.facebook.presto.sql.analyzer.AnalyzerOptions; -import com.facebook.presto.sql.analyzer.QueryPreparer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; +import com.facebook.presto.sql.analyzer.AnalyzerProvider; +import com.facebook.presto.sql.analyzer.PreparedQuery; import com.facebook.presto.transaction.TransactionManager; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; @@ -55,9 +55,8 @@ import java.util.Optional; import java.util.concurrent.Executor; +import static com.facebook.presto.SystemSessionProperties.getAnalyzerType; import static com.facebook.presto.spi.StandardErrorCode.QUERY_TEXT_TOO_LARGE; -import static com.facebook.presto.sql.analyzer.utils.StatementUtils.getQueryType; -import static com.facebook.presto.sql.analyzer.utils.StatementUtils.isTransactionControlStatement; import static com.facebook.presto.util.AnalyzerUtil.createAnalyzerOptions; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -68,7 +67,7 @@ public class DispatchManager { private final QueryIdGenerator queryIdGenerator; - private final QueryPreparer queryPreparer; + private final AnalyzerProvider analyzerProvider; private final ResourceGroupManager resourceGroupManager; private final WarningCollectorFactory warningCollectorFactory; private final DispatchQueryFactory dispatchQueryFactory; @@ -94,7 +93,7 @@ public class DispatchManager @Inject public DispatchManager( QueryIdGenerator queryIdGenerator, - QueryPreparer queryPreparer, + AnalyzerProvider analyzerProvider, @SuppressWarnings("rawtypes") ResourceGroupManager resourceGroupManager, WarningCollectorFactory warningCollectorFactory, DispatchQueryFactory dispatchQueryFactory, @@ -109,7 +108,7 @@ public DispatchManager( SecurityConfig securityConfig) { this.queryIdGenerator = requireNonNull(queryIdGenerator, "queryIdGenerator is null"); - this.queryPreparer = requireNonNull(queryPreparer, "queryPreparer is null"); + this.analyzerProvider = requireNonNull(analyzerProvider, "analyzerClient is null"); this.resourceGroupManager = requireNonNull(resourceGroupManager, "resourceGroupManager is null"); this.warningCollectorFactory = requireNonNull(warningCollectorFactory, "warningCollectorFactory is null"); this.dispatchQueryFactory = requireNonNull(dispatchQueryFactory, "dispatchQueryFactory is null"); @@ -201,11 +200,11 @@ private void createQueryInternal(QueryId queryId, String slug, int retryCoun // prepare query AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, session.getWarningCollector()); - preparedQuery = queryPreparer.prepareQuery(analyzerOptions, query, session.getPreparedStatements(), session.getWarningCollector()); + preparedQuery = analyzerProvider.getQueryPreparer(getAnalyzerType(session)).prepareQuery(analyzerOptions, query, session.getPreparedStatements(), session.getWarningCollector()); query = preparedQuery.getFormattedQuery().orElse(query); // select resource group - Optional queryType = getQueryType(preparedQuery.getStatement().getClass()); + Optional queryType = preparedQuery.getQueryType(); SelectionContext selectionContext = resourceGroupManager.selectGroup(new SelectionCriteria( sessionContext.getIdentity().getPrincipal().isPresent(), sessionContext.getIdentity().getUser(), @@ -219,7 +218,7 @@ private void createQueryInternal(QueryId queryId, String slug, int retryCoun session = sessionPropertyDefaults.newSessionWithDefaultProperties(session, queryType.map(Enum::name), Optional.of(selectionContext.getResourceGroupId())); // mark existing transaction as active - transactionManager.activateTransaction(session, isTransactionControlStatement(preparedQuery.getStatement()), accessControl); + transactionManager.activateTransaction(session, preparedQuery.isTransactionControlStatement(), accessControl); DispatchQuery dispatchQuery = dispatchQueryFactory.createDispatchQuery( session, @@ -230,7 +229,7 @@ private void createQueryInternal(QueryId queryId, String slug, int retryCoun selectionContext.getResourceGroupId(), queryType, session.getWarningCollector(), - (dq) -> resourceGroupManager.submit(preparedQuery.getStatement(), dq, selectionContext, queryExecutor)); + (dq) -> resourceGroupManager.submit(dq, selectionContext, queryExecutor)); boolean queryAdded = queryCreated(dispatchQuery); if (queryAdded && !dispatchQuery.isDone()) { diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchQueryFactory.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchQueryFactory.java index 8ed239150d71f..e55bd99faa456 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchQueryFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/DispatchQueryFactory.java @@ -17,7 +17,7 @@ import com.facebook.presto.common.resourceGroups.QueryType; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; +import com.facebook.presto.sql.analyzer.PreparedQuery; import java.util.Optional; import java.util.function.Consumer; diff --git a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java index 5e685c20af88a..188abb6b51411 100644 --- a/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/dispatcher/LocalDispatchQueryFactory.java @@ -27,8 +27,8 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; -import com.facebook.presto.sql.tree.Statement; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.PreparedQuery; import com.facebook.presto.tracing.NoopTracerProvider; import com.facebook.presto.tracing.QueryStateTracingListener; import com.facebook.presto.transaction.TransactionManager; @@ -42,7 +42,7 @@ import java.util.function.Consumer; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.sql.analyzer.utils.StatementUtils.isTransactionControlStatement; +import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class LocalDispatchQueryFactory @@ -57,7 +57,7 @@ public class LocalDispatchQueryFactory private final ClusterSizeMonitor clusterSizeMonitor; - private final Map, QueryExecutionFactory> executionFactories; + private final Map> executionFactories; private final ListeningExecutorService executor; private final QueryPrerequisitesManager queryPrerequisitesManager; @@ -70,7 +70,7 @@ public LocalDispatchQueryFactory( Metadata metadata, QueryMonitor queryMonitor, LocationFactory locationFactory, - Map, QueryExecutionFactory> executionFactories, + Map> executionFactories, ClusterSizeMonitor clusterSizeMonitor, DispatchExecutor dispatchExecutor, QueryPrerequisitesManager queryPrerequisitesManager) @@ -108,7 +108,7 @@ public DispatchQuery createDispatchQuery( locationFactory.createQueryLocation(session.getQueryId()), resourceGroup, queryType, - isTransactionControlStatement(preparedQuery.getStatement()), + preparedQuery.isTransactionControlStatement(), transactionManager, accessControl, executor, @@ -119,12 +119,14 @@ public DispatchQuery createDispatchQuery( queryMonitor.queryCreatedEvent(stateMachine.getBasicQueryInfo(Optional.empty())); ListenableFuture queryExecutionFuture = executor.submit(() -> { - QueryExecutionFactory queryExecutionFactory = executionFactories.get(preparedQuery.getStatement().getClass()); + QueryExecutionFactory queryExecutionFactory = executionFactories.get(queryType.get()); if (queryExecutionFactory == null) { - throw new PrestoException(NOT_SUPPORTED, "Unsupported statement type: " + preparedQuery.getStatement().getClass().getSimpleName()); + throw new PrestoException(NOT_SUPPORTED, "Unsupported statement type: " + preparedQuery.getStatementClass().getSimpleName()); } - return queryExecutionFactory.createQueryExecution(preparedQuery, stateMachine, slug, retryCount, warningCollector, queryType); + //TODO: PreparedQuery should be passed all the way to analyzer + checkState(preparedQuery instanceof BuiltInQueryPreparer.BuiltInPreparedQuery, "Unsupported prepared query type: %s", preparedQuery.getClass().getSimpleName()); + return queryExecutionFactory.createQueryExecution((BuiltInQueryPreparer.BuiltInPreparedQuery) preparedQuery, stateMachine, slug, retryCount, warningCollector, queryType); }); return new LocalDispatchQuery( diff --git a/presto-main/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java index 6140f92a58dcb..3657cdc8550f2 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/DDLDefinitionExecution.java @@ -18,7 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.analyzer.QueryPreparer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; @@ -82,7 +82,7 @@ public DDLDefinitionExecutionFactory( @Override public DDLDefinitionExecution createQueryExecution( - QueryPreparer.PreparedQuery preparedQuery, + BuiltInQueryPreparer.BuiltInPreparedQuery preparedQuery, QueryStateMachine stateMachine, String slug, int retryCount, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/QueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/QueryExecution.java index a8a5f1d255aef..51d222d63cfe0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/QueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/QueryExecution.java @@ -21,7 +21,7 @@ import com.facebook.presto.server.BasicQueryInfo; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.resourceGroups.ResourceGroupQueryLimits; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.planner.Plan; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -98,7 +98,7 @@ public interface QueryExecution interface QueryExecutionFactory { T createQueryExecution( - PreparedQuery preparedQuery, + BuiltInPreparedQuery preparedQuery, QueryStateMachine stateMachine, String slug, int retryCount, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java index 9d2805e3c0e00..157a849d773b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SessionDefinitionExecution.java @@ -18,7 +18,7 @@ import com.facebook.presto.metadata.MetadataManager; import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.WarningCollector; -import com.facebook.presto.sql.analyzer.QueryPreparer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.TransactionManager; @@ -82,7 +82,7 @@ public SessionDefinitionExecutionFactory( @Override public SessionDefinitionExecution createQueryExecution( - QueryPreparer.PreparedQuery preparedQuery, + BuiltInQueryPreparer.BuiltInPreparedQuery preparedQuery, QueryStateMachine stateMachine, String slug, int retryCount, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java index b7383b9877b76..d9d4dac9bb50d 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/SqlQueryExecution.java @@ -44,8 +44,8 @@ import com.facebook.presto.split.SplitManager; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.InputExtractor; import com.facebook.presto.sql.planner.LogicalPlanner; @@ -136,7 +136,7 @@ public class SqlQueryExecution private final PlanCanonicalInfoProvider planCanonicalInfoProvider; private SqlQueryExecution( - PreparedQuery preparedQuery, + BuiltInPreparedQuery preparedQuery, QueryStateMachine stateMachine, String slug, int retryCount, @@ -401,8 +401,8 @@ public void start() Thread.currentThread(), timeoutThreadExecutor, getQueryAnalyzerTimeout(getSession()))) { - // analyze query - plan = analyzeQuery(); + // create logical plan for the query + plan = createLogicalPlan(); } metadata.beginQuery(getSession(), plan.getConnectors()); @@ -464,52 +464,47 @@ public void addFinalQueryInfoListener(StateChangeListener stateChange stateMachine.addQueryInfoStateChangeListener(stateChangeListener); } - private PlanRoot analyzeQuery() + private PlanRoot createLogicalPlan() { try { - return doAnalyzeQuery(); + // time analysis phase + stateMachine.beginAnalysis(); + + // plan query + LogicalPlanner logicalPlanner = new LogicalPlanner(false, stateMachine.getSession(), planOptimizers, idAllocator, metadata, sqlParser, statsCalculator, costCalculator, stateMachine.getWarningCollector(), planChecker); + Plan plan = getSession().getRuntimeStats().profileNanos( + LOGICAL_PLANNER_TIME_NANOS, + () -> logicalPlanner.plan(analysis)); + queryPlan.set(plan); + stateMachine.setPlanStatsAndCosts(plan.getStatsAndCosts()); + stateMachine.setPlanCanonicalInfo(getCanonicalInfo(getSession(), plan.getRoot(), planCanonicalInfoProvider)); + + // extract inputs + List inputs = new InputExtractor(metadata, stateMachine.getSession()).extractInputs(plan.getRoot()); + stateMachine.setInputs(inputs); + + // extract output + Optional output = new OutputExtractor().extractOutput(plan.getRoot()); + stateMachine.setOutput(output); + + // fragment the plan + // the variableAllocator is finally passed to SqlQueryScheduler for runtime cost-based optimizations + variableAllocator.set(new PlanVariableAllocator(plan.getTypes().allVariables())); + SubPlan fragmentedPlan = getSession().getRuntimeStats().profileNanos( + FRAGMENT_PLAN_TIME_NANOS, + () -> planFragmenter.createSubPlans(stateMachine.getSession(), plan, false, idAllocator, variableAllocator.get(), stateMachine.getWarningCollector())); + + // record analysis time + stateMachine.endAnalysis(); + + boolean explainAnalyze = analysis.getStatement() instanceof Explain && ((Explain) analysis.getStatement()).isAnalyze(); + return new PlanRoot(fragmentedPlan, !explainAnalyze, extractConnectors(analysis)); } catch (StackOverflowError e) { throw new PrestoException(NOT_SUPPORTED, "statement is too large (stack overflow during analysis)", e); } } - private PlanRoot doAnalyzeQuery() - { - // time analysis phase - stateMachine.beginAnalysis(); - - // plan query - LogicalPlanner logicalPlanner = new LogicalPlanner(false, stateMachine.getSession(), planOptimizers, idAllocator, metadata, sqlParser, statsCalculator, costCalculator, stateMachine.getWarningCollector(), planChecker); - Plan plan = getSession().getRuntimeStats().profileNanos( - LOGICAL_PLANNER_TIME_NANOS, - () -> logicalPlanner.plan(analysis)); - queryPlan.set(plan); - stateMachine.setPlanStatsAndCosts(plan.getStatsAndCosts()); - stateMachine.setPlanCanonicalInfo(getCanonicalInfo(getSession(), plan.getRoot(), planCanonicalInfoProvider)); - - // extract inputs - List inputs = new InputExtractor(metadata, stateMachine.getSession()).extractInputs(plan.getRoot()); - stateMachine.setInputs(inputs); - - // extract output - Optional output = new OutputExtractor().extractOutput(plan.getRoot()); - stateMachine.setOutput(output); - - // fragment the plan - // the variableAllocator is finally passed to SqlQueryScheduler for runtime cost-based optimizations - variableAllocator.set(new PlanVariableAllocator(plan.getTypes().allVariables())); - SubPlan fragmentedPlan = getSession().getRuntimeStats().profileNanos( - FRAGMENT_PLAN_TIME_NANOS, - () -> planFragmenter.createSubPlans(stateMachine.getSession(), plan, false, idAllocator, variableAllocator.get(), stateMachine.getWarningCollector())); - - // record analysis time - stateMachine.endAnalysis(); - - boolean explainAnalyze = analysis.getStatement() instanceof Explain && ((Explain) analysis.getStatement()).isAnalyze(); - return new PlanRoot(fragmentedPlan, !explainAnalyze, extractConnectors(analysis)); - } - private static Set extractConnectors(Analysis analysis) { ImmutableSet.Builder connectors = ImmutableSet.builder(); @@ -831,7 +826,7 @@ public static class SqlQueryExecutionFactory @Override public QueryExecution createQueryExecution( - PreparedQuery preparedQuery, + BuiltInPreparedQuery preparedQuery, QueryStateMachine stateMachine, String slug, int retryCount, diff --git a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java index 71a4a787702b6..6c02147d2d519 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/InternalResourceGroupManager.java @@ -29,7 +29,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; -import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.util.PeriodicTaskExecutor; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -144,7 +143,7 @@ public List getPathToRoot(ResourceGroupId id) } @Override - public void submit(Statement statement, ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor) + public void submit(ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor) { checkState(configurationManager.get() != null, "configurationManager not set"); createGroupIfNecessary(selectionContext, executor); diff --git a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/NoOpResourceGroupManager.java b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/NoOpResourceGroupManager.java index b34aee73e46cd..a90ba2c27c345 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/NoOpResourceGroupManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/NoOpResourceGroupManager.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; -import com.facebook.presto.sql.tree.Statement; import java.util.List; import java.util.concurrent.Executor; @@ -31,7 +30,7 @@ public final class NoOpResourceGroupManager implements ResourceGroupManager { @Override - public void submit(Statement statement, ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor) + public void submit(ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor) { throw new UnsupportedOperationException(); } diff --git a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/ResourceGroupManager.java b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/ResourceGroupManager.java index 10d1ac90c6174..00c4f3399dd79 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/ResourceGroupManager.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/resourceGroups/ResourceGroupManager.java @@ -19,7 +19,6 @@ import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.resourceGroups.SelectionContext; import com.facebook.presto.spi.resourceGroups.SelectionCriteria; -import com.facebook.presto.sql.tree.Statement; import javax.annotation.concurrent.ThreadSafe; @@ -33,7 +32,7 @@ @ThreadSafe public interface ResourceGroupManager { - void submit(Statement statement, ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor); + void submit(ManagedQueryExecution queryExecution, SelectionContext selectionContext, Executor executor); SelectionContext selectGroup(SelectionCriteria criteria); diff --git a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java index b25a6fbf4dc1c..bf6eeb6eaa512 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/CoordinatorModule.java @@ -80,11 +80,13 @@ import com.facebook.presto.server.remotetask.RemoteTaskStats; import com.facebook.presto.spi.memory.ClusterMemoryPoolManager; import com.facebook.presto.spi.security.SelectedRole; +import com.facebook.presto.sql.analyzer.AnalyzerModule; +import com.facebook.presto.sql.analyzer.AnalyzerProvider; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.NativeQueryPreparer; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer; import com.facebook.presto.sql.planner.PlanFragmenter; import com.facebook.presto.sql.planner.PlanOptimizers; -import com.facebook.presto.sql.tree.Statement; import com.facebook.presto.transaction.ForTransactionManager; import com.facebook.presto.transaction.InMemoryTransactionManager; import com.facebook.presto.transaction.TransactionManager; @@ -103,7 +105,6 @@ import javax.inject.Singleton; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; @@ -123,7 +124,6 @@ import static com.facebook.presto.execution.SessionDefinitionExecution.SessionDefinitionExecutionFactory; import static com.facebook.presto.execution.SqlQueryExecution.SqlQueryExecutionFactory; import static com.facebook.presto.sql.analyzer.utils.StatementUtils.getAllQueryTypes; -import static com.facebook.presto.sql.analyzer.utils.StatementUtils.isSessionTransactionControlStatement; import static com.google.inject.multibindings.MapBinder.newMapBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -180,7 +180,12 @@ protected void setup(Binder binder) binder.bind(QueryIdGenerator.class).in(Scopes.SINGLETON); binder.bind(QueryManager.class).to(SqlQueryManager.class).in(Scopes.SINGLETON); newExporter(binder).export(QueryManager.class).withGeneratedName(); - binder.bind(QueryPreparer.class).in(Scopes.SINGLETON); + + binder.install(new AnalyzerModule()); + binder.bind(AnalyzerProvider.class).in(Scopes.SINGLETON); + binder.bind(BuiltInQueryPreparer.class).in(Scopes.SINGLETON); + binder.bind(NativeQueryPreparer.class).in(Scopes.SINGLETON); + binder.bind(SessionSupplier.class).to(QuerySessionSupplier.class).in(Scopes.SINGLETON); binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON); newExporter(binder).export(InternalResourceGroupManager.class).withGeneratedName(); @@ -267,29 +272,29 @@ protected void setup(Binder binder) binder.bind(QueryExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(QueryExecutionMBean.class).as(generatedNameOf(QueryExecution.class)); - MapBinder, QueryExecutionFactory> executionBinder = newMapBinder(binder, - new TypeLiteral>() {}, new TypeLiteral>() {}); + MapBinder> executionBinder = newMapBinder(binder, + new TypeLiteral() {}, new TypeLiteral>() {}); binder.bind(SplitSchedulerStats.class).in(Scopes.SINGLETON); newExporter(binder).export(SplitSchedulerStats.class).withGeneratedName(); binder.bind(SqlQueryExecutionFactory.class).in(Scopes.SINGLETON); binder.bind(SectionExecutionFactory.class).in(Scopes.SINGLETON); - Set, QueryType>> queryTypes = getAllQueryTypes().entrySet(); + Set queryTypes = getAllQueryTypes(); - // bind sql query statements to SqlQueryExecutionFactory - queryTypes.stream().filter(entry -> entry.getValue() != QueryType.DATA_DEFINITION) - .forEach(entry -> executionBinder.addBinding(entry.getKey()).to(SqlQueryExecutionFactory.class).in(Scopes.SINGLETON)); + // bind sql query type to SqlQueryExecutionFactory + queryTypes.stream().filter(queryType -> queryType != QueryType.DATA_DEFINITION && queryType != QueryType.CONTROL) + .forEach(queryType -> executionBinder.addBinding(queryType).to(SqlQueryExecutionFactory.class).in(Scopes.SINGLETON)); binder.bind(PartialResultQueryManager.class).in(Scopes.SINGLETON); - // bind data definition statements to DataDefinitionExecutionFactory - queryTypes.stream().filter(entry -> entry.getValue() == QueryType.DATA_DEFINITION && !isSessionTransactionControlStatement(entry.getKey())) - .forEach(entry -> executionBinder.addBinding(entry.getKey()).to(DDLDefinitionExecutionFactory.class).in(Scopes.SINGLETON)); + // bind data definition type to DataDefinitionExecutionFactory + queryTypes.stream().filter(queryType -> queryType == QueryType.DATA_DEFINITION) + .forEach(queryType -> executionBinder.addBinding(queryType).to(DDLDefinitionExecutionFactory.class).in(Scopes.SINGLETON)); binder.bind(DDLDefinitionExecutionFactory.class).in(Scopes.SINGLETON); - // bind session Control statements to SessionTransactionExecutionFactory - queryTypes.stream().filter(entry -> (entry.getValue() == QueryType.DATA_DEFINITION && isSessionTransactionControlStatement(entry.getKey()))) - .forEach(entry -> executionBinder.addBinding(entry.getKey()).to(SessionDefinitionExecutionFactory.class).in(Scopes.SINGLETON)); + // bind session Control types to SessionTransactionExecutionFactory + queryTypes.stream().filter(queryType -> queryType == QueryType.CONTROL) + .forEach(queryType -> executionBinder.addBinding(queryType).to(SessionDefinitionExecutionFactory.class).in(Scopes.SINGLETON)); binder.bind(SessionDefinitionExecutionFactory.class).in(Scopes.SINGLETON); // helper class binding data definition tasks and statements diff --git a/presto-main/src/main/java/com/facebook/presto/server/ResourceManagerModule.java b/presto-main/src/main/java/com/facebook/presto/server/ResourceManagerModule.java index f8a6b2735d55d..7f22e371b4381 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ResourceManagerModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ResourceManagerModule.java @@ -32,7 +32,10 @@ import com.facebook.presto.resourcemanager.ResourceManagerClusterStateProvider; import com.facebook.presto.resourcemanager.ResourceManagerProxy; import com.facebook.presto.resourcemanager.ResourceManagerServer; -import com.facebook.presto.sql.analyzer.QueryPreparer; +import com.facebook.presto.sql.analyzer.AnalyzerModule; +import com.facebook.presto.sql.analyzer.AnalyzerProvider; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.NativeQueryPreparer; import com.facebook.presto.transaction.NoOpTransactionManager; import com.facebook.presto.transaction.TransactionManager; import com.google.inject.Binder; @@ -80,7 +83,13 @@ protected void setup(Binder binder) binder.bind(QueryManager.class).to(NoOpQueryManager.class).in(Scopes.SINGLETON); jaxrsBinder(binder).bind(DistributedResourceGroupInfoResource.class); binder.bind(QueryIdGenerator.class).in(Scopes.SINGLETON); - binder.bind(QueryPreparer.class).in(Scopes.SINGLETON); + + //TODO: Is it really needed here? + binder.install(new AnalyzerModule()); + binder.bind(AnalyzerProvider.class).in(Scopes.SINGLETON); + binder.bind(BuiltInQueryPreparer.class).in(Scopes.SINGLETON); + binder.bind(NativeQueryPreparer.class).in(Scopes.SINGLETON); + binder.bind(SessionSupplier.class).to(QuerySessionSupplier.class).in(Scopes.SINGLETON); binder.bind(ResourceGroupManager.class).to(NoOpResourceGroupManager.class); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index fd74530aab7a5..f8573630c5503 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -324,12 +324,6 @@ public enum AggregationIfToFilterRewriteStrategy UNWRAP_IF // Rewrites AGG(IF(condition, expr)) to AGG(expr) FILTER (WHERE condition). } - public enum AnalyzerType - { - BUILTIN, - NATIVE - } - public double getCpuCostWeight() { return cpuCostWeight; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java index 3c13cfd7d4243..271da723a7268 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ExplainRewrite.java @@ -18,9 +18,9 @@ import com.facebook.presto.security.AccessControl; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.sql.analyzer.AnalyzerOptions; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; import com.facebook.presto.sql.analyzer.SemanticException; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.tree.AstVisitor; @@ -69,7 +69,7 @@ private static final class Visitor extends AstVisitor { private final Session session; - private final QueryPreparer queryPreparer; + private final BuiltInQueryPreparer queryPreparer; private final Optional queryExplainer; private final WarningCollector warningCollector; @@ -80,7 +80,7 @@ public Visitor( WarningCollector warningCollector) { this.session = requireNonNull(session, "session is null"); - this.queryPreparer = new QueryPreparer(requireNonNull(parser, "queryPreparer is null")); + this.queryPreparer = new BuiltInQueryPreparer(requireNonNull(parser, "queryPreparer is null")); this.queryExplainer = requireNonNull(queryExplainer, "queryExplainer is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); } @@ -123,7 +123,7 @@ private Node getQueryPlan(Explain node, ExplainType.Type planType, ExplainFormat throws IllegalArgumentException { AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); - PreparedQuery preparedQuery = queryPreparer.prepareQuery(analyzerOptions, node.getStatement(), session.getPreparedStatements()); + BuiltInPreparedQuery preparedQuery = queryPreparer.prepareQuery(analyzerOptions, node.getStatement(), session.getPreparedStatements()); if (planType == VALIDATE) { queryExplainer.get().analyze(session, preparedQuery.getStatement(), preparedQuery.getParameters(), warningCollector); return singleValueQuery("Valid", true); diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index 2a44dfa710019..45b0330aa4115 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -150,10 +150,10 @@ import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; import com.facebook.presto.sql.analyzer.AnalyzerOptions; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -987,7 +987,7 @@ public Plan createPlan(Session session, @Language("SQL") String sql, LogicalPlan public Plan createPlan(Session session, @Language("SQL") String sql, LogicalPlanner.Stage stage, boolean forceSingleNode, WarningCollector warningCollector) { AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); - PreparedQuery preparedQuery = new QueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); + BuiltInPreparedQuery preparedQuery = new BuiltInQueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); assertFormattedSql(sqlParser, createParsingOptions(session), preparedQuery.getStatement()); @@ -1023,7 +1023,7 @@ public Plan createPlan(Session session, @Language("SQL") String sql, List optimizers, LogicalPlanner.Stage stage, WarningCollector warningCollector) { AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); - PreparedQuery preparedQuery = new QueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); + BuiltInPreparedQuery preparedQuery = new BuiltInQueryPreparer(sqlParser).prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); assertFormattedSql(sqlParser, createParsingOptions(session), preparedQuery.getStatement()); PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 6c2d72bd6572c..98a17bfd145de 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -19,7 +19,6 @@ import com.facebook.presto.operator.aggregation.histogram.HistogramGroupImplementation; import com.facebook.presto.operator.aggregation.multimapagg.MultimapAggGroupImplementation; import com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationIfToFilterRewriteStrategy; -import com.facebook.presto.sql.analyzer.FeaturesConfig.AnalyzerType; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy; import com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy; @@ -34,9 +33,9 @@ import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.presto.sql.analyzer.AnalyzerType.NATIVE; import static com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy.LEGACY; import static com.facebook.presto.sql.analyzer.FeaturesConfig.AggregationPartitioningMergingStrategy.TOP_DOWN; -import static com.facebook.presto.sql.analyzer.FeaturesConfig.AnalyzerType.NATIVE; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST; import static com.facebook.presto.sql.analyzer.FeaturesConfig.JoinReorderingStrategy.NONE; import static com.facebook.presto.sql.analyzer.FeaturesConfig.PartialMergePushdownStrategy.PUSH_THROUGH_LOW_MEMORY_OPERATORS; diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index f865556137a34..d5b1266fbc39e 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -146,9 +146,9 @@ import com.facebook.presto.sql.Serialization.VariableReferenceExpressionDeserializer; import com.facebook.presto.sql.Serialization.VariableReferenceExpressionSerializer; import com.facebook.presto.sql.SqlEnvironmentConfig; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer; import com.facebook.presto.sql.gen.ExpressionCompiler; import com.facebook.presto.sql.gen.JoinCompiler; import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler; @@ -398,7 +398,7 @@ protected void setup(Binder binder) binder.bind(FragmentCacheStats.class).in(Scopes.SINGLETON); binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON); binder.bind(QueryIdGenerator.class).in(Scopes.SINGLETON); - binder.bind(QueryPreparer.class).in(Scopes.SINGLETON); + binder.bind(BuiltInQueryPreparer.class).in(Scopes.SINGLETON); jsonBinder(binder).addKeySerializerBinding(VariableReferenceExpression.class).to(VariableReferenceExpressionSerializer.class); jsonBinder(binder).addKeyDeserializerBinding(VariableReferenceExpression.class).to(VariableReferenceExpressionDeserializer.class); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java index 42a7f778caa1e..93289cf4e4af2 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkQueryExecutionFactory.java @@ -77,8 +77,8 @@ import com.facebook.presto.spi.security.Identity; import com.facebook.presto.spi.storage.TempStorage; import com.facebook.presto.sql.analyzer.AnalyzerOptions; -import com.facebook.presto.sql.analyzer.QueryPreparer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.utils.StatementUtils; import com.facebook.presto.sql.planner.PartitioningProviderManager; import com.facebook.presto.sql.planner.SubPlan; @@ -150,7 +150,7 @@ public class PrestoSparkQueryExecutionFactory private final QueryIdGenerator queryIdGenerator; private final QuerySessionSupplier sessionSupplier; - private final QueryPreparer queryPreparer; + private final BuiltInQueryPreparer queryPreparer; private final PrestoSparkQueryPlanner queryPlanner; private final PrestoSparkPlanFragmenter planFragmenter; private final PrestoSparkRddFactory rddFactory; @@ -184,7 +184,7 @@ public class PrestoSparkQueryExecutionFactory public PrestoSparkQueryExecutionFactory( QueryIdGenerator queryIdGenerator, QuerySessionSupplier sessionSupplier, - QueryPreparer queryPreparer, + BuiltInQueryPreparer queryPreparer, PrestoSparkQueryPlanner queryPlanner, PrestoSparkPlanFragmenter planFragmenter, PrestoSparkRddFactory rddFactory, @@ -615,9 +615,9 @@ public IPrestoSparkQueryExecution create( queryStateTimer.beginAnalyzing(); AnalyzerOptions analyzerOptions = createAnalyzerOptions(session, warningCollector); - PreparedQuery preparedQuery = queryPreparer.prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); + BuiltInPreparedQuery preparedQuery = queryPreparer.prepareQuery(analyzerOptions, sql, session.getPreparedStatements(), warningCollector); Optional queryType = StatementUtils.getQueryType(preparedQuery.getStatement().getClass()); - if (queryType.isPresent() && (queryType.get() == QueryType.DATA_DEFINITION)) { + if (queryType.isPresent() && (queryType.get() == QueryType.DATA_DEFINITION || queryType.get() == QueryType.CONTROL)) { queryStateTimer.endAnalysis(); DDLDefinitionTask task = (DDLDefinitionTask) ddlTasks.get(preparedQuery.getStatement().getClass()); return new PrestoSparkDataDefinitionExecution(task, preparedQuery.getStatement(), transactionManager, accessControl, metadata, session, queryStateTimer, warningCollector); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java index d35cfde2a9915..75cd343764306 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/planner/PrestoSparkQueryPlanner.java @@ -27,8 +27,8 @@ import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.sql.analyzer.Analysis; import com.facebook.presto.sql.analyzer.Analyzer; +import com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.BuiltInPreparedQuery; import com.facebook.presto.sql.analyzer.QueryExplainer; -import com.facebook.presto.sql.analyzer.QueryPreparer.PreparedQuery; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.InputExtractor; import com.facebook.presto.sql.planner.LogicalPlanner; @@ -83,7 +83,7 @@ public PrestoSparkQueryPlanner( this.planChecker = requireNonNull(planChecker, "planChecker is null"); } - public PlanAndMore createQueryPlan(Session session, PreparedQuery preparedQuery, WarningCollector warningCollector) + public PlanAndMore createQueryPlan(Session session, BuiltInPreparedQuery preparedQuery, WarningCollector warningCollector) { PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index 918c2949cd7cf..6121f492db4f2 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -112,6 +112,7 @@ public enum StandardErrorCode PLAN_SERIALIZATION_ERROR(0x0001_0026, INTERNAL_ERROR), QUERY_PLANNING_TIMEOUT(0x0001_0027, INTERNAL_ERROR), NATIVE_EXECUTION_TASK_ERROR(0x0001_0028, INTERNAL_ERROR), + UNSUPPORTED_ANALYZER_TYPE(0x0001_0029, INTERNAL_ERROR), GENERIC_INSUFFICIENT_RESOURCES(0x0002_0000, INSUFFICIENT_RESOURCES), EXCEEDED_GLOBAL_MEMORY_LIMIT(0x0002_0001, INSUFFICIENT_RESOURCES), diff --git a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueues.java b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueues.java index 07c159b7e3ee5..005047fead023 100644 --- a/presto-tests/src/test/java/com/facebook/presto/execution/TestQueues.java +++ b/presto-tests/src/test/java/com/facebook/presto/execution/TestQueues.java @@ -324,7 +324,7 @@ public void testQueryTypeBasedSelection() assertResourceGroup(queryRunner, newAdhocSession(), "SHOW TABLES", createResourceGroupId("global", "describe")); assertResourceGroup(queryRunner, newAdhocSession(), "EXPLAIN " + LONG_LASTING_QUERY, createResourceGroupId("global", "explain")); assertResourceGroup(queryRunner, newAdhocSession(), "DESCRIBE lineitem", createResourceGroupId("global", "describe")); - assertResourceGroup(queryRunner, newAdhocSession(), "RESET SESSION " + HASH_PARTITION_COUNT, createResourceGroupId("global", "data_definition")); + assertResourceGroup(queryRunner, newAdhocSession(), "RESET SESSION " + HASH_PARTITION_COUNT, createResourceGroupId("global", "control")); } } diff --git a/presto-tests/src/test/resources/resource_groups_query_type_based_config.json b/presto-tests/src/test/resources/resource_groups_query_type_based_config.json index fe1290f46030f..f59665a7979e6 100644 --- a/presto-tests/src/test/resources/resource_groups_query_type_based_config.json +++ b/presto-tests/src/test/resources/resource_groups_query_type_based_config.json @@ -44,6 +44,12 @@ "hardConcurrencyLimit": 3, "maxQueued": 4 }, + { + "name": "control", + "softMemoryLimit": "2MB", + "hardConcurrencyLimit": 3, + "maxQueued": 4 + }, { "name": "other", "softMemoryLimit": "2MB", @@ -78,6 +84,10 @@ "queryType" : "data_definition", "group": "global.data_definition" }, + { + "queryType" : "control", + "group": "global.control" + }, { "group": "global.other" }