From c9feef4e1e435c52bf0e8c27d41e5e4767e423ed Mon Sep 17 00:00:00 2001 From: Wenlei Xie Date: Tue, 14 Jul 2020 10:38:41 -0700 Subject: [PATCH] Allow configurable parser option in Presto-on-Spark --- .../presto/spark/PrestoSparkInjectorFactory.java | 16 ++++++++++++++-- .../facebook/presto/spark/PrestoSparkModule.java | 6 ++++-- .../presto/spark/PrestoSparkServiceFactory.java | 7 +++++++ .../presto/spark/PrestoSparkQueryRunner.java | 2 ++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java index df2241ab69be7..006e9edd2eadd 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkInjectorFactory.java @@ -26,6 +26,7 @@ import com.facebook.presto.server.SessionPropertyDefaults; import com.facebook.presto.server.security.PasswordAuthenticatorManager; import com.facebook.presto.spark.classloader_interface.SparkProcessType; +import com.facebook.presto.sql.parser.SqlParserOptions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Injector; @@ -47,6 +48,7 @@ public class PrestoSparkInjectorFactory private final Map configProperties; private final Map> catalogProperties; private final Optional> eventListenerProperties; + private final SqlParserOptions sqlParserOptions; private final List additionalModules; private final Optional accessControlModuleOverride; @@ -55,9 +57,17 @@ public PrestoSparkInjectorFactory( Map configProperties, Map> catalogProperties, Optional> eventListenerProperties, + SqlParserOptions sqlParserOptions, List additionalModules) { - this(sparkProcessType, configProperties, catalogProperties, eventListenerProperties, additionalModules, Optional.empty()); + this( + sparkProcessType, + configProperties, + catalogProperties, + eventListenerProperties, + sqlParserOptions, + additionalModules, + Optional.empty()); } public PrestoSparkInjectorFactory( @@ -65,6 +75,7 @@ public PrestoSparkInjectorFactory( Map configProperties, Map> catalogProperties, Optional> eventListenerProperties, + SqlParserOptions sqlParserOptions, List additionalModules, Optional accessControlModuleOverride) { @@ -73,6 +84,7 @@ public PrestoSparkInjectorFactory( this.catalogProperties = requireNonNull(catalogProperties, "catalogProperties is null").entrySet().stream() .collect(toImmutableMap(Entry::getKey, entry -> ImmutableMap.copyOf(entry.getValue()))); this.eventListenerProperties = requireNonNull(eventListenerProperties, "eventListenerProperties is null").map(ImmutableMap::copyOf); + this.sqlParserOptions = requireNonNull(sqlParserOptions, "sqlParserOptions is null"); this.additionalModules = ImmutableList.copyOf(requireNonNull(additionalModules, "additionalModules is null")); this.accessControlModuleOverride = requireNonNull(accessControlModuleOverride, "accessControlModuleOverride is null"); } @@ -87,7 +99,7 @@ public Injector create() modules.add( new JsonModule(), new EventListenerModule(), - new PrestoSparkModule(sparkProcessType)); + new PrestoSparkModule(sparkProcessType, sqlParserOptions)); boolean initializeAccessControl = false; if (accessControlModuleOverride.isPresent()) { diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index 9f89b3c7b2f99..2c5503d045002 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -178,10 +178,12 @@ public class PrestoSparkModule extends AbstractConfigurationAwareModule { private final SparkProcessType sparkProcessType; + private final SqlParserOptions sqlParserOptions; - public PrestoSparkModule(SparkProcessType sparkProcessType) + public PrestoSparkModule(SparkProcessType sparkProcessType, SqlParserOptions sqlParserOptions) { this.sparkProcessType = requireNonNull(sparkProcessType, "sparkProcessType is null"); + this.sqlParserOptions = requireNonNull(sqlParserOptions, "sqlParserOptions is null"); } @Override @@ -316,7 +318,7 @@ protected void setup(Binder binder) // parser binder.bind(SqlParser.class).in(Scopes.SINGLETON); - binder.bind(SqlParserOptions.class).toInstance(new SqlParserOptions()); + binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions); // planner binder.bind(PlanFragmenter.class).in(Scopes.SINGLETON); diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java index 7cdedd72a9f39..c1b0a2e771297 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkServiceFactory.java @@ -18,6 +18,7 @@ import com.facebook.presto.spark.classloader_interface.IPrestoSparkServiceFactory; import com.facebook.presto.spark.classloader_interface.PrestoSparkConfiguration; import com.facebook.presto.spark.classloader_interface.SparkProcessType; +import com.facebook.presto.sql.parser.SqlParserOptions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Injector; @@ -42,6 +43,7 @@ public IPrestoSparkService createService(SparkProcessType sparkProcessType, Pres properties.build(), configuration.getCatalogProperties(), configuration.getEventListenerProperties(), + getSqlParserOptions(), getAdditionalModules()); Injector injector = prestoSparkInjectorFactory.create(); @@ -54,4 +56,9 @@ protected List getAdditionalModules() { return ImmutableList.of(); } + + protected SqlParserOptions getSqlParserOptions() + { + return new SqlParserOptions(); + } } diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index c65e255473c72..9cb9503e6d43b 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -49,6 +49,7 @@ import com.facebook.presto.spi.security.PrincipalType; import com.facebook.presto.split.PageSourceManager; import com.facebook.presto.split.SplitManager; +import com.facebook.presto.sql.parser.SqlParserOptions; import com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager; import com.facebook.presto.sql.planner.NodePartitioningManager; import com.facebook.presto.testing.MaterializedResult; @@ -206,6 +207,7 @@ public PrestoSparkQueryRunner(String defaultCatalog) "prefer-distributed-union", "false"), ImmutableMap.of(), Optional.empty(), + new SqlParserOptions(), ImmutableList.of(), Optional.of(new TestingAccessControlModule()));