From 85c2664990af581e8c57a4af6f5d7821f7e91614 Mon Sep 17 00:00:00 2001 From: Hazmi Date: Thu, 17 Jul 2025 18:24:43 +0300 Subject: [PATCH 1/2] Add a DB based session property manager Co-authored-by: Ashish Tadose Co-authored-by: Ariel Weisberg Co-authored-by: Jalpreet Singh Nanda (:imjalpreet) --- README.md | 2 + pom.xml | 6 + .../admin/session-property-managers.rst | 84 ++++++++++- presto-function-namespace-managers/pom.xml | 1 - presto-main/etc/session-property-config.json | 0 .../etc/session-property-config.properties | 0 presto-session-property-managers/pom.xml | 63 +++++++++ .../AbstractSessionPropertyManager.java | 65 +++++++++ .../presto/session/SessionMatchSpec.java | 103 +++++++++++++- ...onPropertyConfigurationManagerPlugin.java} | 8 +- .../session/db/DbSessionPropertyManager.java | 47 +++++++ .../db/DbSessionPropertyManagerConfig.java | 54 ++++++++ .../db/DbSessionPropertyManagerFactory.java | 53 +++++++ .../db/DbSessionPropertyManagerModule.java | 33 +++++ .../presto/session/db/DbSpecsProvider.java | 29 ++++ .../session/db/RefreshingDbSpecsProvider.java | 94 +++++++++++++ .../session/db/SessionPropertiesDao.java | 130 ++++++++++++++++++ .../db/SessionPropertiesDaoProvider.java | 46 +++++++ .../FileSessionPropertyManager.java | 45 ++---- .../FileSessionPropertyManagerConfig.java | 2 +- .../FileSessionPropertyManagerFactory.java | 2 +- .../FileSessionPropertyManagerModule.java | 2 +- ...> AbstractTestSessionPropertyManager.java} | 77 ++++++----- .../db/TestDbSessionPropertyManager.java | 112 +++++++++++++++ .../TestDbSessionPropertyManagerConfig.java | 52 +++++++ .../TestDbSessionPropertyManagerMariadb.java | 29 ++++ .../db/TestDbSessionPropertyManagerMysql.java | 23 ++++ .../file/TestFileSessionPropertyManager.java | 62 +++++++++ .../TestFileSessionPropertyManagerConfig.java | 2 +- ...estoSparkLauncherIntegrationSmokeTest.java | 5 +- 30 files changed, 1144 insertions(+), 87 deletions(-) create mode 100644 presto-main/etc/session-property-config.json create mode 100644 presto-main/etc/session-property-config.properties create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java rename presto-session-property-managers/src/main/java/com/facebook/presto/session/{FileSessionPropertyManagerPlugin.java => SessionPropertyConfigurationManagerPlugin.java} (73%) create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java create mode 100644 presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java rename presto-session-property-managers/src/main/java/com/facebook/presto/session/{ => file}/FileSessionPropertyManager.java (60%) rename presto-session-property-managers/src/main/java/com/facebook/presto/session/{ => file}/FileSessionPropertyManagerConfig.java (96%) rename presto-session-property-managers/src/main/java/com/facebook/presto/session/{ => file}/FileSessionPropertyManagerFactory.java (97%) rename presto-session-property-managers/src/main/java/com/facebook/presto/session/{ => file}/FileSessionPropertyManagerModule.java (96%) rename presto-session-property-managers/src/test/java/com/facebook/presto/session/{TestFileSessionPropertyManager.java => AbstractTestSessionPropertyManager.java} (72%) create mode 100644 presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java create mode 100644 presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java create mode 100644 presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java create mode 100644 presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java create mode 100644 presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java rename presto-session-property-managers/src/test/java/com/facebook/presto/session/{ => file}/TestFileSessionPropertyManagerConfig.java (97%) diff --git a/README.md b/README.md index 2da36a8a31aea..2b4aa6d998c1d 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ Additionally, the Hive plugin must be configured with location of your Hive meta -Dhive.metastore.uri=thrift://localhost:9083 +To modify the loaded plugins in IntelliJ, modify the `config.properties` located in `presto-main/etc`. You can modify `plugin.bundles` with the location of the plugin pom.xml + ### Additional configuration for Java 17 When running with Java 17, additional `--add-opens` flags are required to allow reflective access used by certain catalogs based on which catalogs are configured. diff --git a/pom.xml b/pom.xml index f979056f76891..e9b5909ff5d29 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 3.18.0 6.0.0 17.0.0 + 3.5.4 2 @@ -2678,6 +2679,11 @@ ${dep.arrow.version} + + org.mariadb.jdbc + mariadb-java-client + ${dep.mariadb.version} + diff --git a/presto-docs/src/main/sphinx/admin/session-property-managers.rst b/presto-docs/src/main/sphinx/admin/session-property-managers.rst index 5ebdfb1a88bc3..8147d9741f051 100644 --- a/presto-docs/src/main/sphinx/admin/session-property-managers.rst +++ b/presto-docs/src/main/sphinx/admin/session-property-managers.rst @@ -5,10 +5,11 @@ Session Property Managers Administrators can add session properties to control the behavior for subsets of their workload. These properties are defaults and can be overridden by users (if authorized to do so). Session properties can be used to control resource usage, enable or disable features, and change query -characteristics. Session property managers are pluggable. +characteristics. Session property managers are pluggable. A session property manager can either +be database-based or file-based. -Add an ``etc/session-property-config.properties`` file with the following contents to enable -the built-in manager that reads a JSON config file: +To enable a built-in manager that reads a JSON configuration file, add an +``etc/session-property-config.properties`` file with the following contents: .. code-block:: none @@ -24,6 +25,66 @@ by default. All matching rules contribute to constructing a list of session prop are applied in the order they are specified. Rules specified later in the file override values for properties that have been previously encountered. + +For the database-based built-in manager, add an +``etc/session-property-config.properties`` file with the following contents: + +.. code-block:: text + + session-property-config.configuration-manager=db + session-property-manager.db.url=jdbc:mysql://localhost:3306/session_properties?user=user&password=pass&createDatabaseIfNotExist=true + session-property-manager.db.refresh-period=50s + +Change the value of ``session-property-manager.db.url`` to the JDBC URL of a database. + +``session-property-manager.db.refresh-period`` should be set to how often Presto refreshes +to fetch the latest session properties from the database. + +This database consists of three tables: ``session_specs``, ``session_client_tags`` and ``session_property_values``. +Presto will create the database on startup if you set ``createDatabaseIfNotExist`` to ``true`` in your JDBC URL. +If the tables do not exist, Presto will create them on startup. + +.. code-block:: text + + mysql> DESCRIBE session_specs; + +-----------------------------+--------------+------+-----+---------+----------------+ + | Field | Type | Null | Key | Default | Extra | + +-----------------------------+--------------+------+-----+---------+----------------+ + | spec_id | bigint | NO | PRI | NULL | auto_increment | + | user_regex | varchar(512) | YES | | NULL | | + | source_regex | varchar(512) | YES | | NULL | | + | query_type | varchar(512) | YES | | NULL | | + | group_regex | varchar(512) | YES | | NULL | | + | client_info_regex | varchar(512) | YES | | NULL | | + | override_session_properties | tinyint(1) | YES | | NULL | | + | priority | int | NO | | NULL | | + +-----------------------------+--------------+------+-----+---------+----------------+ + 8 rows in set (0.016 sec) + +.. code-block:: text + + mysql> DESCRIBE session_client_tags; + +-------------+--------------+------+-----+---------+-------+ + | Field | Type | Null | Key | Default | Extra | + +-------------+--------------+------+-----+---------+-------+ + | tag_spec_id | bigint | NO | PRI | NULL | | + | client_tag | varchar(512) | NO | PRI | NULL | | + +-------------+--------------+------+-----+---------+-------+ + 2 rows in set (0.062 sec) + +.. code-block:: text + + mysql> DESCRIBE session_property_values; + +--------------------------+--------------+------+-----+---------+-------+ + | Field | Type | Null | Key | Default | Extra | + +--------------------------+--------------+------+-----+---------+-------+ + | property_spec_id | bigint | NO | PRI | NULL | | + | session_property_name | varchar(512) | NO | PRI | NULL | | + | session_property_value | varchar(512) | YES | | NULL | | + | session_property_catalog | varchar(512) | YES | | NULL | | + +--------------------------+--------------+------+-----+---------+-------+ + 3 rows in set (0.009 sec) + Match Rules ----------- @@ -52,9 +113,15 @@ Match Rules Note that once a session property has been overridden by ANY rule it remains overridden even if later higher precedence rules change the value, but don't specify override. -* ``sessionProperties``: map with string keys and values. Each entry is a system or catalog property name and +* ``sessionProperties``: map with string keys and values. Each entry is a system property name and corresponding value. Values must be specified as strings, no matter the actual data type. +* ``catalogSessionProperties``: map with string keys corresponding to the catalog name, and a map with string keys + and values as the value. Each entry is a catalog name and corresponding map of session property values. + +* For the database session property manager, catalog & system session properties are located in the same table. + ``session_property_catalog`` should be null for system session properties. + Example ------- @@ -71,6 +138,8 @@ Consider the following set of requirements: * All high memory ETL queries (tagged with 'high_mem_etl') are routed to subgroups under the ``global.pipeline`` group, and must be configured to enable :doc:`/admin/exchange-materialization`. +* All iceberg catalog queries should override the ``delete-as-join-rewrite-enabled`` property + These requirements can be expressed with the following rules: .. code-block:: json @@ -104,5 +173,12 @@ These requirements can be expressed with the following rules: "partitioning_provider_catalog": "hive", "hash_partition_count": 4096 } + }, + { + "catalogSessionProperties": { + "iceberg": { + "delete_as_join_rewrite_enabled": "true" + } + } } ] diff --git a/presto-function-namespace-managers/pom.xml b/presto-function-namespace-managers/pom.xml index 7e3a3e95425df..a882b50e35df6 100644 --- a/presto-function-namespace-managers/pom.xml +++ b/presto-function-namespace-managers/pom.xml @@ -148,7 +148,6 @@ org.mariadb.jdbc mariadb-java-client - 3.5.4 runtime diff --git a/presto-main/etc/session-property-config.json b/presto-main/etc/session-property-config.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/presto-main/etc/session-property-config.properties b/presto-main/etc/session-property-config.properties new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/presto-session-property-managers/pom.xml b/presto-session-property-managers/pom.xml index 6386c244b92e5..0ba95f4c6ee88 100644 --- a/presto-session-property-managers/pom.xml +++ b/presto-session-property-managers/pom.xml @@ -24,6 +24,11 @@ bootstrap + + com.facebook.airlift + log + + com.facebook.airlift json @@ -44,6 +49,11 @@ guice + + javax.annotation + javax.annotation-api + + javax.inject javax.inject @@ -54,6 +64,11 @@ jakarta.inject-api + + com.facebook.airlift + concurrent + + jakarta.validation jakarta.validation-api @@ -69,6 +84,30 @@ jackson-core + + org.jdbi + jdbi3-core + + + + org.jdbi + jdbi3-sqlobject + + + + com.mysql + mysql-connector-j + true + runtime + + + + org.mariadb.jdbc + mariadb-java-client + true + runtime + + com.facebook.presto @@ -88,12 +127,24 @@ provided + + io.airlift + slice + provided + + com.fasterxml.jackson.core jackson-annotations provided + + org.openjdk.jol + jol-core + provided + + com.facebook.presto @@ -112,5 +163,17 @@ testing test + + + com.facebook.presto + testing-mysql-server-8 + test + + + + com.facebook.presto + testing-mysql-server-base + test + diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java new file mode 100644 index 0000000000000..7780652db038a --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session; + +import com.facebook.presto.spi.session.SessionConfigurationContext; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class AbstractSessionPropertyManager + implements SessionPropertyConfigurationManager +{ + private static final String DEFAULT_PROPERTIES = "defaultProperties"; + private static final String OVERRIDE_PROPERTIES = "overrideProperties"; + + @Override + public final SystemSessionPropertyConfiguration getSystemSessionProperties(SessionConfigurationContext context) + { + Map defaultProperties = new HashMap<>(); + Set overridePropertyNames = new HashSet<>(); + for (SessionMatchSpec sessionMatchSpec : getSessionMatchSpecs()) { + Map newProperties = sessionMatchSpec.match(sessionMatchSpec.getSessionProperties(), context); + defaultProperties.putAll(newProperties); + if (sessionMatchSpec.getOverrideSessionProperties().orElse(false)) { + overridePropertyNames.addAll(newProperties.keySet()); + } + } + + // Once a property has been overridden it stays that way and the value is updated by any rule + Map overrideProperties = new HashMap<>(); + for (String propertyName : overridePropertyNames) { + overrideProperties.put(propertyName, defaultProperties.get(propertyName)); + } + + return new SystemSessionPropertyConfiguration(defaultProperties, overrideProperties); + } + + @Override + public final Map> getCatalogSessionProperties(SessionConfigurationContext context) + { + Map> catalogProperties = new HashMap<>(); + for (SessionMatchSpec sessionMatchSpec : getSessionMatchSpecs()) { + Map> newProperties = sessionMatchSpec.match(sessionMatchSpec.getCatalogSessionProperties(), context); + catalogProperties.putAll(newProperties); + } + return catalogProperties; + } + + protected abstract List getSessionMatchSpecs(); +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java index 18f34a80adf9d..f5e655c705eef 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java @@ -17,16 +17,24 @@ import com.facebook.presto.spi.session.SessionConfigurationContext; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; +import static com.facebook.presto.session.db.SessionPropertiesDao.EMPTY_CATALOG; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public class SessionMatchSpec @@ -39,6 +47,7 @@ public class SessionMatchSpec private final Optional resourceGroupRegex; private final Optional overrideSessionProperties; private final Map sessionProperties; + private final Map> catalogSessionProperties; @JsonCreator public SessionMatchSpec( @@ -49,7 +58,8 @@ public SessionMatchSpec( @JsonProperty("group") Optional resourceGroupRegex, @JsonProperty("clientInfo") Optional clientInfoRegex, @JsonProperty("overrideSessionProperties") Optional overrideSessionProperties, - @JsonProperty("sessionProperties") Map sessionProperties) + @JsonProperty("sessionProperties") Map sessionProperties, + @JsonProperty("catalogSessionProperties") Map> catalogSessionProperties) { this.userRegex = requireNonNull(userRegex, "userRegex is null"); this.sourceRegex = requireNonNull(sourceRegex, "sourceRegex is null"); @@ -61,9 +71,10 @@ public SessionMatchSpec( this.overrideSessionProperties = requireNonNull(overrideSessionProperties, "overrideSessionProperties is null"); requireNonNull(sessionProperties, "sessionProperties is null"); this.sessionProperties = ImmutableMap.copyOf(sessionProperties); + this.catalogSessionProperties = ImmutableMap.copyOf(catalogSessionProperties); } - public Map match(SessionConfigurationContext context) + public Map match(Map object, SessionConfigurationContext context) { if (userRegex.isPresent() && !userRegex.get().matcher(context.getUser()).matches()) { return ImmutableMap.of(); @@ -99,7 +110,7 @@ public Map match(SessionConfigurationContext context) } } - return sessionProperties; + return object; } @JsonProperty("user") @@ -149,4 +160,90 @@ public Map getSessionProperties() { return sessionProperties; } + + @JsonProperty + public Map> getCatalogSessionProperties() + { + return catalogSessionProperties; + } + + public static class Mapper + implements RowMapper + { + @Override + public SessionMatchSpec map(ResultSet resultSet, StatementContext context) + throws SQLException + { + SessionInfo sessionInfo = getProperties( + Optional.ofNullable(resultSet.getString("session_property_names")), + Optional.ofNullable(resultSet.getString("session_property_values")), + Optional.ofNullable(resultSet.getString("session_property_catalogs"))); + + return new SessionMatchSpec( + Optional.ofNullable(resultSet.getString("user_regex")).map(Pattern::compile), + Optional.ofNullable(resultSet.getString("source_regex")).map(Pattern::compile), + Optional.ofNullable(resultSet.getString("client_tags")).map(tag -> Splitter.on(",").splitToList(tag)), + Optional.ofNullable(resultSet.getString("query_type")), + Optional.ofNullable(resultSet.getString("group_regex")).map(Pattern::compile), + Optional.ofNullable(resultSet.getString("client_info_regex")).map(Pattern::compile), + Optional.of(resultSet.getBoolean("override_session_properties")), + sessionInfo.getSessionProperties(), + sessionInfo.getCatalogProperties()); + } + + private SessionInfo getProperties(Optional names, Optional values, Optional catalogs) + { + if (!names.isPresent()) { + return new SessionInfo(ImmutableMap.of(), ImmutableMap.of()); + } + + checkArgument(catalogs.isPresent(), "names are present, but catalogs are not"); + checkArgument(values.isPresent(), "names are present, but values are not"); + List sessionPropertyNames = Splitter.on(",").splitToList(names.get()); + List sessionPropertyValues = Splitter.on(",").splitToList(values.get()); + List sessionPropertyCatalogs = Splitter.on(",").splitToList(catalogs.get()); + checkArgument(sessionPropertyNames.size() == sessionPropertyValues.size(), + "The number of property names and values should be the same"); + + checkArgument(sessionPropertyNames.size() == sessionPropertyCatalogs.size(), + "The number of property names and catalog values should be the same"); + + Map> catalogSessionProperties = new HashMap<>(); + Map systemSessionProperties = new HashMap<>(); + for (int i = 0; i < sessionPropertyNames.size(); i++) { + if (sessionPropertyCatalogs.get(i).equals(EMPTY_CATALOG)) { + systemSessionProperties.put(sessionPropertyNames.get(i), sessionPropertyValues.get(i)); + } + else { + catalogSessionProperties + .computeIfAbsent(sessionPropertyCatalogs.get(i), k -> new HashMap<>()) + .put(sessionPropertyNames.get(i), sessionPropertyValues.get(i)); + } + } + + return new SessionInfo(systemSessionProperties, catalogSessionProperties); + } + static class SessionInfo + { + private final Map sessionProperties; + private final Map> catalogProperties; + + public SessionInfo(Map sessionProperties, + Map> catalogProperties) + { + this.sessionProperties = sessionProperties; + this.catalogProperties = catalogProperties; + } + + public Map getSessionProperties() + { + return sessionProperties; + } + + public Map> getCatalogProperties() + { + return catalogProperties; + } + } + } } diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerPlugin.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java similarity index 73% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerPlugin.java rename to presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java index bde86e9400f31..b89048a8733f5 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerPlugin.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java @@ -13,16 +13,20 @@ */ package com.facebook.presto.session; +import com.facebook.presto.session.db.DbSessionPropertyManagerFactory; +import com.facebook.presto.session.file.FileSessionPropertyManagerFactory; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.google.common.collect.ImmutableList; -public class FileSessionPropertyManagerPlugin +public class SessionPropertyConfigurationManagerPlugin implements Plugin { @Override public Iterable getSessionPropertyConfigurationManagerFactories() { - return ImmutableList.of(new FileSessionPropertyManagerFactory()); + return ImmutableList.of( + new FileSessionPropertyManagerFactory(), + new DbSessionPropertyManagerFactory()); } } diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java new file mode 100644 index 0000000000000..528d51c395c6e --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.presto.session.AbstractSessionPropertyManager; +import com.facebook.presto.session.SessionMatchSpec; +import com.facebook.presto.spi.session.SessionConfigurationContext; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; + +import javax.inject.Inject; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +/** + * A {@link SessionPropertyConfigurationManager} implementation that connects to a database for fetching information + * about session property overrides given {@link SessionConfigurationContext}. + */ +public class DbSessionPropertyManager + extends AbstractSessionPropertyManager +{ + private final DbSpecsProvider specsProvider; + + @Inject + public DbSessionPropertyManager(DbSpecsProvider specsProvider) + { + this.specsProvider = requireNonNull(specsProvider, "specsProvider is null"); + } + + @Override + protected List getSessionMatchSpecs() + { + return this.specsProvider.get(); + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java new file mode 100644 index 0000000000000..4a94ac0d9e61a --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.airlift.configuration.Config; +import com.facebook.airlift.units.Duration; +import com.facebook.airlift.units.MinDuration; +import jakarta.validation.constraints.NotNull; + +import static java.util.concurrent.TimeUnit.SECONDS; + +public class DbSessionPropertyManagerConfig +{ + private String configDbUrl; + private Duration specsRefreshPeriod = new Duration(10, SECONDS); + + @NotNull + public String getConfigDbUrl() + { + return configDbUrl; + } + + @Config("session-property-manager.db.url") + public DbSessionPropertyManagerConfig setConfigDbUrl(String configDbUrl) + { + this.configDbUrl = configDbUrl; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getSpecsRefreshPeriod() + { + return specsRefreshPeriod; + } + + @Config("session-property-manager.db.refresh-period") + public DbSessionPropertyManagerConfig setSpecsRefreshPeriod(Duration specsRefreshPeriod) + { + this.specsRefreshPeriod = specsRefreshPeriod; + return this; + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java new file mode 100644 index 0000000000000..7a6eba7821ef0 --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.airlift.json.JsonModule; +import com.facebook.presto.spi.resourceGroups.SessionPropertyConfigurationManagerContext; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; +import com.google.inject.Injector; + +import java.util.Map; + +import static com.google.common.base.Throwables.throwIfUnchecked; + +public class DbSessionPropertyManagerFactory + implements SessionPropertyConfigurationManagerFactory +{ + @Override + public String getName() + { + return "db"; + } + + @Override + public SessionPropertyConfigurationManager create(Map config, SessionPropertyConfigurationManagerContext context) + { + try { + Bootstrap app = new Bootstrap(new JsonModule(), new DbSessionPropertyManagerModule()); + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(DbSessionPropertyManager.class); + } + catch (Exception e) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java new file mode 100644 index 0000000000000..48f480b7603aa --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; + +public class DbSessionPropertyManagerModule + implements Module +{ + @Override + public void configure(Binder binder) + { + configBinder(binder).bindConfig(DbSessionPropertyManagerConfig.class); + binder.bind(DbSessionPropertyManager.class).in(Scopes.SINGLETON); + binder.bind(SessionPropertiesDao.class).toProvider(SessionPropertiesDaoProvider.class).in(Scopes.SINGLETON); + binder.bind(DbSpecsProvider.class).to(RefreshingDbSpecsProvider.class).in(Scopes.SINGLETON); + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java new file mode 100644 index 0000000000000..278b52daadb67 --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.presto.session.SessionMatchSpec; + +import java.util.List; +import java.util.function.Supplier; + +/** + * This interface was created to separate the scheduling logic for {@link SessionMatchSpec} loading. This also helps + * us test the core logic of {@link DbSessionPropertyManager} in a modular fashion by letting us use a test + * implementation of this interface. + */ +public interface DbSpecsProvider + extends Supplier> +{ +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java new file mode 100644 index 0000000000000..2694fd5e54e9e --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.session.SessionMatchSpec; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.inject.Inject; + +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; + +/** + * Periodically schedules the loading of specs from the database during initialization. Returns the most recent successfully + * loaded specs on every get() invocation. + */ +public class RefreshingDbSpecsProvider + implements DbSpecsProvider +{ + private static final Logger log = Logger.get(RefreshingDbSpecsProvider.class); + + private final AtomicReference> sessionMatchSpecs = new AtomicReference<>(ImmutableList.of()); + private final SessionPropertiesDao dao; + + private final ScheduledExecutorService executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("RefreshingDbSpecsProvider")); + private final AtomicBoolean started = new AtomicBoolean(); + private final long refreshPeriodMillis; + + @Inject + public RefreshingDbSpecsProvider(DbSessionPropertyManagerConfig config, SessionPropertiesDao dao) + { + requireNonNull(config, "config is null"); + this.dao = requireNonNull(dao, "dao is null"); + this.refreshPeriodMillis = config.getSpecsRefreshPeriod().toMillis(); + dao.createSessionSpecsTable(); + dao.createSessionClientTagsTable(); + dao.createSessionPropertiesTable(); + } + + @PostConstruct + public void initialize() + { + if (!started.getAndSet(true)) { + executor.scheduleWithFixedDelay(this::refresh, 0, refreshPeriodMillis, TimeUnit.MILLISECONDS); + } + } + + @VisibleForTesting + void refresh() + { + try { + sessionMatchSpecs.set(ImmutableList.copyOf(dao.getSessionMatchSpecs())); + } + catch (Throwable e) { + // Catch all exceptions here since throwing an exception from executor#scheduleWithFixedDelay method + // suppresses all future scheduled invocations + log.error(e, "Error loading configuration from database"); + } + } + + @PreDestroy + public void destroy() + { + executor.shutdownNow(); + } + + @Override + public List get() + { + return sessionMatchSpecs.get(); + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java new file mode 100644 index 0000000000000..71a93f0838c4a --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.presto.session.SessionMatchSpec; +import com.google.common.annotations.VisibleForTesting; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.statement.UseRowMapper; + +import java.util.List; + +/** + * Dao should guarantee that the list of SessionMatchSpecs is returned in increasing order of priority. i.e. if two + * rows in the ResultSet specify different values for the same property, the row coming in later will override the + * value set by the row coming in earlier. + */ +public interface SessionPropertiesDao +{ + String SESSION_SPECS_TABLE = "session_specs"; + String CLIENT_TAGS_TABLE = "session_client_tags"; + String PROPERTIES_TABLE = "session_property_values"; + public static String EMPTY_CATALOG = "__NULL__"; + + @SqlUpdate("CREATE TABLE IF NOT EXISTS " + SESSION_SPECS_TABLE + "(\n" + + "spec_id BIGINT NOT NULL AUTO_INCREMENT,\n" + + "user_regex VARCHAR(512),\n" + + "source_regex VARCHAR(512),\n" + + "query_type VARCHAR(512),\n" + + "group_regex VARCHAR(512),\n" + + "client_info_regex VARCHAR(512),\n" + + "override_session_properties TINYINT(1),\n" + + "priority INT NOT NULL,\n" + + "PRIMARY KEY (spec_id)\n" + + ")") + void createSessionSpecsTable(); + + @SqlUpdate("CREATE TABLE IF NOT EXISTS " + CLIENT_TAGS_TABLE + "(\n" + + "tag_spec_id BIGINT NOT NULL,\n" + + "client_tag VARCHAR(512) NOT NULL,\n" + + "PRIMARY KEY (tag_spec_id, client_tag),\n" + + "FOREIGN KEY (tag_spec_id) REFERENCES session_specs (spec_id)\n" + + ")") + void createSessionClientTagsTable(); + + @SqlUpdate("CREATE TABLE IF NOT EXISTS " + PROPERTIES_TABLE + "(\n" + + "property_spec_id BIGINT NOT NULL,\n" + + "session_property_name VARCHAR(512),\n" + + "session_property_value VARCHAR(512),\n" + + "catalog VARCHAR(512),\n" + + "PRIMARY KEY (property_spec_id, session_property_name),\n" + + "FOREIGN KEY (property_spec_id) REFERENCES session_specs (spec_id)\n" + + ")") + void createSessionPropertiesTable(); + + @SqlUpdate("DROP TABLE IF EXISTS " + SESSION_SPECS_TABLE) + void dropSessionSpecsTable(); + + @SqlUpdate("DROP TABLE IF EXISTS " + CLIENT_TAGS_TABLE) + void dropSessionClientTagsTable(); + + @SqlUpdate("DROP TABLE IF EXISTS " + PROPERTIES_TABLE) + void dropSessionPropertiesTable(); + + @SqlQuery("SELECT " + + "S.spec_id,\n" + + "S.user_regex,\n" + + "S.source_regex,\n" + + "S.query_type,\n" + + "S.group_regex,\n" + + "S.client_info_regex,\n" + + "S.override_session_properties,\n" + + "S.client_tags,\n" + + "GROUP_CONCAT(P.session_property_name ORDER BY P.session_property_name) session_property_names,\n" + + "GROUP_CONCAT(P.session_property_value ORDER BY P.session_property_name) session_property_values,\n" + + "GROUP_CONCAT(COALESCE(P.catalog, '" + EMPTY_CATALOG + "') ORDER BY P.session_property_name) session_property_catalogs\n" + + "FROM\n" + + "(SELECT\n" + + "A.spec_id, A.user_regex, A.source_regex, A.query_type, A.group_regex, A.client_info_regex, A.override_session_properties, A.priority,\n" + + "GROUP_CONCAT(DISTINCT B.client_tag) client_tags\n" + + "FROM " + SESSION_SPECS_TABLE + " A\n" + + "LEFT JOIN " + CLIENT_TAGS_TABLE + " B\n" + + "ON A.spec_id = B.tag_spec_id\n" + + "GROUP BY A.spec_id, A.user_regex, A.source_regex, A.query_type, A.group_regex, A.client_info_regex, A.override_session_properties, A.priority)\n" + + " S JOIN\n" + + PROPERTIES_TABLE + " P\n" + + "ON S.spec_id = P.property_spec_id\n" + + "GROUP BY S.spec_id, S.user_regex, S.source_regex, S.query_type, S.group_regex, S.client_info_regex, S.override_session_properties, S.priority, S.client_tags\n" + + "ORDER BY S.priority asc") + @UseRowMapper(SessionMatchSpec.Mapper.class) + List getSessionMatchSpecs(); + + @VisibleForTesting + @SqlUpdate("INSERT INTO " + SESSION_SPECS_TABLE + " (spec_id, user_regex, source_regex, query_type, group_regex, client_info_regex, override_session_properties, priority)\n" + + "VALUES (:spec_id, :user_regex, :source_regex, :query_type, :group_regex, :client_info_regex, :override_session_properties, :priority)") + void insertSpecRow( + @Bind("spec_id") long specId, + @Bind("user_regex") String userRegex, + @Bind("source_regex") String sourceRegex, + @Bind("query_type") String queryType, + @Bind("group_regex") String groupRegex, + @Bind("client_info_regex") String clientInfoRegex, + @Bind("override_session_properties") Integer overrideSessionProperties, + @Bind("priority") int priority); + + @VisibleForTesting + @SqlUpdate("INSERT INTO " + CLIENT_TAGS_TABLE + " (tag_spec_id, client_tag) VALUES (:spec_id, :client_tag)") + void insertClientTag(@Bind("spec_id") long specId, @Bind("client_tag") String clientTag); + + @VisibleForTesting + @SqlUpdate("INSERT INTO " + PROPERTIES_TABLE + " (property_spec_id, session_property_name, session_property_value, catalog)\n" + + "VALUES (:property_spec_id, :session_property_name, :session_property_value, :catalog)") + void insertSessionProperty( + @Bind("property_spec_id") long propertySpecId, + @Bind("session_property_name") String sessionPropertyName, + @Bind("session_property_value") String sessionPropertyValue, + @Bind("catalog") String catalog); +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java new file mode 100644 index 0000000000000..06617a1885396 --- /dev/null +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import org.jdbi.v3.core.Jdbi; +import org.jdbi.v3.sqlobject.SqlObjectPlugin; + +import javax.inject.Inject; +import javax.inject.Provider; + +import java.sql.DriverManager; + +import static java.util.Objects.requireNonNull; + +public class SessionPropertiesDaoProvider + implements Provider +{ + private final SessionPropertiesDao dao; + + @Inject + public SessionPropertiesDaoProvider(DbSessionPropertyManagerConfig config) + { + requireNonNull(config, "config is null"); + requireNonNull(config.getConfigDbUrl(), "db url is null"); + this.dao = Jdbi.create(() -> DriverManager.getConnection(config.getConfigDbUrl())) + .installPlugin(new SqlObjectPlugin()) + .onDemand(SessionPropertiesDao.class); + } + + @Override + public SessionPropertiesDao get() + { + return dao; + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java similarity index 60% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java rename to presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java index 9904baad66869..6f559a00f2b91 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java @@ -11,40 +11,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.file; import com.facebook.airlift.json.JsonCodec; import com.facebook.airlift.json.JsonCodecFactory; import com.facebook.airlift.json.JsonObjectMapperProvider; -import com.facebook.presto.spi.session.SessionConfigurationContext; -import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; +import com.facebook.presto.session.AbstractSessionPropertyManager; +import com.facebook.presto.session.SessionMatchSpec; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; import jakarta.inject.Inject; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class FileSessionPropertyManager - implements SessionPropertyConfigurationManager + extends AbstractSessionPropertyManager { public static final JsonCodec> CODEC = new JsonCodecFactory( () -> new JsonObjectMapperProvider().get().enable(FAIL_ON_UNKNOWN_PROPERTIES)) .listJsonCodec(SessionMatchSpec.class); - private final List sessionMatchSpecs; + private final ImmutableList sessionMatchSpecs; @Inject public FileSessionPropertyManager(FileSessionPropertyManagerConfig config) @@ -53,7 +49,7 @@ public FileSessionPropertyManager(FileSessionPropertyManagerConfig config) Path configurationFile = config.getConfigFile().toPath(); try { - sessionMatchSpecs = CODEC.fromJson(Files.readAllBytes(configurationFile)); + sessionMatchSpecs = ImmutableList.copyOf(CODEC.fromJson(Files.readAllBytes(configurationFile))); } catch (IOException e) { throw new UncheckedIOException(e); @@ -80,31 +76,8 @@ public FileSessionPropertyManager(FileSessionPropertyManagerConfig config) } @Override - public SystemSessionPropertyConfiguration getSystemSessionProperties(SessionConfigurationContext context) + protected List getSessionMatchSpecs() { - // later properties override earlier properties - Map defaultProperties = new HashMap<>(); - Set overridePropertyNames = new HashSet<>(); - for (SessionMatchSpec sessionMatchSpec : sessionMatchSpecs) { - Map newProperties = sessionMatchSpec.match(context); - defaultProperties.putAll(newProperties); - if (sessionMatchSpec.getOverrideSessionProperties().orElse(false)) { - overridePropertyNames.addAll(newProperties.keySet()); - } - } - - // Once a property has been overridden it stays that way and the value is updated by any rule - Map overrideProperties = new HashMap<>(); - for (String propertyName : overridePropertyNames) { - overrideProperties.put(propertyName, defaultProperties.get(propertyName)); - } - - return new SystemSessionPropertyConfiguration(ImmutableMap.copyOf(defaultProperties), ImmutableMap.copyOf(overrideProperties)); - } - - @Override - public Map> getCatalogSessionProperties(SessionConfigurationContext context) - { - return ImmutableMap.of(); + return sessionMatchSpecs; } } diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java similarity index 96% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java rename to presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java index b1e72bed0aedd..54164b968b105 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.file; import com.facebook.airlift.configuration.Config; import jakarta.validation.constraints.NotNull; diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java similarity index 97% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java rename to presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java index aec39239cfbb5..08a49b74f54f5 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerFactory.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.file; import com.facebook.airlift.bootstrap.Bootstrap; import com.facebook.airlift.json.JsonModule; diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java similarity index 96% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java rename to presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java index 03fee8d5207dc..9d50e6144df4c 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerModule.java +++ b/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.file; import com.google.inject.Binder; import com.google.inject.Module; diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java similarity index 72% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java rename to presto-session-property-managers/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java index efd2c0f6b6411..3f09af61b1562 100644 --- a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManager.java +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java @@ -11,34 +11,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.facebook.presto.session; -import com.facebook.airlift.testing.TempFile; import com.facebook.presto.common.resourceGroups.QueryType; import com.facebook.presto.spi.resourceGroups.ResourceGroupId; import com.facebook.presto.spi.session.SessionConfigurationContext; -import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; -import com.facebook.presto.spi.session.SessionPropertyConfigurationManager.SystemSessionPropertyConfiguration; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.testng.annotations.Test; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; -import static com.facebook.presto.session.FileSessionPropertyManager.CODEC; -import static org.testng.Assert.assertEquals; - -public class TestFileSessionPropertyManager +@Test(singleThreaded = true) +public abstract class AbstractTestSessionPropertyManager { - private static final SessionConfigurationContext CONTEXT = new SessionConfigurationContext( + protected static final SessionConfigurationContext CONTEXT = new SessionConfigurationContext( "user", Optional.empty(), Optional.of("source"), @@ -48,6 +39,13 @@ public class TestFileSessionPropertyManager Optional.of("bar"), "testversion"); + protected abstract void assertProperties(Map properties, SessionMatchSpec... spec) + throws IOException; + protected abstract void assertProperties(Map defaultProperties, Map overrideProperties, SessionMatchSpec... spec) + throws IOException; + protected abstract void assertProperties(Map defaultProperties, Map overrideProperties, Map> catalogProperties, SessionMatchSpec... spec) + throws IOException; + @Test public void testResourceGroupMatch() throws IOException @@ -61,7 +59,8 @@ public void testResourceGroupMatch() Optional.of(Pattern.compile("global.pipeline.user_.*")), Optional.empty(), Optional.empty(), - properties); + properties, + ImmutableMap.of()); assertProperties(properties, spec); } @@ -79,7 +78,8 @@ public void testClientTagMatch() Optional.empty(), Optional.empty(), Optional.empty(), - properties); + properties, + ImmutableMap.of()); assertProperties(properties, spec); } @@ -96,7 +96,8 @@ public void testMultipleMatch() Optional.empty(), Optional.empty(), Optional.empty(), - ImmutableMap.of("PROPERTY1", "VALUE1")); + ImmutableMap.of("PROPERTY1", "VALUE1", "PROPERTY3", "VALUE3"), + ImmutableMap.of()); SessionMatchSpec spec2 = new SessionMatchSpec( Optional.empty(), Optional.empty(), @@ -105,9 +106,10 @@ public void testMultipleMatch() Optional.empty(), Optional.empty(), Optional.empty(), - ImmutableMap.of("PROPERTY1", "VALUE1", "PROPERTY2", "VALUE2")); + ImmutableMap.of("PROPERTY1", "VALUE1", "PROPERTY2", "VALUE2"), + ImmutableMap.of()); - assertProperties(ImmutableMap.of("PROPERTY1", "VALUE1", "PROPERTY2", "VALUE2"), spec1, spec2); + assertProperties(ImmutableMap.of("PROPERTY1", "VALUE1", "PROPERTY2", "VALUE2", "PROPERTY3", "VALUE3"), spec1, spec2); } @Test @@ -122,7 +124,8 @@ public void testNoMatch() Optional.of(Pattern.compile("global.interactive.user_.*")), Optional.empty(), Optional.empty(), - ImmutableMap.of("PROPERTY", "VALUE")); + ImmutableMap.of("PROPERTY", "VALUE"), + ImmutableMap.of()); assertProperties(ImmutableMap.of(), spec); } @@ -140,7 +143,8 @@ public void testClientInfoMatch() Optional.empty(), Optional.of(Pattern.compile("bar")), Optional.empty(), - properties); + properties, + ImmutableMap.of()); assertProperties(properties, spec); } @@ -160,7 +164,8 @@ public void testOverride() Optional.empty(), Optional.of(Pattern.compile("bar")), Optional.of(true), - overrideProperties); + overrideProperties, + ImmutableMap.of()); SessionMatchSpec specDefault = new SessionMatchSpec( Optional.empty(), @@ -170,7 +175,8 @@ public void testOverride() Optional.empty(), Optional.of(Pattern.compile("bar")), Optional.empty(), - defaultProperties); + defaultProperties, + ImmutableMap.of()); // PROPERTY1 should be an override property with the value from the default (non-override, higher precedence) // spec. @@ -178,22 +184,23 @@ public void testOverride() assertProperties(defaultProperties, ImmutableMap.of("PROPERTY1", "VALUE2"), specOverride, specDefault); } - private static void assertProperties(Map defaultProperties, SessionMatchSpec... spec) + @Test + public void testCatalogProperty() throws IOException { - assertProperties(defaultProperties, ImmutableMap.of(), spec); - } + ImmutableMap defaultProperties = ImmutableMap.of("PROPERTY1", "VALUE1"); + ImmutableMap> catalogProperties = ImmutableMap.of("CATALOG", ImmutableMap.of("PROPERTY", "VALUE")); + SessionMatchSpec spec = new SessionMatchSpec( + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + defaultProperties, + catalogProperties); - private static void assertProperties(Map defaultProperties, Map overrideProperties, SessionMatchSpec... spec) - throws IOException - { - try (TempFile tempFile = new TempFile()) { - Path configurationFile = tempFile.path(); - Files.write(configurationFile, CODEC.toJsonBytes(Arrays.asList(spec))); - SessionPropertyConfigurationManager manager = new FileSessionPropertyManager(new FileSessionPropertyManagerConfig().setConfigFile(configurationFile.toFile())); - SystemSessionPropertyConfiguration propertyConfiguration = manager.getSystemSessionProperties(CONTEXT); - assertEquals(propertyConfiguration.systemPropertyDefaults, defaultProperties); - assertEquals(propertyConfiguration.systemPropertyOverrides, overrideProperties); - } + assertProperties(defaultProperties, ImmutableMap.of(), catalogProperties, spec); } } diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java new file mode 100644 index 0000000000000..865e8f840edf4 --- /dev/null +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.presto.session.AbstractTestSessionPropertyManager; +import com.facebook.presto.session.SessionMatchSpec; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; +import com.facebook.presto.testing.mysql.MySqlOptions; +import com.facebook.presto.testing.mysql.TestingMySqlServer; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.AfterClass; + +import java.util.Map; +import java.util.regex.Pattern; + +import static org.testng.Assert.assertEquals; + +public abstract class TestDbSessionPropertyManager + extends AbstractTestSessionPropertyManager +{ + private static final MySqlOptions MY_SQL_OPTIONS = MySqlOptions.builder() + .build(); + + private final String driver; + + private final TestingMySqlServer mysqlServer; + + public TestDbSessionPropertyManager(String driver) + throws Exception + { + this.driver = driver; + this.mysqlServer = new TestingMySqlServer("testuser", "testpass", ImmutableList.of(), MY_SQL_OPTIONS); + } + + @AfterClass(alwaysRun = true) + public void tearDown() + throws Exception + { + mysqlServer.close(); + } + + @Override + protected void assertProperties(Map defaultProperties, SessionMatchSpec... spec) + { + assertProperties(defaultProperties, ImmutableMap.of(), ImmutableMap.of(), spec); + } + + @Override + protected void assertProperties(Map defaultProperties, Map overrideProperties, SessionMatchSpec... specs) + { + assertProperties(defaultProperties, overrideProperties, ImmutableMap.of(), specs); + } + + @Override + protected void assertProperties(Map defaultProperties, Map overrideProperties, Map> catalogProperties, SessionMatchSpec... specs) + { + DbSessionPropertyManagerConfig config = new DbSessionPropertyManagerConfig() + .setConfigDbUrl(overrideJdbcUrl(mysqlServer.getJdbcUrl("session") + "&createDatabaseIfNotExist=true")); + + SessionPropertiesDaoProvider sessionPropertiesDaoProvider = new SessionPropertiesDaoProvider(config); + SessionPropertiesDao dao = sessionPropertiesDaoProvider.get(); + RefreshingDbSpecsProvider dbSpecsProvider = new RefreshingDbSpecsProvider(config, sessionPropertiesDaoProvider.get()); + SessionPropertyConfigurationManager manager = new DbSessionPropertyManager(dbSpecsProvider); + int id = 1; + try { + for (SessionMatchSpec spec : specs) { + int finalId = id; + dao.insertSpecRow( + finalId, + spec.getUserRegex().map(Pattern::pattern).orElse(null), + spec.getSourceRegex().map(Pattern::pattern).orElse(null), + spec.getQueryType().orElse(null), + spec.getResourceGroupRegex().map(Pattern::pattern).orElse(null), + spec.getClientInfoRegex().map(Pattern::pattern).orElse(null), + spec.getOverrideSessionProperties().map(val -> val ? 1 : 0).orElse(null), + finalId); + spec.getClientTags().forEach(tag -> dao.insertClientTag(finalId, tag)); + spec.getSessionProperties().forEach((key, value) -> dao.insertSessionProperty(finalId, key, value, null)); + spec.getCatalogSessionProperties().forEach((catalog, property) -> property.forEach((key, value) -> dao.insertSessionProperty(finalId, key, value, catalog))); + id++; + } + dbSpecsProvider.refresh(); + SessionPropertyConfigurationManager.SystemSessionPropertyConfiguration propertyConfiguration = manager.getSystemSessionProperties(CONTEXT); + assertEquals(propertyConfiguration.systemPropertyDefaults, defaultProperties); + assertEquals(propertyConfiguration.systemPropertyOverrides, overrideProperties); + assertEquals(manager.getCatalogSessionProperties(CONTEXT), catalogProperties); + } + finally { + dao.dropSessionPropertiesTable(); + dao.dropSessionClientTagsTable(); + dao.dropSessionSpecsTable(); + dbSpecsProvider.destroy(); + } + } + + protected String overrideJdbcUrl(String url) + { + return url; + } +} diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java new file mode 100644 index 0000000000000..d976eb722390c --- /dev/null +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +import com.facebook.airlift.units.Duration; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestDbSessionPropertyManagerConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(DbSessionPropertyManagerConfig.class) + .setConfigDbUrl(null) + .setSpecsRefreshPeriod(new Duration(10, SECONDS))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("session-property-manager.db.url", "foo") + .put("session-property-manager.db.refresh-period", "50s") + .build(); + + DbSessionPropertyManagerConfig expected = new DbSessionPropertyManagerConfig() + .setConfigDbUrl("foo") + .setSpecsRefreshPeriod(new Duration(50, TimeUnit.SECONDS)); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java new file mode 100644 index 0000000000000..3e440b95d3070 --- /dev/null +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +public class TestDbSessionPropertyManagerMariadb + extends TestDbSessionPropertyManager +{ + public TestDbSessionPropertyManagerMariadb() throws Exception + { + super("mariadb"); + } + + @Override + public String overrideJdbcUrl(String url) + { + return url.replaceFirst("jdbc:mysql:", "jdbc:mariadb:"); + } +} diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java new file mode 100644 index 0000000000000..806f5c8da9cac --- /dev/null +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.db; + +public class TestDbSessionPropertyManagerMysql + extends TestDbSessionPropertyManager +{ + public TestDbSessionPropertyManagerMysql() throws Exception + { + super("mysql"); + } +} diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java new file mode 100644 index 0000000000000..77f5f79ffdb1d --- /dev/null +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.session.file; + +import com.facebook.airlift.testing.TempFile; +import com.facebook.presto.session.AbstractTestSessionPropertyManager; +import com.facebook.presto.session.SessionMatchSpec; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManager; +import com.google.common.collect.ImmutableMap; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; + +import static com.facebook.presto.session.file.FileSessionPropertyManager.CODEC; +import static org.testng.Assert.assertEquals; + +public class TestFileSessionPropertyManager + extends AbstractTestSessionPropertyManager +{ + @Override + protected void assertProperties(Map defaultProperties, SessionMatchSpec... specs) + throws IOException + { + assertProperties(defaultProperties, ImmutableMap.of(), ImmutableMap.of(), specs); + } + + @Override + protected void assertProperties(Map defaultProperties, Map overrideProperties, SessionMatchSpec... specs) + throws IOException + { + assertProperties(defaultProperties, overrideProperties, ImmutableMap.of(), specs); + } + + protected void assertProperties(Map defaultProperties, Map overrideProperties, Map> catalogProperties, SessionMatchSpec... specs) + throws IOException + { + try (TempFile tempFile = new TempFile()) { + Path configurationFile = tempFile.path(); + Files.write(configurationFile, CODEC.toJsonBytes(Arrays.asList(specs))); + SessionPropertyConfigurationManager manager = new FileSessionPropertyManager(new FileSessionPropertyManagerConfig().setConfigFile(configurationFile.toFile())); + SessionPropertyConfigurationManager.SystemSessionPropertyConfiguration propertyConfiguration = manager.getSystemSessionProperties(CONTEXT); + assertEquals(propertyConfiguration.systemPropertyDefaults, defaultProperties); + assertEquals(propertyConfiguration.systemPropertyOverrides, overrideProperties); + assertEquals(manager.getCatalogSessionProperties(CONTEXT), catalogProperties); + } + } +} diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java b/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java similarity index 97% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java rename to presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java index b555593fa09a0..3c6c509a4490e 100644 --- a/presto-session-property-managers/src/test/java/com/facebook/presto/session/TestFileSessionPropertyManagerConfig.java +++ b/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.file; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; diff --git a/presto-spark-testing/src/test/java/com/facebook/presto/spark/testing/TestPrestoSparkLauncherIntegrationSmokeTest.java b/presto-spark-testing/src/test/java/com/facebook/presto/spark/testing/TestPrestoSparkLauncherIntegrationSmokeTest.java index 4d73d96734caf..e826d055e57e8 100644 --- a/presto-spark-testing/src/test/java/com/facebook/presto/spark/testing/TestPrestoSparkLauncherIntegrationSmokeTest.java +++ b/presto-spark-testing/src/test/java/com/facebook/presto/spark/testing/TestPrestoSparkLauncherIntegrationSmokeTest.java @@ -51,7 +51,7 @@ import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; -import static com.facebook.presto.session.FileSessionPropertyManager.CODEC; +import static com.facebook.presto.session.file.FileSessionPropertyManager.CODEC; import static com.facebook.presto.spark.testing.Processes.destroyProcess; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.facebook.presto.tests.QueryAssertions.assertEqualsIgnoreOrder; @@ -181,7 +181,8 @@ public void setUp() Optional.of(Pattern.compile("global.*")), Optional.empty(), Optional.empty(), - properties); + properties, + ImmutableMap.of()); sessionPropertyConfigJsonFile = new File(tempDir, "session-property-config.json"); Files.write(sessionPropertyConfigJsonFile.toPath(), CODEC.toJsonBytes(Collections.singletonList(spec))); sessionPropertyConfig = new File(tempDir, "session-property-configuration.properties"); From c2b76eb13c89b94651e17760e2be876e1de9e419 Mon Sep 17 00:00:00 2001 From: Hazmi Date: Thu, 17 Jul 2025 18:24:43 +0300 Subject: [PATCH 2/2] Split presto-session-property-managers module Currently, the presto-session-property-managers module has gotten large due to the different implementations for file based and db based session property managers. If implemented, this commit will split the module into 3, ``presto-session-property-managers-common`` will contain the API implementation for a session property manager along with some tests. ``presto-file-session-property-manager`` will contain the file based implementation, and ``presto-db-session-property-manager`` will contain the db based implementation. --- CODEOWNERS | 4 +- pom.xml | 25 +++- .../pom.xml | 34 ++--- ...ionPropertyConfigurationManagerPlugin.java | 7 +- .../session/db/DbSessionPropertyManager.java | 0 .../db/DbSessionPropertyManagerConfig.java | 0 .../db/DbSessionPropertyManagerFactory.java | 0 .../db/DbSessionPropertyManagerModule.java | 0 .../presto/session/db/DbSpecsProvider.java | 0 .../session/db/RefreshingDbSpecsProvider.java | 0 .../session/db/SessionPropertiesDao.java | 0 .../db/SessionPropertiesDaoProvider.java | 0 .../db/TestDbSessionPropertyManager.java | 0 .../TestDbSessionPropertyManagerConfig.java | 0 .../TestDbSessionPropertyManagerMariadb.java | 0 .../db/TestDbSessionPropertyManagerMysql.java | 0 .../admin/session-property-managers.rst | 3 +- presto-file-session-property-manager/pom.xml | 139 ++++++++++++++++++ ...ionPropertyConfigurationManagerPlugin.java | 29 ++++ .../file/FileSessionPropertyManager.java | 0 .../FileSessionPropertyManagerConfig.java | 0 .../FileSessionPropertyManagerFactory.java | 0 .../FileSessionPropertyManagerModule.java | 0 .../file/TestFileSessionPropertyManager.java | 0 .../TestFileSessionPropertyManagerConfig.java | 4 +- presto-main/etc/session-property-config.json | 0 .../etc/session-property-config.properties | 0 presto-server/src/main/provisio/presto.xml | 10 +- .../pom.xml | 81 ++++++++++ .../AbstractSessionPropertyManager.java | 0 .../presto/session/SessionMatchSpec.java | 4 +- .../AbstractTestSessionPropertyManager.java | 0 presto-spark-package/pom.xml | 10 +- .../src/main/assembly/presto.xml | 8 +- presto-spark-testing/pom.xml | 8 +- presto-test-coverage/pom.xml | 16 +- 36 files changed, 343 insertions(+), 39 deletions(-) rename {presto-session-property-managers => presto-db-session-property-manager}/pom.xml (88%) rename presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java => presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyConfigurationManagerPlugin.java (77%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java (100%) rename {presto-session-property-managers => presto-db-session-property-manager}/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java (100%) create mode 100644 presto-file-session-property-manager/pom.xml create mode 100644 presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyConfigurationManagerPlugin.java rename {presto-session-property-managers => presto-file-session-property-manager}/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java (100%) rename {presto-session-property-managers => presto-file-session-property-manager}/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java (100%) rename {presto-session-property-managers => presto-file-session-property-manager}/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java (100%) rename {presto-session-property-managers => presto-file-session-property-manager}/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java (100%) rename {presto-session-property-managers => presto-file-session-property-manager}/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java (100%) rename {presto-session-property-managers => presto-file-session-property-manager}/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java (94%) delete mode 100644 presto-main/etc/session-property-config.json delete mode 100644 presto-main/etc/session-property-config.properties create mode 100644 presto-session-property-managers-common/pom.xml rename {presto-session-property-managers => presto-session-property-managers-common}/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java (100%) rename {presto-session-property-managers => presto-session-property-managers-common}/src/main/java/com/facebook/presto/session/SessionMatchSpec.java (98%) rename {presto-session-property-managers => presto-session-property-managers-common}/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java (100%) diff --git a/CODEOWNERS b/CODEOWNERS index faf25f724f484..0b2831ba7ee31 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -76,7 +76,9 @@ /presto-resource-group-managers @prestodb/committers /presto-router @prestodb/committers /presto-server @prestodb/committers -/presto-session-property-managers @prestodb/committers +/presto-session-property-managers-common @prestodb/committers +/presto-file-session-property-manager @prestodb/committers +/presto-db-session-property-manager @prestodb/committers /presto-singlestore @prestodb/committers /presto-spi @prestodb/committers /presto-sqlserver @prestodb/committers diff --git a/pom.xml b/pom.xml index e9b5909ff5d29..1da832a32e881 100644 --- a/pom.xml +++ b/pom.xml @@ -172,7 +172,9 @@ presto-plugin-toolkit presto-resource-group-managers presto-password-authenticators - presto-session-property-managers + presto-session-property-managers-common + presto-db-session-property-manager + presto-file-session-property-manager presto-benchto-benchmarks presto-thrift-api presto-thrift-testing-server @@ -342,7 +344,26 @@ com.facebook.presto - presto-session-property-managers + presto-session-property-managers-common + ${project.version} + + + + com.facebook.presto + presto-session-property-managers-common + ${project.version} + test-jar + + + + com.facebook.presto + presto-db-session-property-manager + ${project.version} + + + + com.facebook.presto + presto-file-session-property-manager ${project.version} diff --git a/presto-session-property-managers/pom.xml b/presto-db-session-property-manager/pom.xml similarity index 88% rename from presto-session-property-managers/pom.xml rename to presto-db-session-property-manager/pom.xml index 0ba95f4c6ee88..d0ef4ca921014 100644 --- a/presto-session-property-managers/pom.xml +++ b/presto-db-session-property-manager/pom.xml @@ -8,17 +8,21 @@ 0.295-SNAPSHOT - presto-session-property-managers - presto-session-property-managers - Presto - Session Property Managers + presto-db-session-property-manager + presto-db-session-property-manager + Presto - DB Session Property Manager presto-plugin ${project.parent.basedir} - true + + com.facebook.presto + presto-session-property-managers-common + + com.facebook.airlift bootstrap @@ -59,11 +63,6 @@ javax.inject - - jakarta.inject - jakarta.inject-api - - com.facebook.airlift concurrent @@ -74,16 +73,6 @@ jakarta.validation-api - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.core - jackson-core - - org.jdbi jdbi3-core @@ -175,5 +164,12 @@ testing-mysql-server-base test + + + com.facebook.presto + presto-session-property-managers-common + test-jar + test + diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyConfigurationManagerPlugin.java similarity index 77% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyConfigurationManagerPlugin.java index b89048a8733f5..bbbc653abd2c9 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionPropertyConfigurationManagerPlugin.java +++ b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyConfigurationManagerPlugin.java @@ -11,22 +11,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.session; +package com.facebook.presto.session.db; -import com.facebook.presto.session.db.DbSessionPropertyManagerFactory; -import com.facebook.presto.session.file.FileSessionPropertyManagerFactory; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; import com.google.common.collect.ImmutableList; -public class SessionPropertyConfigurationManagerPlugin +public class DbSessionPropertyConfigurationManagerPlugin implements Plugin { @Override public Iterable getSessionPropertyConfigurationManagerFactories() { return ImmutableList.of( - new FileSessionPropertyManagerFactory(), new DbSessionPropertyManagerFactory()); } } diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManager.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerConfig.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerFactory.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSessionPropertyManagerModule.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/DbSpecsProvider.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/RefreshingDbSpecsProvider.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDao.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java b/presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java rename to presto-db-session-property-manager/src/main/java/com/facebook/presto/session/db/SessionPropertiesDaoProvider.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java rename to presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManager.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java rename to presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerConfig.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java rename to presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMariadb.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java b/presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java rename to presto-db-session-property-manager/src/test/java/com/facebook/presto/session/db/TestDbSessionPropertyManagerMysql.java diff --git a/presto-docs/src/main/sphinx/admin/session-property-managers.rst b/presto-docs/src/main/sphinx/admin/session-property-managers.rst index 8147d9741f051..4035e8dbb474f 100644 --- a/presto-docs/src/main/sphinx/admin/session-property-managers.rst +++ b/presto-docs/src/main/sphinx/admin/session-property-managers.rst @@ -6,7 +6,8 @@ Administrators can add session properties to control the behavior for subsets of These properties are defaults and can be overridden by users (if authorized to do so). Session properties can be used to control resource usage, enable or disable features, and change query characteristics. Session property managers are pluggable. A session property manager can either -be database-based or file-based. +be database-based or file-based. For production environments, the database-based manager is +recommended as the properties can be updated without requiring a cluster restart. To enable a built-in manager that reads a JSON configuration file, add an ``etc/session-property-config.properties`` file with the following contents: diff --git a/presto-file-session-property-manager/pom.xml b/presto-file-session-property-manager/pom.xml new file mode 100644 index 0000000000000..5c3a4ceb2dddd --- /dev/null +++ b/presto-file-session-property-manager/pom.xml @@ -0,0 +1,139 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.295-SNAPSHOT + + + presto-file-session-property-manager + presto-file-session-property-manager + Presto - File Session Property Manager + presto-plugin + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-session-property-managers-common + + + + com.facebook.airlift + bootstrap + + + + com.facebook.airlift + json + + + + com.facebook.airlift + configuration + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.inject + javax.inject + + + + jakarta.inject + jakarta.inject-api + + + + jakarta.validation + jakarta.validation-api + + + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.core + jackson-core + + + + + com.facebook.presto + presto-spi + provided + + + + com.facebook.presto + presto-common + provided + + + + com.facebook.airlift + units + provided + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + org.openjdk.jol + jol-core + provided + + + + + com.facebook.presto + presto-testng-services + test + + + + org.testng + testng + test + + + + com.facebook.airlift + testing + test + + + + com.facebook.presto + presto-session-property-managers-common + test-jar + test + + + diff --git a/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyConfigurationManagerPlugin.java b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyConfigurationManagerPlugin.java new file mode 100644 index 0000000000000..573c62295572c --- /dev/null +++ b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyConfigurationManagerPlugin.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.session.file; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.session.SessionPropertyConfigurationManagerFactory; +import com.google.common.collect.ImmutableList; + +public class FileSessionPropertyConfigurationManagerPlugin + implements Plugin +{ + @Override + public Iterable getSessionPropertyConfigurationManagerFactories() + { + return ImmutableList.of( + new FileSessionPropertyManagerFactory()); + } +} diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java rename to presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManager.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java rename to presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerConfig.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java rename to presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerFactory.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java b/presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java rename to presto-file-session-property-manager/src/main/java/com/facebook/presto/session/file/FileSessionPropertyManagerModule.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java b/presto-file-session-property-manager/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java rename to presto-file-session-property-manager/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManager.java diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java b/presto-file-session-property-manager/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java similarity index 94% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java rename to presto-file-session-property-manager/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java index 3c6c509a4490e..43403ef267bce 100644 --- a/presto-session-property-managers/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java +++ b/presto-file-session-property-manager/src/test/java/com/facebook/presto/session/file/TestFileSessionPropertyManagerConfig.java @@ -16,7 +16,7 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; -import java.io.File; +import java.nio.file.Paths; import java.util.Map; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; @@ -40,7 +40,7 @@ public void testExplicitPropertyMappings() .build(); FileSessionPropertyManagerConfig expected = new FileSessionPropertyManagerConfig() - .setConfigFile(new File("/test.json")); + .setConfigFile(Paths.get("/test.json").toFile()); assertFullMapping(properties, expected); } diff --git a/presto-main/etc/session-property-config.json b/presto-main/etc/session-property-config.json deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/presto-main/etc/session-property-config.properties b/presto-main/etc/session-property-config.properties deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 14bcfd5b3c31a..8cd6b5826a41d 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -41,8 +41,14 @@ - - + + + + + + + + diff --git a/presto-session-property-managers-common/pom.xml b/presto-session-property-managers-common/pom.xml new file mode 100644 index 0000000000000..098b68e7ea58c --- /dev/null +++ b/presto-session-property-managers-common/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.295-SNAPSHOT + + + presto-session-property-managers-common + presto-session-property-managers-common + Presto - Session Property Managers Common + jar + + + ${project.parent.basedir} + + + + + com.google.guava + guava + + + + org.jdbi + jdbi3-core + + + + + com.facebook.presto + presto-spi + provided + + + + com.facebook.presto + presto-common + provided + + + + com.facebook.airlift + units + provided + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + org.openjdk.jol + jol-core + provided + + + + + com.facebook.presto + presto-testng-services + test + + + + org.testng + testng + test + + + diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java b/presto-session-property-managers-common/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java rename to presto-session-property-managers-common/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java diff --git a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java b/presto-session-property-managers-common/src/main/java/com/facebook/presto/session/SessionMatchSpec.java similarity index 98% rename from presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java rename to presto-session-property-managers-common/src/main/java/com/facebook/presto/session/SessionMatchSpec.java index f5e655c705eef..8d7a88826d90d 100644 --- a/presto-session-property-managers/src/main/java/com/facebook/presto/session/SessionMatchSpec.java +++ b/presto-session-property-managers-common/src/main/java/com/facebook/presto/session/SessionMatchSpec.java @@ -33,12 +33,12 @@ import java.util.Set; import java.util.regex.Pattern; -import static com.facebook.presto.session.db.SessionPropertiesDao.EMPTY_CATALOG; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public class SessionMatchSpec { + public static String emptyCatalog = "__NULL__"; private final Optional userRegex; private final Optional sourceRegex; private final Set clientTags; @@ -211,7 +211,7 @@ private SessionInfo getProperties(Optional names, Optional value Map> catalogSessionProperties = new HashMap<>(); Map systemSessionProperties = new HashMap<>(); for (int i = 0; i < sessionPropertyNames.size(); i++) { - if (sessionPropertyCatalogs.get(i).equals(EMPTY_CATALOG)) { + if (sessionPropertyCatalogs.get(i).equals(emptyCatalog)) { systemSessionProperties.put(sessionPropertyNames.get(i), sessionPropertyValues.get(i)); } else { diff --git a/presto-session-property-managers/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java b/presto-session-property-managers-common/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java similarity index 100% rename from presto-session-property-managers/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java rename to presto-session-property-managers-common/src/test/java/com/facebook/presto/session/AbstractTestSessionPropertyManager.java diff --git a/presto-spark-package/pom.xml b/presto-spark-package/pom.xml index 84ee9009f0bd7..44222dece8b37 100644 --- a/presto-spark-package/pom.xml +++ b/presto-spark-package/pom.xml @@ -152,7 +152,15 @@ com.facebook.presto - presto-session-property-managers + presto-file-session-property-manager + ${project.version} + zip + provided + + + + com.facebook.presto + presto-db-session-property-manager ${project.version} zip provided diff --git a/presto-spark-package/src/main/assembly/presto.xml b/presto-spark-package/src/main/assembly/presto.xml index 165bac55826e0..79446c79d59ba 100644 --- a/presto-spark-package/src/main/assembly/presto.xml +++ b/presto-spark-package/src/main/assembly/presto.xml @@ -85,8 +85,12 @@ plugin/sqlserver - ${project.build.directory}/dependency/presto-session-property-managers-${project.version} - plugin/session-property-managers + ${project.build.directory}/dependency/presto-file-session-property-manager-${project.version} + plugin/file-session-property-manager + + + ${project.build.directory}/dependency/presto-db-session-property-manager-${project.version} + plugin/db-session-property-manager diff --git a/presto-spark-testing/pom.xml b/presto-spark-testing/pom.xml index 6ecb44edc0a25..14d2dcf21d548 100644 --- a/presto-spark-testing/pom.xml +++ b/presto-spark-testing/pom.xml @@ -88,7 +88,13 @@ com.facebook.presto - presto-session-property-managers + presto-session-property-managers-common + test + + + + com.facebook.presto + presto-file-session-property-manager test diff --git a/presto-test-coverage/pom.xml b/presto-test-coverage/pom.xml index dc812c9675e32..849974018f171 100644 --- a/presto-test-coverage/pom.xml +++ b/presto-test-coverage/pom.xml @@ -480,7 +480,21 @@ com.facebook.presto - presto-session-property-managers + presto-session-property-managers-common + ${project.version} + compile + + + + com.facebook.presto + presto-db-session-property-manager + ${project.version} + compile + + + + com.facebook.presto + presto-file-session-property-manager ${project.version} compile