Skip to content

Commit dff7591

Browse files
ashishtadoseaweisbergimjalpreetinfvg
committed
Add a DB based session property manager
Co-authored-by: Ashish Tadose <[email protected]> Co-authored-by: Ariel Weisberg <[email protected]> Co-authored-by: Jalpreet Singh Nanda (:imjalpreet) <[email protected]> Co-authored-by: Hazmi <[email protected]>
1 parent 33498e4 commit dff7591

26 files changed

+1127
-78
lines changed

‎presto-docs/src/main/sphinx/admin/session-property-managers.rst‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@ Session Property Managers
55
Administrators can add session properties to control the behavior for subsets of their workload.
66
These properties are defaults and can be overridden by users (if authorized to do so). Session
77
properties can be used to control resource usage, enable or disable features, and change query
8-
characteristics. Session property managers are pluggable.
8+
characteristics. Session property managers are pluggable. A session property manager can either
9+
be database- or file-based.
910

10-
Add an ``etc/session-property-config.properties`` file with the following contents to enable
11-
the built-in manager that reads a JSON config file:
11+
To enable a built-in manager that reads a JSON configuration file, add an
12+
``etc/session-property-config.properties`` file with the following contents:
1213

1314
.. code-block:: none
1415
@@ -24,6 +25,53 @@ by default. All matching rules contribute to constructing a list of session prop
2425
are applied in the order they are specified. Rules specified later in the file override values
2526
for properties that have been previously encountered.
2627

28+
29+
For the database-based built-in manager, add an
30+
``etc/session-property-config.properties`` file with the following contents:
31+
32+
.. code-block:: none
33+
34+
session-property-manager.db.url=url
35+
session-property-manager.db.username=username
36+
session-property-manager.db.password=password
37+
session-property-manager.db.refresh-period=50s
38+
39+
This database consists of three tables: session specs, client tags and properties.
40+
41+
.. code-block:: text
42+
43+
presto:tpch> DESCRIBE session_specs;
44+
Column | Type | Extra | Comment
45+
-----------------------------+---------+-------+---------
46+
spec_id | bigint | |
47+
user_regex | varchar | |
48+
source_regex | varchar | |
49+
query_type | varchar | |
50+
group_regex | varchar | |
51+
client_info_regex | varchar | |
52+
override_session_properties | tinyint | |
53+
priority | varchar | |
54+
(8 rows)
55+
56+
.. code-block:: text
57+
58+
presto:tpch> DESCRIBE session_client_tags;
59+
Column | Type | Extra | Comment
60+
-------------+---------+-------+---------
61+
tag_spec_id | bigint | |
62+
client_tag | varchar | |
63+
(2 rows)
64+
65+
.. code-block:: text
66+
67+
presto:tpch> DESCRIBE session_property_values;
68+
Column | Type | Extra | Comment
69+
------------------------+---------+-------+---------
70+
property_spec_id | bigint | |
71+
session_property_name | varchar | |
72+
session_property_value | varchar | |
73+
(3 rows)
74+
2775
Match Rules
2876
-----------
2977

‎presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import com.facebook.presto.server.PluginManager;
6262
import com.facebook.presto.server.ServerInfoResource;
6363
import com.facebook.presto.server.ServerMainModule;
64+
import com.facebook.presto.server.SessionPropertyDefaults;
6465
import com.facebook.presto.server.ShutdownAction;
6566
import com.facebook.presto.server.security.ServerSecurityModule;
6667
import com.facebook.presto.spi.ClientRequestFilterFactory;
@@ -159,6 +160,7 @@ public class TestingPrestoServer
159160
private final TestingAccessControlManager accessControl;
160161
private final ProcedureTester procedureTester;
161162
private final Optional<InternalResourceGroupManager<?>> resourceGroupManager;
163+
private final SessionPropertyDefaults sessionPropertyDefaults;
162164
private final SplitManager splitManager;
163165
private final PageSourceManager pageSourceManager;
164166
private final NodePartitioningManager nodePartitioningManager;
@@ -388,6 +390,7 @@ public TestingPrestoServer(
388390
if (!skipLoadingResourceGroupConfigurationManager) {
389391
resourceGroupManager.loadConfigurationManager();
390392
}
393+
sessionPropertyDefaults = injector.getInstance(SessionPropertyDefaults.class);
391394
nodePartitioningManager = injector.getInstance(NodePartitioningManager.class);
392395
planOptimizerManager = injector.getInstance(ConnectorPlanOptimizerManager.class);
393396
clusterMemoryManager = injector.getInstance(ClusterMemoryManager.class);
@@ -403,6 +406,7 @@ else if (resourceManager) {
403406
resourceGroupManager = Optional.empty();
404407
nodePartitioningManager = injector.getInstance(NodePartitioningManager.class);
405408
planOptimizerManager = injector.getInstance(ConnectorPlanOptimizerManager.class);
409+
sessionPropertyDefaults = null;
406410
clusterMemoryManager = null;
407411
statsCalculator = null;
408412
eventListenerManager = ((TestingEventListenerManager) injector.getInstance(EventListenerManager.class));
@@ -414,6 +418,7 @@ else if (coordinatorSidecar) {
414418
queryManager = null;
415419
resourceGroupManager = Optional.empty();
416420
nodePartitioningManager = null;
421+
sessionPropertyDefaults = null;
417422
planOptimizerManager = null;
418423
clusterMemoryManager = null;
419424
statsCalculator = null;
@@ -426,6 +431,7 @@ else if (catalogServer) {
426431
queryManager = null;
427432
resourceGroupManager = Optional.empty();
428433
nodePartitioningManager = null;
434+
sessionPropertyDefaults = null;
429435
planOptimizerManager = null;
430436
clusterMemoryManager = null;
431437
statsCalculator = null;
@@ -437,6 +443,7 @@ else if (catalogServer) {
437443
dispatchManager = null;
438444
queryManager = null;
439445
resourceGroupManager = Optional.empty();
446+
sessionPropertyDefaults = null;
440447
nodePartitioningManager = null;
441448
planOptimizerManager = null;
442449
clusterMemoryManager = null;
@@ -659,6 +666,11 @@ public Optional<InternalResourceGroupManager<?>> getResourceGroupManager()
659666
return resourceGroupManager;
660667
}
661668

669+
public SessionPropertyDefaults getSessionPropertyDefaults()
670+
{
671+
return sessionPropertyDefaults;
672+
}
673+
662674
public InternalNodeManager getNodeManager()
663675
{
664676
return nodeManager;

‎presto-session-property-managers/pom.xml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
<artifactId>bootstrap</artifactId>
2323
</dependency>
2424

25+
<dependency>
26+
<groupId>com.facebook.airlift</groupId>
27+
<artifactId>log</artifactId>
28+
</dependency>
29+
2530
<dependency>
2631
<groupId>com.facebook.airlift</groupId>
2732
<artifactId>json</artifactId>
@@ -32,6 +37,17 @@
3237
<artifactId>configuration</artifactId>
3338
</dependency>
3439

40+
<dependency>
41+
<groupId>org.weakref</groupId>
42+
<artifactId>jmxutils</artifactId>
43+
</dependency>
44+
45+
<dependency>
46+
<groupId>com.google.code.findbugs</groupId>
47+
<artifactId>jsr305</artifactId>
48+
<optional>true</optional>
49+
</dependency>
50+
3551
<dependency>
3652
<groupId>com.google.guava</groupId>
3753
<artifactId>guava</artifactId>
@@ -42,6 +58,11 @@
4258
<artifactId>guice</artifactId>
4359
</dependency>
4460

61+
<dependency>
62+
<groupId>javax.annotation</groupId>
63+
<artifactId>javax.annotation-api</artifactId>
64+
</dependency>
65+
4566
<dependency>
4667
<groupId>javax.inject</groupId>
4768
<artifactId>javax.inject</artifactId>
@@ -62,6 +83,21 @@
6283
<artifactId>jackson-core</artifactId>
6384
</dependency>
6485

86+
<dependency>
87+
<groupId>org.jdbi</groupId>
88+
<artifactId>jdbi3-core</artifactId>
89+
</dependency>
90+
91+
<dependency>
92+
<groupId>org.jdbi</groupId>
93+
<artifactId>jdbi3-sqlobject</artifactId>
94+
</dependency>
95+
96+
<dependency>
97+
<groupId>com.mysql</groupId>
98+
<artifactId>mysql-connector-j</artifactId>
99+
</dependency>
100+
65101
<!-- Presto SPI -->
66102
<dependency>
67103
<groupId>com.facebook.presto</groupId>
@@ -81,12 +117,24 @@
81117
<scope>provided</scope>
82118
</dependency>
83119

120+
<dependency>
121+
<groupId>io.airlift</groupId>
122+
<artifactId>slice</artifactId>
123+
<scope>provided</scope>
124+
</dependency>
125+
84126
<dependency>
85127
<groupId>com.fasterxml.jackson.core</groupId>
86128
<artifactId>jackson-annotations</artifactId>
87129
<scope>provided</scope>
88130
</dependency>
89131

132+
<dependency>
133+
<groupId>org.openjdk.jol</groupId>
134+
<artifactId>jol-core</artifactId>
135+
<scope>provided</scope>
136+
</dependency>
137+
90138
<!-- for testing -->
91139
<dependency>
92140
<groupId>com.facebook.presto</groupId>
@@ -105,5 +153,17 @@
105153
<artifactId>testing</artifactId>
106154
<scope>test</scope>
107155
</dependency>
156+
157+
<dependency>
158+
<groupId>com.facebook.presto</groupId>
159+
<artifactId>testing-mysql-server-8</artifactId>
160+
<scope>test</scope>
161+
</dependency>
162+
163+
<dependency>
164+
<groupId>com.facebook.presto</groupId>
165+
<artifactId>testing-mysql-server-base</artifactId>
166+
<scope>test</scope>
167+
</dependency>
108168
</dependencies>
109169
</project>

‎presto-session-property-managers/src/main/java/com/facebook/presto/session/AbstractSessionPropertyManager.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.session;
15+
16+
import com.facebook.presto.spi.session.SessionConfigurationContext;
17+
import com.facebook.presto.spi.session.SessionPropertyConfigurationManager;
18+
import com.google.common.collect.ImmutableMap;
19+
import com.google.common.collect.Table;
20+
21+
import java.util.HashMap;
22+
import java.util.HashSet;
23+
import java.util.List;
24+
import java.util.Map;
25+
import java.util.Set;
26+
27+
import static com.google.common.collect.ImmutableMap.toImmutableMap;
28+
import static com.google.common.collect.ImmutableTable.toImmutableTable;
29+
30+
public abstract class AbstractSessionPropertyManager
31+
implements SessionPropertyConfigurationManager
32+
{
33+
public static final String DEFAULT_PROPERTIES = "defaultProperties";
34+
public static final String OVERRIDE_PROPERTIES = "overrideProperties";
35+
36+
@Override
37+
public final SystemSessionPropertyConfiguration getSystemSessionProperties(SessionConfigurationContext context)
38+
{
39+
Map<String, Map<String, String>> sessionProperties = getSessionProperties(context);
40+
Map<String, String> defaultProperties = sessionProperties.get(DEFAULT_PROPERTIES);
41+
Map<String, String> overrideProperties = sessionProperties.get(OVERRIDE_PROPERTIES);
42+
43+
return new SystemSessionPropertyConfiguration(ImmutableMap.copyOf(defaultProperties.entrySet().stream()
44+
.filter(property -> !isCatalogSessionProperty(property))
45+
.collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))), ImmutableMap.copyOf(overrideProperties.entrySet().stream()
46+
.filter(property -> !isCatalogSessionProperty(property))
47+
.collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))));
48+
}
49+
50+
@Override
51+
public final Map<String, Map<String, String>> getCatalogSessionProperties(SessionConfigurationContext context)
52+
{
53+
Map<String, Map<String, String>> sessionProperties = getSessionProperties(context);
54+
Table<String, String, String> catalogsSessionProperties = sessionProperties.get(DEFAULT_PROPERTIES).entrySet().stream()
55+
.filter(AbstractSessionPropertyManager::isCatalogSessionProperty)
56+
.collect(toImmutableTable(
57+
catalogProperty -> catalogProperty.getKey().split("\\.", 2)[0],
58+
catalogProperty -> catalogProperty.getKey().split("\\.", 2)[1],
59+
Map.Entry::getValue));
60+
return catalogsSessionProperties.rowMap();
61+
}
62+
63+
protected abstract List<SessionMatchSpec> getSessionMatchSpecs();
64+
65+
private Map<String, Map<String, String>> getSessionProperties(SessionConfigurationContext context)
66+
{
67+
// later properties override earlier properties
68+
Map<String, String> defaultProperties = new HashMap<>();
69+
Set<String> overridePropertyNames = new HashSet<String>();
70+
for (SessionMatchSpec sessionMatchSpec : getSessionMatchSpecs()) {
71+
Map<String, String> newProperties = sessionMatchSpec.match(context);
72+
defaultProperties.putAll(newProperties);
73+
if (sessionMatchSpec.getOverrideSessionProperties().orElse(false)) {
74+
overridePropertyNames.addAll(newProperties.keySet());
75+
}
76+
}
77+
78+
// Once a property has been overridden it stays that way and the value is updated by any rule
79+
Map<String, String> overrideProperties = new HashMap<>();
80+
for (String propertyName : overridePropertyNames) {
81+
overrideProperties.put(propertyName, defaultProperties.get(propertyName));
82+
}
83+
84+
Map<String, Map<String, String>> sessionProperties = new HashMap<>();
85+
sessionProperties.put(DEFAULT_PROPERTIES, defaultProperties);
86+
sessionProperties.put(OVERRIDE_PROPERTIES, overrideProperties);
87+
88+
return sessionProperties;
89+
}
90+
91+
private static boolean isCatalogSessionProperty(Map.Entry<String, String> property)
92+
{
93+
return property.getKey().contains(".");
94+
}
95+
}

0 commit comments

Comments
 (0)