Skip to content

Commit 688d401

Browse files
kevintang2022kevintang2022
authored
Native built in namespace manager (prestodb#25826)
Summary: - Add abstract class BuiltInSpecialFunctionNamespaceManager - Add BuiltInNativeFunctionNamespaceManager - Refactor BuiltInPluginFunctionNamespaceManager to extend the abstract class - Deduplicate sidecar function registry logic by moving some of it to presto-main-base module from presto-native-sidecar-plugin module - Add function name conflict logic to FunctionAndTypeManager that overrides SQL built in functions but does not override Java built in functions. - Add retry logic in to fetch function registry from worker: retry interval is every 1 minute Note: `show functions` will show both built in functions in the same namespace. This is already similar behavior to regular Native sidecar namespace enabled with default presto.default prefix. The `show functions` logic is not addressed in this change. Can add some unit tests for show functions as well Tests: Added unit tests that enable to flag for this feature, and it is overriding the SQL function implementation properly. ## Release Notes Please follow [release notes guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines) and fill in the release notes below. ``` == NO RELEASE NOTE == ```
1 parent f3907c8 commit 688d401

File tree

29 files changed

+1230
-367
lines changed

29 files changed

+1230
-367
lines changed

‎pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@
153153
<module>presto-parser</module>
154154
<module>presto-main-base</module>
155155
<module>presto-main</module>
156+
<module>presto-built-in-worker-function-tools</module>
156157
<module>presto-ml</module>
157158
<module>presto-geospatial-toolkit</module>
158159
<module>presto-benchmark</module>
@@ -815,6 +816,12 @@
815816
<version>${project.version}</version>
816817
</dependency>
817818

819+
<dependency>
820+
<groupId>com.facebook.presto</groupId>
821+
<artifactId>presto-built-in-worker-function-tools</artifactId>
822+
<version>${project.version}</version>
823+
</dependency>
824+
818825
<dependency>
819826
<groupId>com.facebook.presto</groupId>
820827
<artifactId>presto-main-base</artifactId>

‎presto-built-in-worker-function-tools/pom.xml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<parent>
4+
<artifactId>presto-root</artifactId>
5+
<groupId>com.facebook.presto</groupId>
6+
<version>0.295-SNAPSHOT</version>
7+
</parent>
8+
<modelVersion>4.0.0</modelVersion>
9+
10+
<artifactId>presto-built-in-worker-function-tools</artifactId>
11+
<name>presto-built-in-worker-function-tools</name>
12+
13+
<properties>
14+
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
15+
<project.build.targetJdk>17</project.build.targetJdk>
16+
<air.check.skip-modernizer>true</air.check.skip-modernizer>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>com.facebook.presto</groupId>
22+
<artifactId>presto-spi</artifactId>
23+
</dependency>
24+
<dependency>
25+
<groupId>com.facebook.presto</groupId>
26+
<artifactId>presto-function-namespace-managers-common</artifactId>
27+
</dependency>
28+
<dependency>
29+
<groupId>com.facebook.airlift</groupId>
30+
<artifactId>http-client</artifactId>
31+
</dependency>
32+
<dependency>
33+
<groupId>com.google.inject</groupId>
34+
<artifactId>guice</artifactId>
35+
</dependency>
36+
<dependency>
37+
<groupId>com.google.guava</groupId>
38+
<artifactId>guava</artifactId>
39+
</dependency>
40+
<dependency>
41+
<groupId>com.facebook.airlift</groupId>
42+
<artifactId>json</artifactId>
43+
</dependency>
44+
<dependency>
45+
<groupId>com.facebook.presto</groupId>
46+
<artifactId>presto-common</artifactId>
47+
</dependency>
48+
<dependency>
49+
<groupId>com.facebook.airlift</groupId>
50+
<artifactId>log</artifactId>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.facebook.airlift</groupId>
54+
<artifactId>configuration</artifactId>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.testng</groupId>
58+
<artifactId>testng</artifactId>
59+
<scope>test</scope>
60+
</dependency>
61+
</dependencies>
62+
</project>

‎presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/ForNativeFunctionRegistryInfo.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.builtin.tools;
15+
16+
import com.google.inject.BindingAnnotation;
17+
18+
import java.lang.annotation.Retention;
19+
20+
import static java.lang.annotation.RetentionPolicy.RUNTIME;
21+
22+
@Retention(RUNTIME)
23+
@BindingAnnotation
24+
public @interface ForNativeFunctionRegistryInfo
25+
{
26+
}

‎presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/NativeSidecarFunctionRegistryTool.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.builtin.tools;
15+
16+
import com.facebook.airlift.http.client.HttpClient;
17+
import com.facebook.airlift.http.client.HttpUriBuilder;
18+
import com.facebook.airlift.http.client.JsonResponseHandler;
19+
import com.facebook.airlift.http.client.Request;
20+
import com.facebook.airlift.json.JsonCodec;
21+
import com.facebook.airlift.log.Logger;
22+
import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata;
23+
import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap;
24+
import com.facebook.presto.spi.Node;
25+
import com.facebook.presto.spi.NodeManager;
26+
import com.facebook.presto.spi.PrestoException;
27+
import com.facebook.presto.spi.StandardErrorCode;
28+
import com.facebook.presto.spi.function.SqlFunction;
29+
import com.google.common.collect.ImmutableMap;
30+
31+
import java.net.URI;
32+
import java.util.List;
33+
import java.util.Map;
34+
35+
import static com.google.common.collect.ImmutableList.toImmutableList;
36+
import static java.util.Objects.requireNonNull;
37+
38+
public class NativeSidecarFunctionRegistryTool
39+
implements WorkerFunctionRegistryTool
40+
{
41+
private final int maxRetries;
42+
private final long retryDelayMs;
43+
private static final Logger log = Logger.get(NativeSidecarFunctionRegistryTool.class);
44+
private final JsonCodec<Map<String, List<JsonBasedUdfFunctionMetadata>>> nativeFunctionSignatureMapJsonCodec;
45+
private final NodeManager nodeManager;
46+
private final HttpClient httpClient;
47+
private static final String FUNCTION_SIGNATURES_ENDPOINT = "/v1/functions";
48+
49+
public NativeSidecarFunctionRegistryTool(
50+
HttpClient httpClient,
51+
JsonCodec<Map<String, List<JsonBasedUdfFunctionMetadata>>> nativeFunctionSignatureMapJsonCodec,
52+
NodeManager nodeManager,
53+
int nativeSidecarRegistryToolNumRetries,
54+
long nativeSidecarRegistryToolRetryDelayMs)
55+
{
56+
this.nativeFunctionSignatureMapJsonCodec =
57+
requireNonNull(nativeFunctionSignatureMapJsonCodec, "nativeFunctionSignatureMapJsonCodec is null");
58+
this.nodeManager = requireNonNull(nodeManager, "nodeManager is null");
59+
this.httpClient = requireNonNull(httpClient, "typeManager is null");
60+
this.maxRetries = nativeSidecarRegistryToolNumRetries;
61+
this.retryDelayMs = nativeSidecarRegistryToolRetryDelayMs;
62+
}
63+
64+
@Override
65+
public List<? extends SqlFunction> getWorkerFunctions()
66+
{
67+
return getNativeFunctionSignatureMap()
68+
.getUDFSignatureMap()
69+
.entrySet()
70+
.stream()
71+
.flatMap(entry -> entry.getValue().stream()
72+
.map(metaInfo -> WorkerFunctionUtil.createSqlInvokedFunction(entry.getKey(), metaInfo, "presto")))
73+
.collect(toImmutableList());
74+
}
75+
76+
private UdfFunctionSignatureMap getNativeFunctionSignatureMap()
77+
{
78+
try {
79+
Request request = Request.Builder.prepareGet().setUri(getSidecarLocationOnStartup(nodeManager, maxRetries, retryDelayMs)).build();
80+
Map<String, List<JsonBasedUdfFunctionMetadata>> nativeFunctionSignatureMap = httpClient.execute(request, JsonResponseHandler.createJsonResponseHandler(nativeFunctionSignatureMapJsonCodec));
81+
return new UdfFunctionSignatureMap(ImmutableMap.copyOf(nativeFunctionSignatureMap));
82+
}
83+
catch (Exception e) {
84+
throw new PrestoException(StandardErrorCode.INVALID_ARGUMENTS, "Failed to get functions from sidecar.", e);
85+
}
86+
}
87+
88+
public static URI getSidecarLocationOnStartup(NodeManager nodeManager, int maxRetries, long retryDelayMs)
89+
{
90+
Node sidecarNode = null;
91+
for (int attempt = 1; attempt <= maxRetries; attempt++) {
92+
try {
93+
sidecarNode = nodeManager.getSidecarNode();
94+
if (sidecarNode != null) {
95+
break;
96+
}
97+
}
98+
catch (Exception e) {
99+
log.error("Error getting sidecar node (attempt " + attempt + "): " + e.getMessage());
100+
if (attempt == maxRetries) {
101+
throw new RuntimeException("Failed to get sidecar node", e);
102+
}
103+
else {
104+
try {
105+
Thread.sleep(retryDelayMs);
106+
}
107+
catch (InterruptedException ie) {
108+
Thread.currentThread().interrupt();
109+
throw new RuntimeException("Retry fetching sidecar function registry interrupted", ie);
110+
}
111+
}
112+
}
113+
}
114+
115+
return HttpUriBuilder
116+
.uriBuilderFrom(sidecarNode.getHttpUri())
117+
.appendPath(FUNCTION_SIGNATURES_ENDPOINT)
118+
.build();
119+
}
120+
}

‎presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/NativeSidecarRegistryToolConfig.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.builtin.tools;
15+
16+
import com.facebook.airlift.configuration.Config;
17+
import com.facebook.airlift.configuration.ConfigDescription;
18+
19+
import java.time.Duration;
20+
21+
public class NativeSidecarRegistryToolConfig
22+
{
23+
private int nativeSidecarRegistryToolNumRetries = 8;
24+
private long nativeSidecarRegistryToolRetryDelayMs = Duration.ofMinutes(1).toMillis();
25+
26+
public int getNativeSidecarRegistryToolNumRetries()
27+
{
28+
return nativeSidecarRegistryToolNumRetries;
29+
}
30+
31+
@Config("native-sidecar-registry-tool.num-retries")
32+
@ConfigDescription("Max times to retry fetching sidecar node")
33+
public NativeSidecarRegistryToolConfig setNativeSidecarRegistryToolNumRetries(int nativeSidecarRegistryToolNumRetries)
34+
{
35+
this.nativeSidecarRegistryToolNumRetries = nativeSidecarRegistryToolNumRetries;
36+
return this;
37+
}
38+
39+
public long getNativeSidecarRegistryToolRetryDelayMs()
40+
{
41+
return nativeSidecarRegistryToolRetryDelayMs;
42+
}
43+
44+
@Config("native-sidecar-registry-tool.retry-delay-ms")
45+
@ConfigDescription("Cooldown period to retry when fetching sidecar node fails")
46+
public NativeSidecarRegistryToolConfig setNativeSidecarRegistryToolRetryDelayMs(long nativeSidecarRegistryToolRetryDelayMs)
47+
{
48+
this.nativeSidecarRegistryToolRetryDelayMs = nativeSidecarRegistryToolRetryDelayMs;
49+
return this;
50+
}
51+
}

‎presto-built-in-worker-function-tools/src/main/java/com/facebook/presto/builtin/tools/WorkerFunctionRegistryTool.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.builtin.tools;
15+
16+
import com.facebook.presto.spi.function.SqlFunction;
17+
18+
import java.util.List;
19+
20+
public interface WorkerFunctionRegistryTool
21+
{
22+
List<? extends SqlFunction> getWorkerFunctions();
23+
}

0 commit comments

Comments
 (0)