Add support and example for dynamically linked function

velox/expression/CMakeLists.txt

@@ -21,6 +21,7 @@ add_library(
   velox_expression
   CastExpr.cpp
   ControlExpr.cpp
+  DynamicLibraryLoader.cpp
   EvalCtx.cpp
   Expr.cpp
   ExprCompiler.cpp

velox/expression/DynamicLibraryLoader.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "velox/expression/DynamicLibraryLoader.h"
+#include <dlfcn.h>
+#include "velox/common/base/Exceptions.h"
+
+namespace facebook::velox::exec {
+
+static constexpr const char* kSymbolName = "registry";
+
+void loadDynamicLibraryFunctions(const char* fileName) {
+  // Try to dynamically load the shared library.
+  void* handler = dlopen(fileName, RTLD_NOW);
+
+  if (handler == nullptr) {
+    VELOX_USER_FAIL("Error while loading shared library: {}", dlerror());
+  }
+
+  // Lookup the symbol.
+  void* registrySymbol = dlsym(handler, kSymbolName);
+  auto registryFunction = reinterpret_cast<void (*)()>(registrySymbol);
+  char* error = dlerror();
+
+  if (error != nullptr) {
+    VELOX_USER_FAIL("Couldn't find Velox registry symbol: {}", error);
+  }
+  registryFunction();
+}
+
+} // namespace facebook::velox::exec

velox/expression/DynamicLibraryLoader.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace facebook::velox::exec {
+
+/// Dynamically opens and registers functions defined in a shared library (.so)
+///
+/// Given a shared library name (.so), this function will open it using dlopen,
+/// search for a "void registry()" C function containing the registration code
+/// for the functions defined in library, and execute it.
+///
+/// The library being linked needs to provide a function with the following
+/// signature:
+///
+///   void registry();
+///
+/// The registration function needs to be defined in the top-level namespace,
+/// and be enclosed by a extern "C" directive to prevent the compiler from
+/// mangling the symbol name.
+void loadDynamicLibraryFunctions(const char* fileName);
+
+} // namespace facebook::velox::exec

velox/functions/tests/CMakeLists.txt

@@ -17,3 +17,19 @@ add_test(NAME velox_function_registry_test COMMAND velox_function_registry_test)
 
 target_link_libraries(velox_function_registry_test velox_function_registry
                       ${GMock} ${GTEST_BOTH_LIBRARIES})
+
+# To test functions being added by dynamically linked libraries, we compile
+# `MyDynamicFunction.cpp` as a small .so library, and use the
+# MY_DYNAMIC_FUNCTION_LIBRARY_PATH macro to locate the .so binary.
+add_compile_definitions(
+  MY_DYNAMIC_FUNCTION_LIBRARY_PATH="${CMAKE_CURRENT_BINARY_DIR}")
+add_library(velox_function_my_dynamic SHARED MyDynamicFunction.cpp)
+
+# Here's the actual test which will dynamically load the library defined above.
+add_executable(velox_function_dynamic_link_test DynamicLinkTest.cpp)
+
+add_test(NAME velox_function_dynamic_link_test
+         COMMAND velox_function_dynamic_link_test)
+
+target_link_libraries(velox_function_dynamic_link_test velox_functions_test_lib
+                      velox_function_registry ${GMock} ${GTEST_BOTH_LIBRARIES})

velox/functions/tests/DynamicLinkTest.cpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "velox/expression/DynamicLibraryLoader.h"
+#include "velox/functions/FunctionRegistry.h"
+#include "velox/functions/prestosql/tests/FunctionBaseTest.h"
+
+namespace facebook::velox::functions::test {
+
+class DynamicLinkTest : public FunctionBaseTest {};
+
+TEST_F(DynamicLinkTest, dynamicLoad) {
+  const auto dynamicFunction = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_123()", a);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  // Function does not exist yet.
+  EXPECT_THROW(dynamicFunction(0), std::invalid_argument);
+
+  // Dynamically load the library.
+  std::string libraryPath = MY_DYNAMIC_FUNCTION_LIBRARY_PATH;
+  libraryPath += "/libvelox_function_my_dynamic.so";
+
+  exec::loadDynamicLibraryFunctions(libraryPath.data());
+
+  auto signaturesAfter = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfter, signaturesBefore + 1);
+
+  // Make sure the function exists now.
+  EXPECT_EQ(123, dynamicFunction(0));
+}
+
+} // namespace facebook::velox::functions::test

velox/functions/tests/MyDynamicFunction.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "velox/functions/Udf.h"
+
+// This file defines a mock function that will be dynamically linked and
+// registered. There are no restrictions as to how the function needs to be
+// defined, but the library (.so) needs to provide a `void registry()` C
+// function in the top-level namespace.
+//
+// (note the extern "C" directive to prevent the compiler from mangling the
+// symbol name).
+
+namespace facebook::velox::functions {
+
+template <typename TExecParams>
+struct Dynamic123Function {
+  FOLLY_ALWAYS_INLINE bool call(int64_t& result) {
+    result = 123;
+    return true;
+  }
+};
+
+} // namespace facebook::velox::functions
+
+extern "C" {
+
+void registry() {
+  facebook::velox::
+      registerFunction<facebook::velox::functions::Dynamic123Function, int64_t>(
+          {"dynamic_123"});
+}
+}