[WIP] MacOS CMake compatibility for dynamic functions

velox/common/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(base)
 add_subdirectory(caching)
 add_subdirectory(compression)
 add_subdirectory(config)
+add_subdirectory(dynamic_registry)
 add_subdirectory(encode)
 add_subdirectory(file)
 add_subdirectory(hyperloglog)

velox/common/dynamic_registry/CMakeLists.txt

@@ -0,0 +1,21 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+velox_add_library(velox_dynamic_function_loader DynamicLibraryLoader.cpp)
+
+velox_link_libraries(velox_dynamic_function_loader PRIVATE velox_exception)
+
+if(${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif()

velox/common/dynamic_registry/DynamicLibraryLoader.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <dlfcn.h>
+#include <iostream>
+#include "velox/common/base/Exceptions.h"
+
+namespace facebook::velox {
+
+static constexpr const char* kSymbolName = "registry";
+
+void loadDynamicLibrary(const char* fileName) {
+  // Try to dynamically load the shared library.
+  void* handler = dlopen(fileName, RTLD_NOW);
+
+  if (handler == nullptr) {
+    VELOX_USER_FAIL("Error while loading shared library: {}", dlerror());
+  }
+
+  // Lookup the symbol.
+  void* registrySymbol = dlsym(handler, kSymbolName);
+  auto loadUserLibrary = reinterpret_cast<void (*)()>(registrySymbol);
+  char* error = dlerror();
+
+  if (error != nullptr) {
+    VELOX_USER_FAIL("Couldn't find Velox registry symbol: {}", error);
+  }
+  // Invoke the registry function.
+  loadUserLibrary();
+  LOG(INFO) << "Loaded: " << fileName;
+}
+
+} // namespace facebook::velox

velox/common/dynamic_registry/DynamicLibraryLoader.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+namespace facebook::velox {
+
+/// Dynamically opens and registers functions defined in a shared library.
+///
+/// The library being linked needs to provide a function with the following
+/// signature:
+///
+/// void registry();
+///
+/// The registration function needs to be defined in the top-level namespace,
+/// and be enclosed by a extern "C" directive to prevent the compiler from
+/// mangling the symbol name.
+
+/// The function uses dlopen to load the shared library.
+/// It then searches for the "void registry()" C function which typically contains all
+/// the registration code for the UDFs defined in library. After locating the function it
+/// executes the registration bringing the UDFs in the scope of the Velox runtime.
+/// 
+/// If the same library is loaded twice then a no-op scenerio will happen.
+
+void loadDynamicLibrary(const char* fileName);
+
+} // namespace facebook::velox

velox/common/dynamic_registry/README.md

@@ -0,0 +1,15 @@
+# Dynamic Loading of Velox Extensions
+
+This generic utility adds extensibility features to load User Defined Functions (UDFs), connectors, or types without having to fork and build Velox, through the use of shared libraries.
+
+## Getting started
+1. Create a cpp file for your dynamic library
+For dynamically loaded function registration, the format followed is mirrored of that of built-in function registration with some noted differences. Using [MyDynamicTestFunction.cpp](tests/MyDynamicTestFunction.cpp) as an example, the function uses the extern "C" keyword to protect against name mangling. A registry() function call is also necessary here.
+
+2. Register functions dynamically by creating .dylib (MacOS) or .so (Linux) shared libraries.
+These shared libraries may be made using CMakeLists like the following:
+
+```
+add_library(name_of_dynamic_fn SHARED TestFunction.cpp)
+target_link_libraries(name_of_dynamic_fn PRIVATE xsimd fmt::fmt velox_expression)
+```

velox/common/dynamic_registry/tests/CMakeLists.txt

@@ -0,0 +1,103 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To test functions being added by dynamically linked libraries, we compile
+# `MyDynamicFunction.cpp` as a small .so library, and use the
+# VELOX_TEST_DYNAMIC_LIBRARY_PATH macro to locate the .so binary.
+
+add_library(velox_function_my_dynamic SHARED MyDynamicFunction.cpp)
+add_library(velox_int_function_my_dynamic SHARED MyDynamicIntFunction.cpp)
+add_library(velox_str_function_my_dynamic SHARED MyDynamicStrFunction.cpp)
+add_library(velox_function_same_twice_my_dynamic SHARED MyDynamicSameTwiceFunction.cpp)
+add_library(velox_function_err_my_dynamic SHARED MyDynamicErrFunction.cpp)
+
+
+if(APPLE)
+target_link_libraries(
+  velox_function_my_dynamic
+  PRIVATE
+    fmt::fmt
+    Folly::folly
+    gflags::gflags
+    xsimd
+)
+
+target_link_libraries(
+  velox_int_function_my_dynamic
+  PRIVATE
+    fmt::fmt
+    Folly::folly
+    gflags::gflags
+    xsimd
+)
+
+target_link_libraries(
+  velox_str_function_my_dynamic
+  PRIVATE
+    fmt::fmt
+    Folly::folly
+    gflags::gflags
+    xsimd
+)
+
+target_link_libraries(
+  velox_function_same_twice_my_dynamic
+  PRIVATE
+    fmt::fmt
+    Folly::folly
+    gflags::gflags
+    xsimd
+)
+
+target_link_libraries(
+  velox_function_err_my_dynamic
+  PRIVATE
+    fmt::fmt
+    Folly::folly
+    gflags::gflags
+    xsimd
+)
+  target_link_options(velox_function_my_dynamic PRIVATE "-Wl,-undefined,dynamic_lookup")
+  target_link_options(velox_int_function_my_dynamic PRIVATE "-Wl,-undefined,dynamic_lookup")
+  target_link_options(velox_str_function_my_dynamic PRIVATE "-Wl,-undefined,dynamic_lookup")
+  target_link_options(velox_function_same_twice_my_dynamic PRIVATE "-Wl,-undefined,dynamic_lookup")
+  target_link_options(velox_function_err_my_dynamic PRIVATE "-Wl,-undefined,dynamic_lookup")
+else()
+  target_link_libraries(velox_function_my_dynamic xsimd fmt)
+  target_link_libraries(velox_int_function_my_dynamic xsimd fmt)
+  target_link_libraries(velox_str_function_my_dynamic xsimd fmt)
+  target_link_libraries(velox_function_same_twice_my_dynamic xsimd fmt)
+  target_link_libraries(velox_function_err_my_dynamic xsimd fmt)
+endif()
+
+# Here's the actual test which will dynamically load the library defined above.
+add_executable(velox_function_dynamic_link_test DynamicLinkTest.cpp)
+
+target_compile_definitions(velox_function_dynamic_link_test PRIVATE
+  VELOX_TEST_DYNAMIC_LIBRARY_PATH="${CMAKE_CURRENT_BINARY_DIR}")
+target_compile_definitions(velox_function_dynamic_link_test PRIVATE
+  VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX="${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+add_test(NAME velox_function_dynamic_link_test
+         COMMAND velox_function_dynamic_link_test)
+
+target_link_libraries(
+  velox_function_dynamic_link_test
+  velox_functions_test_lib
+  velox_dynamic_function_loader
+  velox_function_registry
+  xsimd
+  GTest::gmock
+  GTest::gtest
+  GTest::gtest_main)

velox/common/dynamic_registry/tests/DynamicLinkTest.cpp

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "velox/common/base/tests/GTestUtils.h"
+#include "velox/common/dynamic_registry/DynamicLibraryLoader.h"
+#include "velox/expression/SimpleFunctionRegistry.h"
+#include "velox/functions/FunctionRegistry.h"
+#include "velox/functions/prestosql/tests/utils/FunctionBaseTest.h" 
+
+namespace facebook::velox::functions::test {
+
+class DynamicLinkTest : public FunctionBaseTest {
+};
+
+std::string getLibraryPath(std::string filename){
+  return fmt::format("{}/{}{}",VELOX_TEST_DYNAMIC_LIBRARY_PATH, filename, VELOX_TEST_DYNAMIC_LIBRARY_PATH_SUFFIX);
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadFunc) {
+  const auto dynamicFunction = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_1()", a);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  VELOX_ASSERT_THROW(
+      dynamicFunction(0), "Scalar function doesn't exist: dynamic_1.");
+
+  std::string libraryPath = getLibraryPath("libvelox_function_my_dynamic");
+  loadDynamicLibrary(libraryPath.data());
+  auto signaturesAfter = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfter, signaturesBefore + 1);
+  EXPECT_EQ(123, dynamicFunction(0));
+
+  auto& registry = exec::simpleFunctions();
+  auto resolved = registry.resolveFunction("dynamic_1", {});
+  EXPECT_EQ(TypeKind::BIGINT, resolved->type()->kind());
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadSameFuncTwice) {
+  const auto dynamicFunction = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_2()", a);
+  };
+  auto& registry = exec::simpleFunctions();
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  VELOX_ASSERT_THROW(
+      dynamicFunction(0), "Scalar function doesn't exist: dynamic_2.");
+
+  std::string libraryPath = getLibraryPath("libvelox_function_same_twice_my_dynamic");
+  loadDynamicLibrary(libraryPath.data());
+  auto signaturesAfterFirst = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterFirst, signaturesBefore + 1);
+  EXPECT_EQ(123, dynamicFunction(0));
+  auto resolvedAfterFirst = registry.resolveFunction("dynamic_2", {});
+  EXPECT_EQ(TypeKind::BIGINT, resolvedAfterFirst->type()->kind());
+
+  loadDynamicLibrary(libraryPath.data());
+  auto signaturesAfterSecond = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterSecond, signaturesAfterFirst);
+  auto resolvedAfterSecond = registry.resolveFunction("dynamic_2", {});
+  EXPECT_EQ(TypeKind::BIGINT, resolvedAfterSecond->type()->kind());
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadTwoOfTheSameName) {
+  const auto dynamicFunctionInt = [&](std::optional<double> a) {
+    return evaluateOnce<int64_t>("dynamic_3()", a);
+  };
+  const auto dynamicFunctionStr = [&](std::optional<std::string> a) {
+    return evaluateOnce<std::string>("dynamic_3()", a);
+  };
+
+  auto& registry = exec::simpleFunctions();
+  auto signaturesBefore = getFunctionSignatures().size();
+
+  VELOX_ASSERT_THROW(
+      dynamicFunctionStr("0"), "Scalar function doesn't exist: dynamic_3.");
+
+  std::string libraryPath = getLibraryPath("libvelox_str_function_my_dynamic");
+  loadDynamicLibrary(libraryPath.data());
+  auto signaturesAfterFirst = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterFirst, signaturesBefore + 1);
+  EXPECT_EQ("123", dynamicFunctionStr("0"));
+  auto resolved = registry.resolveFunction("dynamic_3", {});
+  EXPECT_EQ(TypeKind::VARCHAR, resolved->type()->kind());
+
+  VELOX_ASSERT_THROW(
+    dynamicFunctionInt(0), 
+    "Expression evaluation result is not of expected type: dynamic_3() -> CONSTANT vector of type VARCHAR");
+
+  std::string libraryPathInt = getLibraryPath("libvelox_int_function_my_dynamic");
+  loadDynamicLibrary(libraryPathInt.data());
+
+  // The first function loaded should be rewritten.
+  VELOX_ASSERT_THROW(
+    dynamicFunctionStr("0"), 
+    "Expression evaluation result is not of expected type: dynamic_3() -> CONSTANT vector of type BIGINT");
+  EXPECT_EQ(123, dynamicFunctionInt(0));
+  auto signaturesAfterSecond = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfterSecond, signaturesAfterFirst);
+  auto resolvedAfterSecond = registry.resolveFunction("dynamic_3", {});
+  EXPECT_EQ(TypeKind::BIGINT, resolvedAfterSecond->type()->kind());
+
+}
+
+TEST_F(DynamicLinkTest, dynamicLoadErrFunc) {
+  const auto dynamicFunctionErr = [&](const std::optional<int64_t> a, std::optional<int64_t> b) {
+    return evaluateOnce<int64_t>("dynamic_4(c0)", a, b);
+  };
+
+  const auto dynamicFunction = [&](const facebook::velox::RowVectorPtr& arr) {
+    return evaluateOnce<int64_t>("dynamic_4(c0)", arr);
+  };
+
+  auto signaturesBefore = getFunctionSignatures().size();
+  VELOX_ASSERT_THROW(
+      dynamicFunctionErr(0,0), "Scalar function doesn't exist: dynamic_4.");
+
+  std::string libraryPath = getLibraryPath("libvelox_function_err_my_dynamic");
+  loadDynamicLibrary(libraryPath.data());
+
+  auto signaturesAfter = getFunctionSignatures().size();
+  EXPECT_EQ(signaturesAfter, signaturesBefore + 1);
+
+  // Expecting a fail because we are not passing in an array.
+  VELOX_ASSERT_THROW(
+    dynamicFunctionErr(0,0), 
+    "Scalar function signature is not supported: dynamic_4(BIGINT). Supported signatures: (array(bigint)) -> bigint.");
+
+  auto check = makeRowVector({ makeNullableArrayVector(std::vector<std::vector<std::optional<int64_t>>>{{0, 1, 3, 4, 5, 6, 7, 8, 9}})});
+
+  // Expecting a success because we are passing in an array.
+  EXPECT_EQ(123, dynamicFunction(check));
+}
+
+} // namespace facebook::velox::functions::test