diff --git a/velox/expression/CMakeLists.txt b/velox/expression/CMakeLists.txt index 7f9a5029441..c13191f27e8 100644 --- a/velox/expression/CMakeLists.txt +++ b/velox/expression/CMakeLists.txt @@ -21,6 +21,7 @@ add_library( velox_expression CastExpr.cpp ControlExpr.cpp + DynamicLibraryLoader.cpp EvalCtx.cpp Expr.cpp ExprCompiler.cpp diff --git a/velox/expression/DynamicLibraryLoader.cpp b/velox/expression/DynamicLibraryLoader.cpp new file mode 100644 index 00000000000..8eb2792aa91 --- /dev/null +++ b/velox/expression/DynamicLibraryLoader.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/expression/DynamicLibraryLoader.h" +#include +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::exec { + +static constexpr const char* kSymbolName = "registry"; + +void loadDynamicLibraryFunctions(const char* fileName) { + // Try to dynamically load the shared library. + void* handler = dlopen(fileName, RTLD_NOW); + + if (handler == nullptr) { + VELOX_USER_FAIL("Error while loading shared library: {}", dlerror()); + } + + // Lookup the symbol. + void* registrySymbol = dlsym(handler, kSymbolName); + auto registryFunction = reinterpret_cast(registrySymbol); + char* error = dlerror(); + + if (error != nullptr) { + VELOX_USER_FAIL("Couldn't find Velox registry symbol: {}", error); + } + registryFunction(); +} + +} // namespace facebook::velox::exec diff --git a/velox/expression/DynamicLibraryLoader.h b/velox/expression/DynamicLibraryLoader.h new file mode 100644 index 00000000000..b5cb5b25f94 --- /dev/null +++ b/velox/expression/DynamicLibraryLoader.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace facebook::velox::exec { + +/// Dynamically opens and registers functions defined in a shared library (.so) +/// +/// Given a shared library name (.so), this function will open it using dlopen, +/// search for a "void registry()" C function containing the registration code +/// for the functions defined in library, and execute it. +/// +/// The library being linked needs to provide a function with the following +/// signature: +/// +/// void registry(); +/// +/// The registration function needs to be defined in the top-level namespace, +/// and be enclosed by a extern "C" directive to prevent the compiler from +/// mangling the symbol name. +void loadDynamicLibraryFunctions(const char* fileName); + +} // namespace facebook::velox::exec diff --git a/velox/functions/tests/CMakeLists.txt b/velox/functions/tests/CMakeLists.txt index a55922f0c98..85324c6e811 100644 --- a/velox/functions/tests/CMakeLists.txt +++ b/velox/functions/tests/CMakeLists.txt @@ -17,3 +17,19 @@ add_test(NAME velox_function_registry_test COMMAND velox_function_registry_test) target_link_libraries(velox_function_registry_test velox_function_registry ${GMock} ${GTEST_BOTH_LIBRARIES}) + +# To test functions being added by dynamically linked libraries, we compile +# `MyDynamicFunction.cpp` as a small .so library, and use the +# MY_DYNAMIC_FUNCTION_LIBRARY_PATH macro to locate the .so binary. +add_compile_definitions( + MY_DYNAMIC_FUNCTION_LIBRARY_PATH="${CMAKE_CURRENT_BINARY_DIR}") +add_library(velox_function_my_dynamic SHARED MyDynamicFunction.cpp) + +# Here's the actual test which will dynamically load the library defined above. +add_executable(velox_function_dynamic_link_test DynamicLinkTest.cpp) + +add_test(NAME velox_function_dynamic_link_test + COMMAND velox_function_dynamic_link_test) + +target_link_libraries(velox_function_dynamic_link_test velox_functions_test_lib + velox_function_registry ${GMock} ${GTEST_BOTH_LIBRARIES}) diff --git a/velox/functions/tests/DynamicLinkTest.cpp b/velox/functions/tests/DynamicLinkTest.cpp new file mode 100644 index 00000000000..e2fc3d85b33 --- /dev/null +++ b/velox/functions/tests/DynamicLinkTest.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "velox/expression/DynamicLibraryLoader.h" +#include "velox/functions/FunctionRegistry.h" +#include "velox/functions/prestosql/tests/FunctionBaseTest.h" + +namespace facebook::velox::functions::test { + +class DynamicLinkTest : public FunctionBaseTest {}; + +TEST_F(DynamicLinkTest, dynamicLoad) { + const auto dynamicFunction = [&](std::optional a) { + return evaluateOnce("dynamic_123()", a); + }; + + auto signaturesBefore = getFunctionSignatures().size(); + + // Function does not exist yet. + EXPECT_THROW(dynamicFunction(0), std::invalid_argument); + + // Dynamically load the library. + std::string libraryPath = MY_DYNAMIC_FUNCTION_LIBRARY_PATH; + libraryPath += "/libvelox_function_my_dynamic.so"; + + exec::loadDynamicLibraryFunctions(libraryPath.data()); + + auto signaturesAfter = getFunctionSignatures().size(); + EXPECT_EQ(signaturesAfter, signaturesBefore + 1); + + // Make sure the function exists now. + EXPECT_EQ(123, dynamicFunction(0)); +} + +} // namespace facebook::velox::functions::test diff --git a/velox/functions/tests/MyDynamicFunction.cpp b/velox/functions/tests/MyDynamicFunction.cpp new file mode 100644 index 00000000000..a48ea593969 --- /dev/null +++ b/velox/functions/tests/MyDynamicFunction.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/functions/Udf.h" + +// This file defines a mock function that will be dynamically linked and +// registered. There are no restrictions as to how the function needs to be +// defined, but the library (.so) needs to provide a `void registry()` C +// function in the top-level namespace. +// +// (note the extern "C" directive to prevent the compiler from mangling the +// symbol name). + +namespace facebook::velox::functions { + +template +struct Dynamic123Function { + FOLLY_ALWAYS_INLINE bool call(int64_t& result) { + result = 123; + return true; + } +}; + +} // namespace facebook::velox::functions + +extern "C" { + +void registry() { + facebook::velox:: + registerFunction( + {"dynamic_123"}); +} +}