diff --git a/taskfile.yaml b/taskfile.yaml index 0f6cad6f..96705498 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -3,12 +3,16 @@ version: "3" includes: lint: "taskfiles/lint.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + velox-connector: "taskfiles/velox-connector/main.yaml" vars: G_BUILD_DIR: "{{.ROOT_DIR}}/build" tasks: clean: + deps: + - "lint:clean" + - "velox-connector:clean" cmd: "rm -rf '{{.G_BUILD_DIR}}'" init: diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 6825cd5f..e63b2a8f 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -8,6 +8,9 @@ tasks: cmds: - task: "check-yaml" + clean: + cmd: "rm -rf '{{.G_LINT_VENV_DIR}}'" + fix: cmds: - task: "fix-yaml" diff --git a/taskfiles/velox-connector/deps.yaml b/taskfiles/velox-connector/deps.yaml new file mode 100644 index 00000000..9ca0103f --- /dev/null +++ b/taskfiles/velox-connector/deps.yaml @@ -0,0 +1,346 @@ +version: "3" + +includes: + yscope-dev-utils: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + +vars: + G_DEPS_CPP_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_DIR}}/cmake-settings" + + # Library names (must match CMake package names for settings files) + G_BOOST_LIB_NAME: "Boost" + G_FMT_LIB_NAME: "fmt" + G_GSL_LIB_NAME: "Microsoft.GSL" + G_YSTDLIB_LIB_NAME: "ystdlib" + +tasks: + install-all: + cmds: + - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake" + DEP_TASK: "install-all-parallel" + + install-all-parallel: + internal: true + run: "once" + deps: + - "absl" + - "antlr-runtime" + - "boost" + - "date" + - "fmt" + - "log-surgeon" + - "microsoft-gsl" + - "msgpack-cxx" + - "nlohmann-json" + - "simdjson" + - "spdlog" + - "ystdlib" + - "zstd" + + absl: + internal: true + vars: + VERSION: "20250512.0" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_CXX_STANDARD=20" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + CMAKE_PACKAGE_NAME: "absl" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "7262daa7c1711406248c10f41026d685e88223bc92817d16fb93c19adb57f669" + TAR_URL: "https://github.com/abseil/abseil-cpp/releases/download/{{.VERSION}}/\ + abseil-cpp-{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + antlr-runtime: + internal: true + vars: + BUILD_DIR: "{{.G_DEPS_CPP_DIR}}/antlr4-runtime-build" + SOURCE_DIR: "{{.G_DEPS_CPP_DIR}}/antlr4-runtime-extracted" + VERSION: "4.13.2" + run: "once" + + # NOTE: We can't use `yscope-dev-utils:cmake:install-remote-tar` (and by extension + # `utils:install-remote-cmake-lib`) since it doesn't yet support the `INCLUDE_PATTERNS` and + # `NUM_COMPONENTS_TO_STRIP` parameters. + cmds: + - task: "yscope-dev-utils:remote:download-and-extract-tar" + vars: + FILE_SHA256: "9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb" + INCLUDE_PATTERNS: ["*/runtime/Cpp"] + NUM_COMPONENTS_TO_STRIP: 3 + OUTPUT_DIR: "{{.SOURCE_DIR}}" + URL: "https://github.com/antlr/antlr4/archive/refs/tags/{{.VERSION}}.tar.gz" + - task: "yscope-dev-utils:cmake:generate" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + EXTRA_ARGS: + - "-DANTLR4_INSTALL=ON" + - "-DANTLR_BUILD_CPP_TESTS=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + + # Set CMP0135 so that extracted files use the current timestamp as their modification + # timestamp, which ensures the library gets rebuilt if the extracted files change. + - "-DCMAKE_POLICY_DEFAULT_CMP0135=NEW" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + SOURCE_DIR: "{{.SOURCE_DIR}}" + - task: "yscope-dev-utils:cmake:build" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" + - task: "yscope-dev-utils:cmake:install" + vars: + BUILD_DIR: "{{.BUILD_DIR}}" + CMAKE_PACKAGE_NAME: "antlr4-runtime" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + INSTALL_PREFIX: "{{.G_DEPS_CPP_DIR}}/antlr4-runtime-install" + + boost: + internal: true + vars: + VERSION: "1.87.0" + run: "once" + cmds: + - task: "yscope-dev-utils:boost:download-and-install" + vars: + BUILD_AND_INSTALL_ARGS: + - "cxxflags=-fPIC" + - "linkflags=-fPIC" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + FILE_SHA256: "d6c69e4459eb5d6ec208250291221e7ff4a2affde9af6e49c9303b89c687461f" + TARGETS: + - "filesystem" + - "iostreams" + - "process" + - "program_options" + - "regex" + - "url" + URL: "https://github.com/boostorg/boost/releases/download/boost-{{.VERSION}}/\ + boost-{{.VERSION}}-b2-nodocs.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + date: + internal: true + vars: + VERSION: "v3.0.1" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + CMAKE_PACKAGE_NAME: "date" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538" + TAR_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + fmt: + internal: true + vars: + VERSION: "11.2.0" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DFMT_DOC=OFF" + - "-DFMT_TEST=OFF" + CMAKE_PACKAGE_NAME: "{{.G_FMT_LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af" + TAR_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + log-surgeon: + internal: true + run: "once" + deps: + - "fmt" + - "microsoft-gsl" + - "ystdlib" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_GSL_LIB_NAME}}.cmake" + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_YSTDLIB_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-Dlog_surgeon_BUILD_TESTING=OFF" + CMAKE_PACKAGE_NAME: "log_surgeon" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "eebf857962f8fc022bfa5de8d00e10ee4ac36bb8f126a556935a93dde1435aa2" + TAR_URL: "https://github.com/y-scope/log-surgeon/archive/193e1f9.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + microsoft-gsl: + internal: true + vars: + VERSION: "v4.0.0" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DGSL_TEST=OFF" + CMAKE_PACKAGE_NAME: "{{.G_GSL_LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "f0e32cb10654fea91ad56bde89170d78cfbf4363ee0b01d8f097de2ba49f6ce9" + TAR_URL: "https://github.com/microsoft/GSL/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + msgpack-cxx: + internal: true + vars: + VERSION: "7.0.0" + run: "once" + deps: + - task: "boost" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DMSGPACK_BUILD_DOCS=OFF" + - "-DMSGPACK_CXX20=ON" + CMAKE_PACKAGE_NAME: "msgpack-cxx" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TAR_URL: "https://github.com/msgpack/msgpack-c/releases/download/cpp-{{.VERSION}}\ + /msgpack-cxx-{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + nlohmann-json: + internal: true + vars: + VERSION: "v3.11.3" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DJSON_BuildTests=OFF" + CMAKE_PACKAGE_NAME: "nlohmann_json" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406" + + # NOTE: We use the GitHub-generated source tarball for this version rather than the + # release tarball, since the latter is served from githubusercontent.com which is blocked + # by some developers' firewalls. The contents of the former are a superset of the latter. + TAR_URL: "https://github.com/nlohmann/json/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + simdjson: + internal: true + vars: + VERSION: "v4.6.4" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + CMAKE_PACKAGE_NAME: "simdjson" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "b091107844fe928158c5c2265c20360fff312889ddf7ebc4528a0f0f8f2ff9cd" + TAR_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + spdlog: + internal: true + vars: + VERSION: "v1.15.3" + run: "once" + deps: + - "fmt" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DSPDLOG_BUILD_EXAMPLE=OFF" + - "-DSPDLOG_BUILD_EXAMPLE_HO=OFF" + - "-DSPDLOG_FMT_EXTERNAL=ON" + CMAKE_PACKAGE_NAME: "spdlog" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "15a04e69c222eb6c01094b5c7ff8a249b36bb22788d72519646fb85feb267e67" + TAR_URL: "https://github.com/gabime/spdlog/archive/refs/tags/{{.VERSION}}.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + ystdlib: + internal: true + run: "once" + deps: + - task: "boost" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-Dystdlib_BUILD_TESTING=OFF" + CMAKE_PACKAGE_NAME: "{{.G_YSTDLIB_LIB_NAME}}" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + TAR_SHA256: "65990dc2bcc4a355c2181bfe31a7800f492309d1bcd340f52a34e85047e61bc8" + TAR_URL: "https://github.com/y-scope/ystdlib-cpp/archive/9ed78cd.tar.gz" + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" + + zstd: + internal: true + vars: + VERSION: "1.5.7" + run: "once" + cmds: + - task: "yscope-dev-utils:cmake:install-remote-tar" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-DZSTD_BUILD_CONTRIB=OFF" + - "-DZSTD_BUILD_PROGRAMS=OFF" + - "-DZSTD_BUILD_SHARED=OFF" + - "-DZSTD_BUILD_STATIC=ON" + - "-DZSTD_BUILD_TESTS=OFF" + CMAKE_PACKAGE_NAME: "zstd" + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" + CMAKE_SOURCE_DIR: "build/cmake" + TAR_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" + TAR_URL: >- + https://github.com/facebook/zstd/releases/download/v{{.VERSION}}/zstd-{{.VERSION}}.tar.gz + WORK_DIR: "{{.G_DEPS_CPP_DIR}}" diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml new file mode 100644 index 00000000..6afd9cf8 --- /dev/null +++ b/taskfiles/velox-connector/main.yaml @@ -0,0 +1,37 @@ +version: "3" + +includes: + deps: "deps.yaml" + utils: + internal: true + taskfile: "../../tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" + +vars: + G_VELOX_CONNECTOR_BUILD_DIR: "{{.G_BUILD_DIR}}/velox-connector" + G_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" + +tasks: + build: + deps: + - "generate" + cmds: + - task: "utils:cmake:build" + vars: + BUILD_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}" + TARGETS: ["clp-plugin-velox-connector"] + + clean: + cmd: "rm -rf '{{.G_VELOX_CONNECTOR_BUILD_DIR}}'" + + generate: + internal: true + deps: + - "deps:install-all" + cmds: + - task: "utils:cmake:generate" + vars: + BUILD_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}" + EXTRA_ARGS: + - "-DLIBCLP_PLUGIN_VELOX_CONNECTOR_DEPS_CMAKE_SETTINGS=\ + {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake" + SOURCE_DIR: "{{.ROOT_DIR}}/velox-connector" diff --git a/velox-connector/CMakeLists.txt b/velox-connector/CMakeLists.txt index cbf95304..b5675a5c 100644 --- a/velox-connector/CMakeLists.txt +++ b/velox-connector/CMakeLists.txt @@ -18,6 +18,26 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) # causing dlopen to fail with an unresolved symbol error. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") +# ============================================================================== +# Dependencies +# ============================================================================== + +# Use the dependencies installed via `task velox-connector:deps:install-all`. +# +# NOTE: `all-deps.cmake` is auto-generated and sets `_ROOT` cache variables (e.g. +# `fmt_ROOT`) that point to the `task`-installed dependencies. CMake's `find_package()` searches +# `_ROOT` first, so the `task`-installed dependencies will be used instead of any system +# packages. +set( + LIBCLP_PLUGIN_VELOX_CONNECTOR_DEPS_CMAKE_SETTINGS + "${CMAKE_CURRENT_LIST_DIR}/../build/velox-connector/deps/cpp/cmake-settings/all-deps.cmake" + CACHE + STRING + "Path to the CMake settings file describing this library's dependencies." +) + +include("${LIBCLP_PLUGIN_VELOX_CONNECTOR_DEPS_CMAKE_SETTINGS}" REQUIRED) + # ============================================================================== # Presto/Velox framework headers # ============================================================================== @@ -38,8 +58,10 @@ set(PRESTO_ROOT ${presto_native_execution_SOURCE_DIR}/presto-native-execution) # CLP # ============================================================================== -# CLP options — only build what the plugin needs. +# CLP options - only build what the plugin needs. set(CLP_BUILD_CLP_REGEX_UTILS OFF CACHE BOOL "Build CLP regex utils") +set(CLP_BUILD_CLP_S_ARCHIVEWRITER OFF CACHE BOOL "Build CLP-S archive writer") +set(CLP_BUILD_CLP_S_FILTER OFF CACHE BOOL "Build CLP-S filter") set(CLP_BUILD_CLP_S_JSONCONSTRUCTOR OFF CACHE BOOL "Build CLP-S JSON constructor") set(CLP_BUILD_CLP_S_REDUCER_DEPENDENCIES OFF CACHE BOOL "Build CLP-S reducer dependencies") set(CLP_BUILD_CLP_S_SEARCH_SQL OFF CACHE BOOL "Build CLP-S search SQL") diff --git a/velox-connector/README.md b/velox-connector/README.md new file mode 100644 index 00000000..13e11fce --- /dev/null +++ b/velox-connector/README.md @@ -0,0 +1,36 @@ +# Presto CLP Velox connector plugin + +## Requirements + +* An environment with Velox's [dependencies][velox-deps-setup] installed + * For example, Presto's dev container: + [prestodb/presto-native-dependency][presto-native-dependency] + * NOTE: Due a bug in GCC 12 that's used in the container, log-surgeon won't compile. Instead, + you'll need to explicitly set the following environment variables to point at GCC 11: + + ```shell + export CC="gcc" + export CXX="g++" + ``` + +* [Task] >= 3.49.1 + +## Building + +```shell +task velox-connector:build +``` + +Or with CMake directly: + +```shell +task velox-connector:deps:install-all +cmake -S velox-connector -B build/velox-connector +cmake --build build/velox-connector -j +``` + +The built plugin will be at `build/velox-connector/libclp-plugin-velox-connector.so`. + +[presto-native-dependency]: https://hub.docker.com/r/prestodb/presto-native-dependency +[Task]: https://taskfile.dev +[velox-deps-setup]: https://github.com/facebookincubator/velox#setting-up-dependencies