diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt index 30ba84dc5461d..09306cbc6202f 100644 --- a/presto-native-execution/presto_cpp/main/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt @@ -47,6 +47,7 @@ target_link_libraries( $ presto_common presto_exception + presto_function_metadata presto_http presto_operators velox_aggregates diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index 91e3a39ae13b4..9672058718344 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -35,6 +35,7 @@ #include "presto_cpp/main/operators/PartitionAndSerialize.h" #include "presto_cpp/main/operators/ShuffleRead.h" #include "presto_cpp/main/operators/UnsafeRowExchangeSource.h" +#include "presto_cpp/main/types/FunctionMetadata.h" #include "presto_cpp/main/types/PrestoToVeloxQueryPlan.h" #include "velox/common/base/Counters.h" #include "velox/common/base/StatsReporter.h" @@ -469,16 +470,7 @@ void PrestoServer::run() { driverExecutor_.get(), httpSrvCpuExecutor_.get(), spillerExecutor_.get()); if (systemConfig->prestoNativeSidecar()) { - httpServer_->registerGet( - "/v1/properties/session", - [this]( - proxygen::HTTPMessage* /*message*/, - const std::vector>& /*body*/, - proxygen::ResponseHandler* downstream) { - auto sessionProperties = - taskManager_->getQueryContextManager()->getSessionProperties(); - http::sendOkResponse(downstream, sessionProperties.serialize()); - }); + registerSidecarEndpoints(); } std::string taskUri; @@ -1404,6 +1396,27 @@ void PrestoServer::reportNodeStatus(proxygen::ResponseHandler* downstream) { http::sendOkResponse(downstream, json(fetchNodeStatus())); } +void PrestoServer::registerSidecarEndpoints() { + VELOX_CHECK(httpServer_); + httpServer_->registerGet( + "/v1/properties/session", + [this]( + proxygen::HTTPMessage* /*message*/, + const std::vector>& /*body*/, + proxygen::ResponseHandler* downstream) { + auto sessionProperties = + taskManager_->getQueryContextManager()->getSessionProperties(); + http::sendOkResponse(downstream, sessionProperties.serialize()); + }); + httpServer_->registerGet( + "/v1/functions", + [](proxygen::HTTPMessage* /*message*/, + const std::vector>& /*body*/, + proxygen::ResponseHandler* downstream) { + http::sendOkResponse(downstream, getFunctionsMetadata()); + }); +} + protocol::NodeStatus PrestoServer::fetchNodeStatus() { auto systemConfig = SystemConfig::instance(); const int64_t nodeMemoryGb = systemConfig->systemMemoryGb(); diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index bee2d8d43391a..0ab6954dd29b6 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -216,6 +216,8 @@ class PrestoServer { void registerSystemConnector(); + void registerSidecarEndpoints(); + std::unique_ptr setupSsdCache(); const std::string configDirectoryPath_; diff --git a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt index ecdd1bb30b7ff..e37c99c432521 100644 --- a/presto-native-execution/presto_cpp/main/types/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/CMakeLists.txt @@ -26,6 +26,10 @@ target_link_libraries(presto_types presto_type_converter velox_type_fbhive set_property(TARGET presto_types PROPERTY JOB_POOL_LINK presto_link_job_pool) +add_library(presto_function_metadata OBJECT FunctionMetadata.cpp) + +target_link_libraries(presto_function_metadata velox_function_registry) + if(PRESTO_ENABLE_TESTING) add_subdirectory(tests) endif() diff --git a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp new file mode 100644 index 0000000000000..3f5178a83da9d --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp @@ -0,0 +1,289 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "presto_cpp/main/types/FunctionMetadata.h" +#include "presto_cpp/presto_protocol/presto_protocol.h" +#include "velox/exec/Aggregate.h" +#include "velox/exec/AggregateFunctionRegistry.h" +#include "velox/exec/WindowFunction.h" +#include "velox/expression/SimpleFunctionRegistry.h" +#include "velox/functions/FunctionRegistry.h" + +using namespace facebook::velox; +using namespace facebook::velox::exec; + +namespace facebook::presto { + +namespace { + +// Check if the Velox type is supported in Presto. +bool isValidPrestoType(const TypeSignature& typeSignature) { + if (typeSignature.parameters().empty()) { + // Hugeint type is not supported in Presto. + auto kindName = boost::algorithm::to_upper_copy(typeSignature.baseName()); + if (auto typeKind = tryMapNameToTypeKind(kindName)) { + return typeKind.value() != TypeKind::HUGEINT; + } + } else { + for (const auto& paramType : typeSignature.parameters()) { + if (!isValidPrestoType(paramType)) { + return false; + } + } + } + return true; +} + +// The keys in velox function maps are of the format +// `catalog.schema.function_name`. This utility function extracts the +// three parts, {catalog, schema, function_name}, from the registered function. +const std::vector getFunctionNameParts( + const std::string& registeredFunction) { + std::vector parts; + folly::split('.', registeredFunction, parts, true); + VELOX_USER_CHECK( + parts.size() == 3, + fmt::format("Prefix missing for function {}", registeredFunction)); + return parts; +} + +// TODO: Remove this function later and retrieve companion function information +// from velox. Approaches for this under discussion here: +// https://github.com/facebookincubator/velox/discussions/11011. +// A function name is a companion function's if the name is an existing +// aggregation function name followed by specific suffixes. +bool isCompanionFunctionName( + const std::string& name, + const std::unordered_map& + aggregateFunctions) { + auto suffixOffset = name.rfind("_partial"); + if (suffixOffset == std::string::npos) { + suffixOffset = name.rfind("_merge_extract"); + } + if (suffixOffset == std::string::npos) { + suffixOffset = name.rfind("_merge"); + } + if (suffixOffset == std::string::npos) { + suffixOffset = name.rfind("_extract"); + } + if (suffixOffset == std::string::npos) { + return false; + } + return aggregateFunctions.count(name.substr(0, suffixOffset)) > 0; +} + +const protocol::AggregationFunctionMetadata getAggregationFunctionMetadata( + const std::string& name, + const AggregateFunctionSignature& signature) { + protocol::AggregationFunctionMetadata metadata; + metadata.intermediateType = signature.intermediateType().toString(); + metadata.isOrderSensitive = + getAggregateFunctionEntry(name)->metadata.orderSensitive; + return metadata; +} + +const exec::VectorFunctionMetadata getScalarMetadata(const std::string& name) { + auto simpleFunctionMetadata = + exec::simpleFunctions().getFunctionSignaturesAndMetadata(name); + if (simpleFunctionMetadata.size()) { + // Functions like abs are registered as simple functions for primitive + // types, and as a vector function for complex types like DECIMAL. So do not + // throw an error if function metadata is not found in simple function + // signature map. + return simpleFunctionMetadata.back().first; + } + + auto vectorFunctionMetadata = exec::getVectorFunctionMetadata(name); + if (vectorFunctionMetadata.has_value()) { + return vectorFunctionMetadata.value(); + } + VELOX_UNREACHABLE("Metadata for function {} not found", name); +} + +const protocol::RoutineCharacteristics getRoutineCharacteristics( + const std::string& name, + const protocol::FunctionKind& kind) { + protocol::Determinism determinism; + protocol::NullCallClause nullCallClause; + if (kind == protocol::FunctionKind::SCALAR) { + auto metadata = getScalarMetadata(name); + determinism = metadata.deterministic + ? protocol::Determinism::DETERMINISTIC + : protocol::Determinism::NOT_DETERMINISTIC; + nullCallClause = metadata.defaultNullBehavior + ? protocol::NullCallClause::RETURNS_NULL_ON_NULL_INPUT + : protocol::NullCallClause::CALLED_ON_NULL_INPUT; + } else { + // Default metadata values of DETERMINISTIC and CALLED_ON_NULL_INPUT for + // non-scalar functions. + determinism = protocol::Determinism::DETERMINISTIC; + nullCallClause = protocol::NullCallClause::CALLED_ON_NULL_INPUT; + } + + protocol::RoutineCharacteristics routineCharacteristics; + routineCharacteristics.language = + std::make_shared(protocol::Language({"CPP"})); + routineCharacteristics.determinism = + std::make_shared(determinism); + routineCharacteristics.nullCallClause = + std::make_shared(nullCallClause); + return routineCharacteristics; +} + +std::optional buildFunctionMetadata( + const std::string& name, + const std::string& schema, + const protocol::FunctionKind& kind, + const FunctionSignature& signature, + const AggregateFunctionSignaturePtr& aggregateSignature = nullptr) { + protocol::JsonBasedUdfFunctionMetadata metadata; + metadata.docString = name; + metadata.functionKind = kind; + if (!isValidPrestoType(signature.returnType())) { + return std::nullopt; + } + metadata.outputType = signature.returnType().toString(); + + const auto& argumentTypes = signature.argumentTypes(); + std::vector paramTypes(argumentTypes.size()); + for (auto i = 0; i < argumentTypes.size(); i++) { + if (!isValidPrestoType(argumentTypes.at(i))) { + return std::nullopt; + } + paramTypes[i] = argumentTypes.at(i).toString(); + } + metadata.paramTypes = paramTypes; + metadata.schema = schema; + metadata.routineCharacteristics = getRoutineCharacteristics(name, kind); + + if (aggregateSignature) { + metadata.aggregateMetadata = + std::make_shared( + getAggregationFunctionMetadata(name, *aggregateSignature)); + } + return metadata; +} + +json buildScalarMetadata( + const std::string& name, + const std::string& schema, + const std::vector& signatures) { + json j = json::array(); + json tj; + for (const auto& signature : signatures) { + if (auto functionMetadata = buildFunctionMetadata( + name, schema, protocol::FunctionKind::SCALAR, *signature)) { + protocol::to_json(tj, functionMetadata.value()); + j.push_back(tj); + } + } + return j; +} + +json buildAggregateMetadata( + const std::string& name, + const std::string& schema, + const std::vector& signatures) { + // All aggregate functions can be used as window functions. + VELOX_USER_CHECK( + getWindowFunctionSignatures(name).has_value(), + "Aggregate function {} not registered as a window function", + name); + const std::vector kinds = { + protocol::FunctionKind::AGGREGATE, protocol::FunctionKind::WINDOW}; + json j = json::array(); + json tj; + for (const auto& kind : kinds) { + for (const auto& signature : signatures) { + if (auto functionMetadata = buildFunctionMetadata( + name, schema, kind, *signature, signature)) { + protocol::to_json(tj, functionMetadata.value()); + j.push_back(tj); + } + } + } + return j; +} + +json buildWindowMetadata( + const std::string& name, + const std::string& schema, + const std::vector& signatures) { + json j = json::array(); + json tj; + for (const auto& signature : signatures) { + if (auto functionMetadata = buildFunctionMetadata( + name, schema, protocol::FunctionKind::WINDOW, *signature)) { + protocol::to_json(tj, functionMetadata.value()); + j.push_back(tj); + } + } + return j; +} + +} // namespace + +json getFunctionsMetadata() { + json j; + + // Get metadata for all registered scalar functions in velox. + const auto signatures = getFunctionSignatures(); + static const std::unordered_set kBlockList = { + "row_constructor", "in", "is_null"}; + // Exclude aggregate companion functions (extract aggregate companion + // functions are registered as vector functions). + const auto aggregateFunctions = exec::aggregateFunctions().copy(); + for (const auto& entry : signatures) { + const auto name = entry.first; + // Skip internal functions. They don't have any prefix. + if (kBlockList.count(name) != 0 || + name.find("$internal$") != std::string::npos || + isCompanionFunctionName(name, aggregateFunctions)) { + continue; + } + + const auto parts = getFunctionNameParts(name); + const auto schema = parts[1]; + const auto function = parts[2]; + j[function] = buildScalarMetadata(name, schema, entry.second); + } + + // Get metadata for all registered aggregate functions in velox. + for (const auto& entry : aggregateFunctions) { + if (!isCompanionFunctionName(entry.first, aggregateFunctions)) { + const auto name = entry.first; + const auto parts = getFunctionNameParts(name); + const auto schema = parts[1]; + const auto function = parts[2]; + j[function] = + buildAggregateMetadata(name, schema, entry.second.signatures); + } + } + + // Get metadata for all registered window functions in velox. Skip aggregates + // as they have been processed. + const auto& functions = exec::windowFunctions(); + for (const auto& entry : functions) { + if (aggregateFunctions.count(entry.first) == 0) { + const auto name = entry.first; + const auto parts = getFunctionNameParts(entry.first); + const auto schema = parts[1]; + const auto function = parts[2]; + j[function] = buildWindowMetadata(name, schema, entry.second.signatures); + } + } + + return j; +} + +} // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.h b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.h new file mode 100644 index 0000000000000..7a39aebb95037 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.h @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "presto_cpp/external/json/nlohmann/json.hpp" + +namespace facebook::presto { + +// Returns metadata for all registered functions as json. +nlohmann::json getFunctionsMetadata(); + +} // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt index 4cd79ed1bf5bb..d8bb39a9cdf97 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/types/tests/CMakeLists.txt @@ -10,6 +10,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +add_library(presto_type_test_utils OBJECT TestUtils.cpp) + add_executable(presto_velox_split_test PrestoToVeloxSplitTest.cpp) add_test(presto_velox_split_test presto_velox_split_test) @@ -48,6 +50,7 @@ target_link_libraries( $ $ presto_operators + presto_type_test_utils velox_core velox_dwio_common_exception velox_encode @@ -89,3 +92,21 @@ target_link_libraries( velox_tpch_connector GTest::gtest GTest::gtest_main) + +add_executable(presto_function_metadata_test FunctionMetadataTest.cpp) + +add_test( + NAME presto_function_metadata_test + COMMAND presto_function_metadata_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries( + presto_function_metadata_test + presto_function_metadata + presto_protocol + presto_type_test_utils + velox_aggregates + velox_functions_prestosql + velox_window + gtest + gtest_main) diff --git a/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp new file mode 100644 index 0000000000000..fabdc1b151bfa --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "presto_cpp/main/common/tests/test_json.h" +#include "presto_cpp/main/types/FunctionMetadata.h" +#include "presto_cpp/main/types/tests/TestUtils.h" +#include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" +#include "velox/functions/prestosql/registration/RegistrationFunctions.h" +#include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" + +using namespace facebook::velox; +using namespace facebook::presto; + +using json = nlohmann::json; + +static const std::string kPrestoDefaultPrefix = "presto.default."; +static const std::string kDefaultSchema = "default"; + +class FunctionMetadataTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + aggregate::prestosql::registerAllAggregateFunctions(kPrestoDefaultPrefix); + window::prestosql::registerAllWindowFunctions(kPrestoDefaultPrefix); + functions::prestosql::registerAllScalarFunctions(kPrestoDefaultPrefix); + } + + void SetUp() override { + functionMetadata_ = getFunctionsMetadata(); + } + + void testFunction( + const std::string& name, + const std::string& expectedFile, + size_t expectedSize) { + json metadataList = functionMetadata_.at(name); + EXPECT_EQ(metadataList.size(), expectedSize); + std::string expectedStr = slurp(test::utils::getDataPath(expectedFile)); + auto expected = json::parse(expectedStr); + + json::array_t expectedList = expected[name]; + std::sort(expectedList.begin(), expectedList.end()); + std::sort(metadataList.begin(), metadataList.end()); + for (auto i = 0; i < expectedSize; i++) { + EXPECT_EQ(expectedList[i], metadataList[i]); + } + } + + json functionMetadata_; +}; + +TEST_F(FunctionMetadataTest, approxMostFrequent) { + testFunction("approx_most_frequent", "ApproxMostFrequent.json", 12); +} + +TEST_F(FunctionMetadataTest, arrayFrequency) { + testFunction("array_frequency", "ArrayFrequency.json", 10); +} + +TEST_F(FunctionMetadataTest, combinations) { + testFunction("combinations", "Combinations.json", 10); +} + +TEST_F(FunctionMetadataTest, covarSamp) { + testFunction("covar_samp", "CovarSamp.json", 4); +} + +TEST_F(FunctionMetadataTest, elementAt) { + testFunction("element_at", "ElementAt.json", 3); +} + +TEST_F(FunctionMetadataTest, lead) { + testFunction("lead", "Lead.json", 3); +} + +TEST_F(FunctionMetadataTest, ntile) { + testFunction("ntile", "Ntile.json", 1); +} + +TEST_F(FunctionMetadataTest, setAgg) { + testFunction("set_agg", "SetAgg.json", 2); +} + +TEST_F(FunctionMetadataTest, stddevSamp) { + testFunction("stddev_samp", "StddevSamp.json", 10); +} + +TEST_F(FunctionMetadataTest, transformKeys) { + testFunction("transform_keys", "TransformKeys.json", 1); +} + +TEST_F(FunctionMetadataTest, variance) { + testFunction("variance", "Variance.json", 10); +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp index a29db9f97b17c..4779ea5ada394 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/PlanConverterTest.cpp @@ -11,9 +11,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include -#include #include #include "presto_cpp/main/common/tests/test_json.h" @@ -23,43 +20,19 @@ #include "presto_cpp/main/operators/ShuffleWrite.h" #include "presto_cpp/main/types/PrestoToVeloxConnector.h" #include "presto_cpp/main/types/PrestoToVeloxQueryPlan.h" +#include "presto_cpp/main/types/tests/TestUtils.h" #include "velox/connectors/hive/TableHandle.h" #include "velox/exec/tests/utils/TempDirectoryPath.h" -namespace fs = boost::filesystem; - using namespace facebook::presto; using namespace facebook::velox; namespace { -std::string getDataPath(const std::string& fileName) { - std::string currentPath = fs::current_path().c_str(); - - if (boost::algorithm::ends_with(currentPath, "fbcode")) { - return currentPath + - "/github/presto-trunk/presto-native-execution/presto_cpp/main/types/tests/data/" + - fileName; - } - - if (boost::algorithm::ends_with(currentPath, "fbsource")) { - return currentPath + "/third-party/presto_cpp/main/types/tests/data/" + - fileName; - } - - // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ - // directory. Hard-coded json files are not copied there and test fails with - // file not found. Fixing the path so that we can trigger these tests from - // CLion. - boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); - boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); - - return currentPath + "/data/" + fileName; -} core::PlanFragment assertToVeloxFragment( const std::string& fileName, memory::MemoryPool* pool = nullptr) { - std::string fragment = slurp(getDataPath(fileName)); + std::string fragment = slurp(test::utils::getDataPath(fileName)); protocol::PlanFragment prestoPlan = json::parse(fragment); std::shared_ptr poolPtr; @@ -85,7 +58,7 @@ std::shared_ptr assertToBatchVeloxQueryPlan( const std::string& shuffleName, std::shared_ptr&& serializedShuffleWriteInfo, std::shared_ptr&& broadcastBasePath) { - const std::string fragment = slurp(getDataPath(fileName)); + const std::string fragment = slurp(test::utils::getDataPath(fileName)); protocol::PlanFragment prestoPlan = json::parse(fragment); auto pool = memory::deprecatedAddDefaultLeafMemoryPool(); diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp new file mode 100644 index 0000000000000..7d4ee1e956b25 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.cpp @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +namespace facebook::presto::test::utils { + +const std::string getDataPath(const std::string& fileName) { + std::string currentPath = boost::filesystem::current_path().c_str(); + if (boost::algorithm::ends_with(currentPath, "fbcode")) { + return currentPath + + "/github/presto-trunk/presto-native-execution/presto_cpp/main/types/tests/data/" + + fileName; + } + if (boost::algorithm::ends_with(currentPath, "fbsource")) { + return currentPath + "/third-party/presto_cpp/main/types/tests/data/" + + fileName; + } + + // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ + // directory. Hard-coded json files are not copied there and test fails with + // file not found. Fixing the path so that we can trigger these tests from + // CLion. + boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); + boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); + + return currentPath + "/data/" + fileName; +} +} // namespace facebook::presto::test::utils diff --git a/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h new file mode 100644 index 0000000000000..af06a19f4b296 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/TestUtils.h @@ -0,0 +1,16 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +namespace facebook::presto::test::utils { +const std::string getDataPath(const std::string& fileName); +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/ValuesPipeTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/ValuesPipeTest.cpp index 89b6766ecc8f7..e58f26033edcd 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/ValuesPipeTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/ValuesPipeTest.cpp @@ -12,50 +12,22 @@ * limitations under the License. */ #include - -#include -#include #include #include "presto_cpp/main/common/tests/test_json.h" #include "presto_cpp/main/types/PrestoToVeloxQueryPlan.h" +#include "presto_cpp/main/types/tests/TestUtils.h" #include "velox/exec/Operator.h" #include "velox/type/Type.h" #include "velox/vector/FlatVector.h" -namespace fs = boost::filesystem; - using namespace facebook::presto; using namespace facebook::velox; -namespace { -std::string getDataPath(const std::string& fileName) { - std::string currentPath = fs::current_path().c_str(); - if (boost::algorithm::ends_with(currentPath, "fbcode")) { - return currentPath + - "/github/presto-trunk/presto-native-execution/presto_cpp/main/types/tests/data/" + - fileName; - } - if (boost::algorithm::ends_with(currentPath, "fbsource")) { - return currentPath + "/third-party/presto_cpp/main/types/tests/data/" + - fileName; - } - - // CLion runs the tests from cmake-build-release/ or cmake-build-debug/ - // directory. Hard-coded json files are not copied there and test fails with - // file not found. Fixing the path so that we can trigger these tests from - // CLion. - boost::algorithm::replace_all(currentPath, "cmake-build-release/", ""); - boost::algorithm::replace_all(currentPath, "cmake-build-debug/", ""); - - return currentPath + "/data/" + fileName; -} -} // namespace - class TestValues : public ::testing::Test {}; TEST_F(TestValues, valuesRowVector) { - std::string str = slurp(getDataPath("ValuesNode.json")); + std::string str = slurp(test::utils::getDataPath("ValuesNode.json")); json j = json::parse(str); std::shared_ptr p = j; @@ -95,7 +67,7 @@ TEST_F(TestValues, valuesPlan) { // select a, b from (VALUES (1, 'a'), (2, 'b'), (3, 'c')) as t (a, b) where a // = 1; // - std::string str = slurp(getDataPath("ValuesPipeTest.json")); + std::string str = slurp(test::utils::getDataPath("ValuesPipeTest.json")); json j = json::parse(str); std::shared_ptr p = j; diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json b/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json new file mode 100644 index 0000000000000..4d1f793bda5a6 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json @@ -0,0 +1,244 @@ +{ + "approx_most_frequent": [ + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(boolean),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(boolean,bigint)", + "paramTypes": [ + "bigint", + "boolean", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(tinyint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(tinyint,bigint)", + "paramTypes": [ + "bigint", + "tinyint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(smallint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(smallint,bigint)", + "paramTypes": [ + "bigint", + "smallint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(integer),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(integer,bigint)", + "paramTypes": [ + "bigint", + "integer", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(bigint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(bigint,bigint)", + "paramTypes": [ + "bigint", + "bigint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(varchar),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "AGGREGATE", + "outputType": "map(varchar,bigint)", + "paramTypes": [ + "bigint", + "varchar", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(boolean),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(boolean,bigint)", + "paramTypes": [ + "bigint", + "boolean", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(tinyint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(tinyint,bigint)", + "paramTypes": [ + "bigint", + "tinyint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(smallint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(smallint,bigint)", + "paramTypes": [ + "bigint", + "smallint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(integer),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(integer,bigint)", + "paramTypes": [ + "bigint", + "integer", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(bigint),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(bigint,bigint)", + "paramTypes": [ + "bigint", + "bigint", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,bigint,array(varchar),array(bigint))", + "isOrderSensitive": true + }, + "docString": "presto.default.approx_most_frequent", + "functionKind": "WINDOW", + "outputType": "map(varchar,bigint)", + "paramTypes": [ + "bigint", + "varchar", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json b/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json new file mode 100644 index 0000000000000..485028cca4d6d --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json @@ -0,0 +1,144 @@ +{ + "array_frequency": [ + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(double,integer)", + "paramTypes": [ + "array(double)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(real,integer)", + "paramTypes": [ + "array(real)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(integer,integer)", + "paramTypes": [ + "array(integer)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(varchar,integer)", + "paramTypes": [ + "array(varchar)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(bigint,integer)", + "paramTypes": [ + "array(bigint)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(smallint,integer)", + "paramTypes": [ + "array(smallint)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(timestamp,integer)", + "paramTypes": [ + "array(timestamp)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(tinyint,integer)", + "paramTypes": [ + "array(tinyint)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(date,integer)", + "paramTypes": [ + "array(date)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.array_frequency", + "functionKind": "SCALAR", + "outputType": "map(boolean,integer)", + "paramTypes": [ + "array(boolean)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json b/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json new file mode 100644 index 0000000000000..8b2f860cba5da --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json @@ -0,0 +1,154 @@ +{ + "combinations": [ + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(date))", + "paramTypes": [ + "array(date)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(timestamp))", + "paramTypes": [ + "array(timestamp)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(varchar))", + "paramTypes": [ + "array(varchar)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(boolean))", + "paramTypes": [ + "array(boolean)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(double))", + "paramTypes": [ + "array(double)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(real))", + "paramTypes": [ + "array(real)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(tinyint))", + "paramTypes": [ + "array(tinyint)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(bigint))", + "paramTypes": [ + "array(bigint)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(integer))", + "paramTypes": [ + "array(integer)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.combinations", + "functionKind": "SCALAR", + "outputType": "array(array(smallint))", + "paramTypes": [ + "array(smallint)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json b/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json new file mode 100644 index 0000000000000..7b1d2d8e5adb0 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json @@ -0,0 +1,80 @@ +{ + "covar_samp": [ + { + "aggregateMetadata": { + "intermediateType": "row(double,bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.covar_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "double", + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(double,bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.covar_samp", + "functionKind": "AGGREGATE", + "outputType": "real", + "paramTypes": [ + "real", + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(double,bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.covar_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "double", + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(double,bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.covar_samp", + "functionKind": "WINDOW", + "outputType": "real", + "paramTypes": [ + "real", + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json b/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json new file mode 100644 index 0000000000000..ccd6405762abf --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json @@ -0,0 +1,49 @@ +{ + "element_at": [ + { + "docString": "presto.default.element_at", + "functionKind": "SCALAR", + "outputType": "T", + "paramTypes": [ + "array(T)", + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.element_at", + "functionKind": "SCALAR", + "outputType": "T", + "paramTypes": [ + "array(T)", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.element_at", + "functionKind": "SCALAR", + "outputType": "V", + "paramTypes": [ + "map(K,V)", + "K" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json b/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json new file mode 100644 index 0000000000000..6f06a1e8c6965 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json @@ -0,0 +1,49 @@ +{ + "lead": [ + { + "docString": "presto.default.lead", + "functionKind": "WINDOW", + "outputType": "T", + "paramTypes": [ + "T" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.lead", + "functionKind": "WINDOW", + "outputType": "T", + "paramTypes": [ + "T", + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "docString": "presto.default.lead", + "functionKind": "WINDOW", + "outputType": "T", + "paramTypes": [ + "T", + "bigint", + "T" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json b/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json new file mode 100644 index 0000000000000..5685eccc25900 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json @@ -0,0 +1,18 @@ +{ + "ntile": [ + { + "docString": "presto.default.ntile", + "functionKind": "WINDOW", + "outputType": "bigint", + "paramTypes": [ + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json b/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json new file mode 100644 index 0000000000000..2fd34d4800426 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json @@ -0,0 +1,40 @@ +{ + "set_agg": [ + { + "aggregateMetadata": { + "intermediateType": "array(T)", + "isOrderSensitive": true + }, + "docString": "presto.default.set_agg", + "functionKind": "AGGREGATE", + "outputType": "array(T)", + "paramTypes": [ + "T" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "array(T)", + "isOrderSensitive": true + }, + "docString": "presto.default.set_agg", + "functionKind": "WINDOW", + "outputType": "array(T)", + "paramTypes": [ + "T" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json b/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json new file mode 100644 index 0000000000000..857d314a7d3a6 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json @@ -0,0 +1,184 @@ +{ + "stddev_samp": [ + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "smallint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "smallint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.stddev_samp", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json b/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json new file mode 100644 index 0000000000000..2edcbdc98ee0e --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json @@ -0,0 +1,19 @@ +{ + "transform_keys": [ + { + "docString": "presto.default.transform_keys", + "functionKind": "SCALAR", + "outputType": "map(K2,V)", + "paramTypes": [ + "map(K1,V)", + "function(K1,V,K2)" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json b/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json new file mode 100644 index 0000000000000..ce491c58464bf --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json @@ -0,0 +1,184 @@ +{ + "variance": [ + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "smallint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "AGGREGATE", + "outputType": "double", + "paramTypes": [ + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "smallint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "integer" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "bigint" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "real" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + }, + { + "aggregateMetadata": { + "intermediateType": "row(bigint,double,double)", + "isOrderSensitive": true + }, + "docString": "presto.default.variance", + "functionKind": "WINDOW", + "outputType": "double", + "paramTypes": [ + "double" + ], + "routineCharacteristics": { + "determinism": "DETERMINISTIC", + "language": "CPP", + "nullCallClause": "CALLED_ON_NULL_INPUT" + }, + "schema": "default" + } + ] +} diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp index 02f6dcd925e98..9e1f0f5db0f9c 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.cpp @@ -442,6 +442,43 @@ void from_json(const json& j, Aggregation& p) { } } // namespace facebook::presto::protocol namespace facebook::presto::protocol { + +void to_json(json& j, const AggregationFunctionMetadata& p) { + j = json::object(); + to_json_key( + j, + "intermediateType", + p.intermediateType, + "AggregationFunctionMetadata", + "TypeSignature", + "intermediateType"); + to_json_key( + j, + "isOrderSensitive", + p.isOrderSensitive, + "AggregationFunctionMetadata", + "bool", + "isOrderSensitive"); +} + +void from_json(const json& j, AggregationFunctionMetadata& p) { + from_json_key( + j, + "intermediateType", + p.intermediateType, + "AggregationFunctionMetadata", + "TypeSignature", + "intermediateType"); + from_json_key( + j, + "isOrderSensitive", + p.isOrderSensitive, + "AggregationFunctionMetadata", + "bool", + "isOrderSensitive"); +} +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { // Loosly copied this here from NLOHMANN_JSON_SERIALIZE_ENUM() // NOLINTNEXTLINE: cppcoreguidelines-avoid-c-arrays @@ -1458,16 +1495,32 @@ void from_json(const json& j, Determinism& e) { ->first; } } // namespace facebook::presto::protocol +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + namespace facebook::presto::protocol { void to_json(json& j, const Language& p) { - j = json::object(); - to_json_key(j, "language", p.language, "Language", "String", "language"); + json tj = json::object(); + to_json_key(tj, "language", p.language, "Language", "String", "language"); + j = tj["language"]; } void from_json(const json& j, Language& p) { from_json_key(j, "language", p.language, "Language", "String", "language"); } + } // namespace facebook::presto::protocol namespace facebook::presto::protocol { // Loosly copied this here from NLOHMANN_JSON_SERIALIZE_ENUM() @@ -9272,6 +9325,113 @@ void from_json(const json& j, JoinNodeStatsEstimate& p) { "joinProbeKeyCount"); } } // namespace facebook::presto::protocol +namespace facebook::presto::protocol { + +void to_json(json& j, const JsonBasedUdfFunctionMetadata& p) { + j = json::object(); + to_json_key( + j, + "docString", + p.docString, + "JsonBasedUdfFunctionMetadata", + "String", + "docString"); + to_json_key( + j, + "functionKind", + p.functionKind, + "JsonBasedUdfFunctionMetadata", + "FunctionKind", + "functionKind"); + to_json_key( + j, + "outputType", + p.outputType, + "JsonBasedUdfFunctionMetadata", + "TypeSignature", + "outputType"); + to_json_key( + j, + "paramTypes", + p.paramTypes, + "JsonBasedUdfFunctionMetadata", + "List", + "paramTypes"); + to_json_key( + j, + "schema", + p.schema, + "JsonBasedUdfFunctionMetadata", + "String", + "schema"); + to_json_key( + j, + "routineCharacteristics", + p.routineCharacteristics, + "JsonBasedUdfFunctionMetadata", + "RoutineCharacteristics", + "routineCharacteristics"); + to_json_key( + j, + "aggregateMetadata", + p.aggregateMetadata, + "JsonBasedUdfFunctionMetadata", + "AggregationFunctionMetadata", + "aggregateMetadata"); +} + +void from_json(const json& j, JsonBasedUdfFunctionMetadata& p) { + from_json_key( + j, + "docString", + p.docString, + "JsonBasedUdfFunctionMetadata", + "String", + "docString"); + from_json_key( + j, + "functionKind", + p.functionKind, + "JsonBasedUdfFunctionMetadata", + "FunctionKind", + "functionKind"); + from_json_key( + j, + "outputType", + p.outputType, + "JsonBasedUdfFunctionMetadata", + "TypeSignature", + "outputType"); + from_json_key( + j, + "paramTypes", + p.paramTypes, + "JsonBasedUdfFunctionMetadata", + "List", + "paramTypes"); + from_json_key( + j, + "schema", + p.schema, + "JsonBasedUdfFunctionMetadata", + "String", + "schema"); + from_json_key( + j, + "routineCharacteristics", + p.routineCharacteristics, + "JsonBasedUdfFunctionMetadata", + "RoutineCharacteristics", + "routineCharacteristics"); + from_json_key( + j, + "aggregateMetadata", + p.aggregateMetadata, + "JsonBasedUdfFunctionMetadata", + "AggregationFunctionMetadata", + "aggregateMetadata"); +} +} // namespace facebook::presto::protocol /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h index 04fe38e7c3b05..0080b31da78fd 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.h @@ -439,6 +439,14 @@ void to_json(json& j, const Aggregation& p); void from_json(const json& j, Aggregation& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +struct AggregationFunctionMetadata { + TypeSignature intermediateType = {}; + bool isOrderSensitive = {}; +}; +void to_json(json& j, const AggregationFunctionMetadata& p); +void from_json(const json& j, AggregationFunctionMetadata& p); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { enum class AggregationNodeStep { PARTIAL, FINAL, INTERMEDIATE, SINGLE }; extern void to_json(json& j, const AggregationNodeStep& e); extern void from_json(const json& j, AggregationNodeStep& e); @@ -2328,6 +2336,19 @@ void to_json(json& j, const JoinNodeStatsEstimate& p); void from_json(const json& j, JoinNodeStatsEstimate& p); } // namespace facebook::presto::protocol namespace facebook::presto::protocol { +struct JsonBasedUdfFunctionMetadata { + String docString = {}; + FunctionKind functionKind = {}; + TypeSignature outputType = {}; + List paramTypes = {}; + String schema = {}; + RoutineCharacteristics routineCharacteristics = {}; + std::shared_ptr aggregateMetadata = {}; +}; +void to_json(json& j, const JsonBasedUdfFunctionMetadata& p); +void from_json(const json& j, JsonBasedUdfFunctionMetadata& p); +} // namespace facebook::presto::protocol +namespace facebook::presto::protocol { struct LambdaDefinitionExpression : public RowExpression { List argumentTypes = {}; List arguments = {}; diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml index ca4e30ae411e3..aca25d8926289 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml @@ -362,4 +362,6 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableHandle.java - presto-main/src/main/java/com/facebook/presto/connector/system/SystemColumnHandle.java - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java - - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java \ No newline at end of file + - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java + - presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java + - presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java diff --git a/presto-native-execution/presto_cpp/presto_protocol/special/Language.cpp.inc b/presto-native-execution/presto_cpp/presto_protocol/special/Language.cpp.inc new file mode 100644 index 0000000000000..da1c8172869c6 --- /dev/null +++ b/presto-native-execution/presto_cpp/presto_protocol/special/Language.cpp.inc @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace facebook::presto::protocol { + +void to_json(json& j, const Language& p) { + json tj = json::object(); + to_json_key(tj, "language", p.language, "Language", "String", "language"); + j = tj["language"]; +} + +void from_json(const json& j, Language& p) { + from_json_key(j, "language", p.language, "Language", "String", "language"); +} + +} // namespace facebook::presto::protocol