Skip to content

Add TypeSignature Parser using Flex and Bison #7590

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
majetideepak wants to merge 4 commits into from
Closed

Add TypeSignature Parser using Flex and Bison #7590

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d6169e0
Add TypeSignature Parser using Flex and Bison
majetideepak Nov 21, 2023
cf4e355
address review comments
majetideepak Nov 29, 2023
b822a24
fix new line
majetideepak Dec 4, 2023
376b1c0
address review comments
majetideepak Dec 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion velox/exec/tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ add_executable(
GroupedExecutionTest.cpp
Main.cpp
OperatorUtilsTest.cpp
ParseTypeSignatureTest.cpp
PlanBuilderTest.cpp
QueryAssertionsTest.cpp
TaskTest.cpp
Expand Down
6 changes: 3 additions & 3 deletions velox/exec/tests/FunctionSignatureBuilderTest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ TEST_F(FunctionSignatureBuilderTest, typeParamTests) {
.returnType("integer")
.argumentType("Any(T)")
.build(),
"Type 'Any' cannot have parameters");
"Failed to parse type signature [Any(T)]: syntax error, unexpected LPAREN, expecting YYEOF");

// Variable Arity in argument fails.
VELOX_ASSERT_THROW(
Expand All @@ -124,7 +124,7 @@ TEST_F(FunctionSignatureBuilderTest, typeParamTests) {
.returnType("integer")
.argumentType("row(..., varchar)")
.build(),
"Type doesn't exist: '...'");
"Failed to parse type signature [row(..., varchar)]: syntax error, unexpected COMMA");

// Type params cant have type params.
VELOX_ASSERT_THROW(
Expand All @@ -134,7 +134,7 @@ TEST_F(FunctionSignatureBuilderTest, typeParamTests) {
.returnType("integer")
.argumentType("T(M)")
.build(),
"Named type cannot have parameters: 'T(M)'");
"Failed to parse type signature [T(M)]: syntax error, unexpected LPAREN, expecting YYEOF");
}

TEST_F(FunctionSignatureBuilderTest, anyInReturn) {
Expand Down
90 changes: 0 additions & 90 deletions velox/exec/tests/ParseTypeSignatureTest.cpp
View file
Open in desktop

This file was deleted.

10 changes: 8 additions & 2 deletions velox/expression/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.

add_library(velox_type_signature OBJECT TypeSignature.cpp)

target_link_libraries(velox_type_signature velox_common_base)

add_library(
velox_expression_functions FunctionSignature.cpp SignatureBinder.cpp
ReverseSignatureBinder.cpp)

target_link_libraries(velox_expression_functions velox_common_base
velox_type_calculation)
target_link_libraries(
velox_expression_functions velox_common_base velox_type_calculation
velox_type_signature velox_signature_parser)

add_library(
velox_expression
Expand Down Expand Up @@ -48,6 +53,7 @@ target_link_libraries(
velox_expression_functions velox_functions_util)

add_subdirectory(type_calculation)
add_subdirectory(signature_parser)

if(${VELOX_BUILD_TESTING})
add_subdirectory(tests)
Expand Down
77 changes: 0 additions & 77 deletions velox/expression/FunctionSignature.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,24 +51,6 @@ const std::vector<std::string> primitiveTypeNames() {
return kPrimitiveTypeNames;
}

void toAppend(
const facebook::velox::exec::TypeSignature& signature,
std::string* result) {
result->append(signature.toString());
}

std::string TypeSignature::toString() const {
std::ostringstream out;
if (rowFieldName_.has_value()) {
out << *rowFieldName_ << " ";
}
out << baseName_;
if (!parameters_.empty()) {
out << "(" << folly::join(",", parameters_) << ")";
}
return out.str();
}

std::string FunctionSignature::argumentsToString() const {
std::vector<std::string> arguments;
auto size = argumentTypes_.size();
Expand Down Expand Up @@ -110,65 +92,6 @@ size_t findNextComma(const std::string& str, size_t start) {
return std::string::npos;
}

TypeSignature parseTypeBaseSignature(
const std::string& signature,
std::vector<TypeSignature>&& parameters,
bool allowNamed) {
if (allowNamed) {
const auto spacePos = signature.find(' ');

// If there is a space and the name doesn't match a known type,
// e.g. "timestamp with timezone", this is a named row field.
if ((spacePos != std::string::npos) && !hasType(signature)) {
return TypeSignature(
signature.substr(spacePos + 1),
std::move(parameters),
signature.substr(0, spacePos));
}
}
return TypeSignature(signature, std::move(parameters));
}

TypeSignature parseTypeSignatureImpl(
const std::string& signature,
bool allowNamed) {
auto parenPos = signature.find('(');
if (parenPos == std::string::npos) {
return parseTypeBaseSignature(signature, {}, allowNamed);
}

auto baseName = signature.substr(0, parenPos);
std::vector<TypeSignature> nestedTypes;

auto endParenPos = signature.rfind(')');
VELOX_CHECK(
endParenPos != std::string::npos,
"Couldn't find the closing parenthesis.");

const bool isRow = (boost::algorithm::to_upper_copy(baseName) == "ROW");
auto prevPos = parenPos + 1;
auto commaPos = findNextComma(signature, prevPos);

while (commaPos != std::string::npos) {
auto token = signature.substr(prevPos, commaPos - prevPos);
boost::algorithm::trim(token);
nestedTypes.emplace_back(parseTypeSignatureImpl(token, isRow));

prevPos = commaPos + 1;
commaPos = findNextComma(signature, prevPos);
}

auto token = signature.substr(prevPos, endParenPos - prevPos);
boost::algorithm::trim(token);
nestedTypes.emplace_back(parseTypeSignatureImpl(token, isRow));

return parseTypeBaseSignature(baseName, std::move(nestedTypes), allowNamed);
}

TypeSignature parseTypeSignature(const std::string& signature) {
return parseTypeSignatureImpl(signature, false);
}

namespace {
/// Returns true only if 'str' contains digits.
bool isPositiveInteger(const std::string& str) {
Expand Down
78 changes: 9 additions & 69 deletions velox/expression/FunctionSignature.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#include <vector>

#include "velox/common/base/Exceptions.h"
#include "velox/expression/TypeSignature.h"
#include "velox/expression/signature_parser/SignatureParser.h"
#include "velox/type/Type.h"

namespace facebook::velox::exec {
Expand Down Expand Up @@ -97,53 +99,6 @@ class SignatureVariable {
bool comparableTypesOnly_ = false;
};

// Base type (e.g. map) and optional parameters (e.g. K, V).
class TypeSignature {
public:
/// @param baseName The base name of the type. Could describe a concrete type
/// name (e.g. map, bigint, double), or a variable (e.g. K, V).
/// @param parameters The optional parameters for the type. For example, the
/// signature "map(K, V)" would have two parameters, "K", and "V". All
/// parameters must be of the same ParameterType.
/// @param rowFieldName if this type signature is a field of another parent
/// row type, it can optionally have a name. E.g. `row(id bigint)` would have
/// "id" set as rowFieldName in the "bigint" parameter.
TypeSignature(
std::string baseName,
std::vector<TypeSignature> parameters,
std::optional<std::string> rowFieldName = std::nullopt)
: baseName_{std::move(baseName)},
parameters_{std::move(parameters)},
rowFieldName_(rowFieldName) {}

const std::string& baseName() const {
return baseName_;
}

const std::vector<TypeSignature>& parameters() const {
return parameters_;
}

const std::optional<std::string>& rowFieldName() const {
return rowFieldName_;
}

std::string toString() const;

bool operator==(const TypeSignature& rhs) const {
return baseName_ == rhs.baseName_ && parameters_ == rhs.parameters_ &&
rowFieldName_ == rhs.rowFieldName_;
}

private:
const std::string baseName_;
const std::vector<TypeSignature> parameters_;

// If this object is a field of another parent row type, it can optionally
// have a name, e.g, `row(id bigint)`
std::optional<std::string> rowFieldName_;
};

class FunctionSignature {
public:
/// @param variables_ Generic type names used in return type
Expand Down Expand Up @@ -263,21 +218,6 @@ inline void addVariable(

} // namespace

/// Parses a string into TypeSignature. The format of the string is type name,
/// optionally followed by type parameters enclosed in parenthesis.
///
/// Examples:
/// - bigint
/// - double
/// - array(T)
/// - map(K,V)
/// - row(named bigint,array(tinyint),T)
/// - function(S,T,R)
///
/// Row fields are allowed to be named or anonymous, e.g. "row(foo bigint)" or
/// "row(bigint)"
TypeSignature parseTypeSignature(const std::string& signature);

/// Convenience class for creating FunctionSignature instances.
///
/// Example of usage:
Expand Down Expand Up @@ -328,18 +268,18 @@ class FunctionSignatureBuilder {
}

FunctionSignatureBuilder& returnType(const std::string& type) {
returnType_.emplace(parseTypeSignature(type));
returnType_.emplace(*parseTypeSignature(type));
return *this;
}

FunctionSignatureBuilder& argumentType(const std::string& type) {
argumentTypes_.emplace_back(parseTypeSignature(type));
argumentTypes_.emplace_back(*parseTypeSignature(type));
constantArguments_.push_back(false);
return *this;
}

FunctionSignatureBuilder& constantArgumentType(const std::string& type) {
argumentTypes_.emplace_back(parseTypeSignature(type));
argumentTypes_.emplace_back(*parseTypeSignature(type));
constantArguments_.push_back(true);
return *this;
}
Expand Down Expand Up @@ -399,25 +339,25 @@ class AggregateFunctionSignatureBuilder {
}

AggregateFunctionSignatureBuilder& returnType(const std::string& type) {
returnType_.emplace(parseTypeSignature(type));
returnType_.emplace(*parseTypeSignature(type));
return *this;
}

AggregateFunctionSignatureBuilder& argumentType(const std::string& type) {
argumentTypes_.emplace_back(parseTypeSignature(type));
argumentTypes_.emplace_back(*parseTypeSignature(type));
constantArguments_.push_back(false);
return *this;
}

AggregateFunctionSignatureBuilder& constantArgumentType(
const std::string& type) {
argumentTypes_.emplace_back(parseTypeSignature(type));
argumentTypes_.emplace_back(*parseTypeSignature(type));
constantArguments_.push_back(true);
return *this;
}

AggregateFunctionSignatureBuilder& intermediateType(const std::string& type) {
intermediateType_.emplace(parseTypeSignature(type));
intermediateType_.emplace(*parseTypeSignature(type));
return *this;
}

Expand Down
Loading