Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query serialisation part 3 #2989

Merged
merged 16 commits into from
Feb 14, 2018
Prev Previous commit
Next Next commit
Parser support for ANY/SOME/ALL/NONE
James Stone committed Feb 9, 2018
commit 6d769eb8f6ede9e7dfe36c25d721dd9068da3fd9
5 changes: 5 additions & 0 deletions src/realm/parser/keypath_mapping.cpp
Original file line number Diff line number Diff line change
@@ -52,6 +52,11 @@ void KeyPathMapping::remove_mapping(ConstTableRef table, std::string name)
m_mapping.erase(it);
}

bool KeyPathMapping::has_mapping(ConstTableRef table, std::string name)
{
return m_mapping.find({table, name}) != m_mapping.end();
}

// This may be premature optimisation, but it'll be super fast and it doesn't
// bother dragging in anything locale specific for case insensitive comparisons.
bool is_backlinks_prefix(std::string& s) {
1 change: 1 addition & 0 deletions src/realm/parser/keypath_mapping.hpp
Original file line number Diff line number Diff line change
@@ -58,6 +58,7 @@ class KeyPathMapping
// returns true if added, false if duplicate key already exists
bool add_mapping(ConstTableRef table, std::string name, std::string alias);
void remove_mapping(ConstTableRef table, std::string name);
bool has_mapping(ConstTableRef table, std::string name);
KeyPathElement process_next_path(ConstTableRef table, KeyPath& path, size_t& index);
void set_allow_backlinks(bool allow);
static Table* table_getter(TableRef table, const std::vector<KeyPathElement>& links);
63 changes: 54 additions & 9 deletions src/realm/parser/parser.cpp
Original file line number Diff line number Diff line change
@@ -33,6 +33,12 @@ using namespace tao::pegtl;
namespace realm {
namespace parser {

// forward declarations
struct expr;
struct string_oper;
struct symbolic_oper;
struct pred;

// strings
struct unicode : list< seq< one< 'u' >, rep< 4, must< xdigit > > >, one< '\\' > > {};
struct escaped_char : one< '"', '\'', '\\', '/', 'b', 'f', 'n', 'r', 't', '0' > {};
@@ -103,7 +109,6 @@ struct collection_operator_match : sor< seq< key_path_prefix, key_collection_ope
struct argument_index : plus< digit > {};
struct argument : seq< one< '$' >, argument_index > {};

struct pred;
// subquery eg: SUBQUERY(items, $x, $x.name CONTAINS 'a' && $x.price > 5).@count
struct subq_prefix : seq< string_token_t("subquery"), star< blank >, one< '(' > > {};
struct subq_suffix : one< ')' > {};
@@ -113,11 +118,19 @@ struct sub_result_op : sor< count, size > {};
struct sub_preamble : seq< subq_prefix, pad< sub_path, blank >, one< ',' >, pad< sub_var_name, blank >, one< ',' > > {};
struct subquery : seq< sub_preamble, pad< pred, blank >, pad< subq_suffix, blank >, sub_result_op > {};

// list aggregate operations
struct agg_target : seq< key_path > {};
struct agg_any : seq< sor< string_token_t("any"), string_token_t("some") >, plus<blank>, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {};
struct agg_all : seq< string_token_t("all"), plus<blank>, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {};
struct agg_none : seq< string_token_t("none"), plus<blank>, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {};
//struct agg_in : seq< pad< expr, blank>, string_token_t("in"), plus< blank >, agg_target > {};
struct agg_shortcut_pred : sor< agg_any, agg_all, agg_none > {};

// expressions and operators
struct expr : sor< dq_string, sq_string, timestamp, number, argument, true_value, false_value, null_value, base64, collection_operator_match, subquery, key_path > {};
struct case_insensitive : TAOCPP_PEGTL_ISTRING("[c]") {};

struct eq : seq< sor< two< '=' >, one< '=' > >, star< blank >, opt< case_insensitive > >{};
struct eq : seq< sor< two< '=' >, one< '=' >, string_token_t("in") >, star< blank >, opt< case_insensitive > >{};
struct noteq : seq< sor< tao::pegtl::string< '!', '=' >, tao::pegtl::string< '<', '>' > >, star< blank >, opt< case_insensitive > > {};
struct lteq : sor< tao::pegtl::string< '<', '=' >, tao::pegtl::string< '=', '<' > > {};
struct lt : one< '<' > {};
@@ -156,7 +169,7 @@ struct true_pred : string_token_t("truepredicate") {};
struct false_pred : string_token_t("falsepredicate") {};

struct not_pre : seq< sor< one< '!' >, string_token_t("not") > > {};
struct atom_pred : seq< opt< not_pre >, pad< sor< group_pred, true_pred, false_pred, comparison_pred >, blank >, star< pad< descriptor_ordering, blank > > > {};
struct atom_pred : seq< opt< not_pre >, pad< sor<group_pred, true_pred, false_pred, agg_shortcut_pred, comparison_pred>, blank >, star< pad< descriptor_ordering, blank > > > {};

struct and_op : pad< sor< two< '&' >, string_token_t("and") >, blank > {};
struct or_op : pad< sor< two< '|' >, string_token_t("or") >, blank > {};
@@ -178,6 +191,7 @@ struct ParserState
DescriptorOrderingState::SingleOrderingState temp_ordering;
std::string subquery_path, subquery_var;
std::vector<Predicate> subqueries;
Predicate::ComparisonType pending_comparison_type;

Predicate *current_group()
{
@@ -220,6 +234,12 @@ struct ParserState
pending_op = Expression::KeyPathOp::None;
}

void apply_list_aggregate_operation()
{
last_predicate()->cmpr.compare_type = pending_comparison_type;
pending_comparison_type = Predicate::ComparisonType::Unspecified;
}

void add_expression(Expression && exp)
{
Predicate *current = last_predicate();
@@ -285,11 +305,17 @@ struct ParserState
template< typename Rule >
struct action : nothing< Rule > {};

#ifdef REALM_PARSER_PRINT_TOKENS
#define DEBUG_PRINT_TOKEN(string) do { std::cout << string << std::endl; } while (0)
#else
#define DEBUG_PRINT_TOKEN(string) do { static_cast<void>(string); } while (0)
#endif
//#ifdef REALM_PARSER_PRINT_TOKENS
//#if 1
// #define DEBUG_PRINT_TOKEN(string) do { std::cout << string << std::endl; } while (0)
//#else
// #define DEBUG_PRINT_TOKEN(string) do { static_cast<void>(string); } while (0)
//#endif

void DEBUG_PRINT_TOKEN(std::string s)
{
//std::cout << s << std::endl;
}

template<> struct action< and_op >
{
@@ -536,6 +562,25 @@ template<> struct action< collection_operator_match > {
}
};

#define LIST_AGG_OP_TYPE_ACTION(rule, type) \
template<> struct action< rule > { \
template< typename Input > \
static void apply(const Input& in, ParserState& state) { \
DEBUG_PRINT_TOKEN(in.string() + #rule); \
state.pending_comparison_type = type; }};

LIST_AGG_OP_TYPE_ACTION(agg_any, Predicate::ComparisonType::Any)
LIST_AGG_OP_TYPE_ACTION(agg_all, Predicate::ComparisonType::All)
LIST_AGG_OP_TYPE_ACTION(agg_none, Predicate::ComparisonType::None)

template<> struct action< agg_shortcut_pred > {
template< typename Input >
static void apply(const Input& in, ParserState& state) {
DEBUG_PRINT_TOKEN(in.string() + " Aggregate shortcut matched");
state.apply_list_aggregate_operation();
}
};

template<> struct action< true_pred >
{
template< typename Input >
@@ -560,7 +605,7 @@ template<> struct action< false_pred >
template<> struct action< rule > { \
template< typename Input > \
static void apply(const Input& in, ParserState& state) { \
DEBUG_PRINT_TOKEN(in.string()); \
DEBUG_PRINT_TOKEN(in.string() + #oper); \
state.last_predicate()->cmpr.op = oper; }};

OPERATOR_ACTION(eq, Predicate::Operator::Equal)
9 changes: 9 additions & 0 deletions src/realm/parser/parser.hpp
Original file line number Diff line number Diff line change
@@ -75,11 +75,20 @@ struct Predicate
CaseInsensitive,
};

enum class ComparisonType
{
Unspecified,
Any,
All,
None,
};

struct Comparison
{
Operator op = Operator::None;
OperatorOption option = OperatorOption::None;
Expression expr[2] = {{Expression::Type::None, ""}, {Expression::Type::None, ""}};
ComparisonType compare_type = ComparisonType::Unspecified;
};

struct Compound
15 changes: 15 additions & 0 deletions src/realm/parser/parser_utils.cpp
Original file line number Diff line number Diff line change
@@ -126,6 +126,21 @@ const char* collection_operator_to_str(parser::Expression::KeyPathOp op)
return "";
}

const char* comparison_type_to_str(parser::Predicate::ComparisonType type)
{
switch (type) {
case parser::Predicate::ComparisonType::Unspecified:
return "";
case parser::Predicate::ComparisonType::All:
return "ALL";
case parser::Predicate::ComparisonType::None:
return "NONE";
case parser::Predicate::ComparisonType::Any:
return "ANY";
}
return "";
}

using KeyPath = std::vector<std::string>;

KeyPath key_path_from_string(const std::string &s) {
2 changes: 1 addition & 1 deletion src/realm/parser/parser_utils.hpp
Original file line number Diff line number Diff line change
@@ -61,8 +61,8 @@ template <>
const char* type_to_str<Link>();

const char* data_type_to_str(DataType type);

const char* collection_operator_to_str(parser::Expression::KeyPathOp op);
const char* comparison_type_to_str(parser::Predicate::ComparisonType type);

using KeyPath = std::vector<std::string>;
KeyPath key_path_from_string(const std::string &s);
108 changes: 105 additions & 3 deletions src/realm/parser/query_builder.cpp
Original file line number Diff line number Diff line change
@@ -287,7 +287,7 @@ void add_link_constraint_to_query(realm::Query &query,
Predicate::Operator op,
const PropertyExpression &prop_expr,
const ValueExpression &value_expr) {
size_t row_index = value_expr.arguments->object_index_for_argument(stot<int>(value_expr.value->s));
size_t row_index = value_expr.arguments->object_index_for_argument(stot<int>(value_expr.value.s));
realm_precondition(prop_expr.link_chain.size() == 1, "KeyPath queries not supported for object comparisons.");
switch (op) {
case Predicate::Operator::NotEqual:
@@ -525,24 +525,126 @@ void add_comparison_to_query(Query &query, ExpressionContainer& lhs, Predicate::
}
}


std::pair<std::string, std::string> separate_list_parts(PropertyExpression& pe) {
std::string pre_and_list = "";
std::string post_list = "";
bool past_list = false;
for (KeyPathElement& e : pe.link_chain) {
std::string cur_name;
if (e.is_backlink) {
cur_name = std::string("@links") + util::serializer::value_separator + std::string(e.table->get_name()) + util::serializer::value_separator + std::string(e.table->get_column_name(e.col_ndx));
} else {
cur_name = e.table->get_column_name(e.col_ndx);
}
if (!past_list) {
if (!pre_and_list.empty()) {
pre_and_list += util::serializer::value_separator;
}
pre_and_list += cur_name;
} else {
realm_precondition(!e.is_backlink && e.col_type != type_LinkList, util::format("The keypath after '%1' must not contain any additional list properties, but '%2' is a list.", pre_and_list, cur_name));
if (!post_list.empty()) {
post_list += util::serializer::value_separator;
}
post_list += cur_name;
}
if (e.is_backlink || e.col_type == type_LinkList) {
past_list = true;
}
}
return {pre_and_list, post_list};
}


// some query types are not supported in core but can be produced by a transformation:
// "ALL path.to.list.property >= 20" --> "SUBQUERY(path.to.list, $x, $x.property >= 20).@count == path.to.list.@count"
// "NONE path.to.list.property >= 20" --> "SUBQUERY(path.to.list, $x, $x.property >= 20).@count == 0"
void preprocess_for_comparison_types(Query &query, Predicate::Comparison &cmpr, ExpressionContainer &lhs, ExpressionContainer &rhs, Arguments &args, parser::KeyPathMapping& mapping)
{
auto get_cmp_type_name = [&]() {
if (cmpr.compare_type == Predicate::ComparisonType::Any) {
return util::format("'%1' or 'SOME'", comparison_type_to_str(Predicate::ComparisonType::Any));
}
return util::format("'%1'", comparison_type_to_str(cmpr.compare_type));
};

if (cmpr.compare_type != Predicate::ComparisonType::Unspecified)
{
realm_precondition(lhs.type == ExpressionContainer::ExpressionInternal::exp_Property,
util::format("The expression after %1 must be a keypath containing a list",
get_cmp_type_name()));
bool found_some_list = false;
for (KeyPathElement e : lhs.get_property().link_chain) {
if (e.col_type == type_LinkList || e.is_backlink) {
found_some_list = true;
break;
}
}
realm_precondition(found_some_list, util::format("The keypath following %1 must contain a list",
get_cmp_type_name()));
}

if (cmpr.compare_type == Predicate::ComparisonType::All || cmpr.compare_type == Predicate::ComparisonType::None) {
realm_precondition(rhs.type == ExpressionContainer::ExpressionInternal::exp_Value,
util::format("The comparison in an %1 clause must be between a keypath and a value",
get_cmp_type_name()));

parser::Expression exp(parser::Expression::Type::SubQuery);
std::pair<std::string, std::string> path_parts = separate_list_parts(lhs.get_property());
exp.subquery_path = path_parts.first;

util::serializer::SerialisationState temp_state;
std::string var_name;
for (var_name = temp_state.get_variable_name(query.get_table());
mapping.has_mapping(query.get_table(), var_name);
var_name = temp_state.get_variable_name(query.get_table())) {
temp_state.subquery_prefix_list.push_back(var_name);
}

exp.subquery_var = var_name;
exp.subquery = std::make_shared<Predicate>(Predicate::Type::Comparison);
exp.subquery->cmpr.expr[0] = parser::Expression(parser::Expression::Type::KeyPath, var_name + util::serializer::value_separator + path_parts.second);
exp.subquery->cmpr.op = cmpr.op;
exp.subquery->cmpr.option = cmpr.option;
exp.subquery->cmpr.expr[1] = cmpr.expr[1];

lhs = ExpressionContainer(query, exp, args, mapping);
cmpr.op = parser::Predicate::Operator::Equal;
cmpr.option = parser::Predicate::OperatorOption::None;

if (cmpr.compare_type == Predicate::ComparisonType::All) {
parser::Expression rhs_exp(path_parts.first, parser::Expression::KeyPathOp::Count, "");
rhs = ExpressionContainer(query, rhs_exp, args, mapping);
} else if (cmpr.compare_type == Predicate::ComparisonType::None) {
parser::Expression rhs_exp(parser::Expression::Type::Number, "0");
rhs = ExpressionContainer(query, rhs_exp, args, mapping);
} else {
REALM_UNREACHABLE();
}
}
}


bool is_property_operation(parser::Expression::Type type)
{
return type == parser::Expression::Type::KeyPath || type == parser::Expression::Type::SubQuery;
}

void add_comparison_to_query(Query &query, const Predicate &pred, Arguments &args, parser::KeyPathMapping& mapping)
{
const Predicate::Comparison &cmpr = pred.cmpr;
Predicate::Comparison cmpr = pred.cmpr;
auto lhs_type = cmpr.expr[0].type, rhs_type = cmpr.expr[1].type;

if (!is_property_operation(lhs_type) && !is_property_operation(rhs_type)) {
// value vs value expressions are not supported (ex: 2 < 3 or null != null)
throw std::logic_error("Predicate expressions must compare a keypath and another keypath or a constant value");
}

ExpressionContainer lhs(query, cmpr.expr[0], args, mapping);
ExpressionContainer rhs(query, cmpr.expr[1], args, mapping);

preprocess_for_comparison_types(query, cmpr, lhs, rhs, args, mapping);

if (lhs.is_null()) {
add_null_comparison_to_query(query, cmpr, rhs, NullLocation::NullOnLHS);
}
Loading