diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c8844c00bc..35271f1f771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,16 @@ ### Enhancements -* None. +* Parser improvements: + - Support subquery count expressions, for example: "SUBQUERY(list, $x, $x.price > 5 && $x.colour == 'blue').@count > 1" + - Subqueries can be nested, but all properties must start with the closest variable (no parent scope properties) + - Support queries over unnamed backlinks, for example: "@links.class_Person.items.cost > 10" + - Backlinks can be used like lists in expressions including: min, max, sum, avg, count/size, and subqueries + - Keypath substitution is supported to allow querying over named backlinks and property aliases, see `KeyPathMapping` + - Parsing backlinks can be disabled at runtime by configuring `KeyPathMapping::set_allow_backlinks` + - Support for ANY/SOME/ALL/NONE on list properties (parser only). For example: `ALL items.price > 10` + - Support for operator 'IN' on list properties (parser only). For example: `'milk' IN ingredients.name` + PR [#2989](https://github.com/realm/realm-core/pull/2989). ----------- diff --git a/src/realm/CMakeLists.txt b/src/realm/CMakeLists.txt index b603094c77b..c6fcb591677 100644 --- a/src/realm/CMakeLists.txt +++ b/src/realm/CMakeLists.txt @@ -134,20 +134,24 @@ set(REALM_INSTALL_GENERAL_HEADERS set(REALM_PARSER_SOURCES parser/expression_container.cpp + parser/keypath_mapping.cpp parser/parser.cpp parser/parser_utils.cpp parser/property_expression.cpp parser/query_builder.cpp + parser/subquery_expression.cpp parser/value_expression.cpp ) # REALM_PARSER_SOURCES set(REALM_PARSER_HEADERS parser/collection_operator_expression.hpp parser/expression_container.hpp + parser/keypath_mapping.hpp parser/parser.hpp parser/parser_utils.hpp parser/property_expression.hpp parser/query_builder.hpp + parser/subquery_expression.hpp parser/value_expression.hpp ) # REALM_PARSER_HEADERS diff --git a/src/realm/parser/collection_operator_expression.hpp b/src/realm/parser/collection_operator_expression.hpp index 5ec417bbc97..6575b302c68 100644 --- a/src/realm/parser/collection_operator_expression.hpp +++ b/src/realm/parser/collection_operator_expression.hpp @@ -40,7 +40,7 @@ struct CollectionOperatorExpression size_t post_link_col_ndx; DataType post_link_col_type; - CollectionOperatorExpression(PropertyExpression&& exp, std::string suffix_path) + CollectionOperatorExpression(PropertyExpression&& exp, std::string suffix_path, parser::KeyPathMapping& mapping) : pe(std::move(exp)) , post_link_col_ndx(realm::not_found) { @@ -53,35 +53,44 @@ struct CollectionOperatorExpression if (requires_suffix_path) { Table* pre_link_table = pe.table_getter(); REALM_ASSERT(pre_link_table); - StringData list_property_name = pre_link_table->get_column_name(pe.col_ndx); - precondition(pe.col_type == type_LinkList, + StringData list_property_name; + if (pe.dest_type_is_backlink()) { + list_property_name = "linking object"; + } else { + list_property_name = pre_link_table->get_column_name(pe.get_dest_ndx()); + } + realm_precondition(pe.get_dest_type() == type_LinkList || pe.dest_type_is_backlink(), util::format("The '%1' operation must be used on a list property, but '%2' is not a list", util::collection_operator_to_str(OpType), list_property_name)); - TableRef post_link_table = pe.table_getter()->get_link_target(pe.col_ndx); + ConstTableRef post_link_table; + if (pe.dest_type_is_backlink()) { + post_link_table = pe.get_dest_table(); + } else { + post_link_table = pe.get_dest_table()->get_link_target(pe.get_dest_ndx()); + } REALM_ASSERT(post_link_table); StringData printable_post_link_table_name = get_printable_table_name(*post_link_table); KeyPath suffix_key_path = key_path_from_string(suffix_path); - precondition(suffix_path.size() > 0 && suffix_key_path.size() > 0, + + realm_precondition(suffix_path.size() > 0 && suffix_key_path.size() > 0, util::format("A property from object '%1' must be provided to perform operation '%2'", printable_post_link_table_name, util::collection_operator_to_str(OpType))); + size_t index = 0; + KeyPathElement element = mapping.process_next_path(post_link_table, suffix_key_path, index); - precondition(suffix_key_path.size() == 1, + realm_precondition(suffix_key_path.size() == 1, util::format("Unable to use '%1' because collection aggreate operations are only supported " "for direct properties at this time", suffix_path)); - post_link_col_ndx = pe.table_getter()->get_link_target(pe.col_ndx)->get_column_index(suffix_key_path[0]); - - precondition(post_link_col_ndx != realm::not_found, - util::format("No property '%1' on object of type '%2'", - suffix_path, printable_post_link_table_name)); - post_link_col_type = pe.table_getter()->get_link_target(pe.col_ndx)->get_column_type(post_link_col_ndx); + post_link_col_ndx = element.col_ndx; + post_link_col_type = element.col_type; } else { // !requires_suffix_path - post_link_col_type = pe.col_type; + post_link_col_type = pe.get_dest_type(); - precondition(suffix_path.empty(), + realm_precondition(suffix_path.empty(), util::format("An extraneous property '%1' was found for operation '%2'", suffix_path, util::collection_operator_to_str(OpType))); } @@ -109,66 +118,66 @@ struct CollectionOperatorGetter { template struct CollectionOperatorGetter::value || -std::is_same::value || -std::is_same::value -> >{ +typename std::enable_if_t::value> >{ static SubColumnAggregate > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).template column(expr.post_link_col_ndx).min(); + if (expr.pe.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.pe.get_dest_table(), expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).min(); + } else { + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).min(); + } } }; template struct CollectionOperatorGetter::value || -std::is_same::value || -std::is_same::value -> >{ +typename std::enable_if_t::value> >{ static SubColumnAggregate > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).template column(expr.post_link_col_ndx).max(); + if (expr.pe.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.pe.get_dest_table(), expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).max(); + } else { + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).max(); + } } }; template struct CollectionOperatorGetter::value || -std::is_same::value || -std::is_same::value -> >{ +typename std::enable_if_t::value> >{ static SubColumnAggregate > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).template column(expr.post_link_col_ndx).sum(); + if (expr.pe.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.pe.get_dest_table(), expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).sum(); + } else { + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).sum(); + } } }; template struct CollectionOperatorGetter::value || -std::is_same::value || -std::is_same::value -> >{ +typename std::enable_if_t::value> >{ static SubColumnAggregate > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).template column(expr.post_link_col_ndx).average(); + if (expr.pe.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.pe.get_dest_table(), expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).average(); + } else { + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).template column(expr.post_link_col_ndx).average(); + } } }; template struct CollectionOperatorGetter::value || - std::is_same::value || - std::is_same::value - > >{ + typename std::enable_if_t::value> >{ static LinkCount convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).count(); + if (expr.pe.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.pe.get_dest_table(), expr.pe.get_dest_ndx()).count(); + } else { + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).count(); + } } }; @@ -176,7 +185,7 @@ template <> struct CollectionOperatorGetter{ static SizeOperator > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).size(); + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).size(); } }; @@ -184,7 +193,7 @@ template <> struct CollectionOperatorGetter{ static SizeOperator > convert(const CollectionOperatorExpression& expr) { - return expr.table_getter()->template column(expr.pe.col_ndx).size(); + return expr.table_getter()->template column(expr.pe.get_dest_ndx()).size(); } }; diff --git a/src/realm/parser/expression_container.cpp b/src/realm/parser/expression_container.cpp index 7c3c3fcd50a..948aeb416bc 100644 --- a/src/realm/parser/expression_container.cpp +++ b/src/realm/parser/expression_container.cpp @@ -24,43 +24,43 @@ namespace realm { namespace parser { -ExpressionContainer::ExpressionContainer(Query& query, const parser::Expression& e, query_builder::Arguments& args) +ExpressionContainer::ExpressionContainer(Query& query, const parser::Expression& e, query_builder::Arguments& args, parser::KeyPathMapping& mapping) { if (e.type == parser::Expression::Type::KeyPath) { - PropertyExpression pe(query, e.s); + PropertyExpression pe(query, e.s, mapping); switch (e.collection_op) { case parser::Expression::KeyPathOp::Min: type = ExpressionInternal::exp_OpMin; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); break; case parser::Expression::KeyPathOp::Max: type = ExpressionInternal::exp_OpMax; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); break; case parser::Expression::KeyPathOp::Sum: type = ExpressionInternal::exp_OpSum; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); break; case parser::Expression::KeyPathOp::Avg: type = ExpressionInternal::exp_OpAvg; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); break; case parser::Expression::KeyPathOp::Count: REALM_FALLTHROUGH; case parser::Expression::KeyPathOp::SizeString: REALM_FALLTHROUGH; case parser::Expression::KeyPathOp::SizeBinary: - if (pe.col_type == type_LinkList || pe.col_type == type_Link) { + if (pe.get_dest_type() == type_LinkList || pe.get_dest_type() == type_Link) { type = ExpressionInternal::exp_OpCount; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); } - else if (pe.col_type == type_String) { + else if (pe.get_dest_type() == type_String) { type = ExpressionInternal::exp_OpSizeString; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); } - else if (pe.col_type == type_Binary) { + else if (pe.get_dest_type() == type_Binary) { type = ExpressionInternal::exp_OpSizeBinary; - storage = CollectionOperatorExpression(std::move(pe), e.op_suffix); + storage = CollectionOperatorExpression(std::move(pe), e.op_suffix, mapping); } else { throw std::runtime_error("Invalid query: @size and @count can only operate on types list, binary, or string"); @@ -72,6 +72,20 @@ ExpressionContainer::ExpressionContainer(Query& query, const parser::Expression& break; } } + else if (e.type == parser::Expression::Type::SubQuery) { + REALM_ASSERT_DEBUG(e.subquery); + type = ExpressionInternal::exp_SubQuery; + SubqueryExpression exp(query, e.subquery_path, e.subquery_var, mapping); + // The least invasive way to do the variable substituion is to simply remove the variable prefix + // from all query keypaths. This only works because core does not support anything else (such as referencing + // other properties of the parent table). + // This means that every keypath must start with the variable, we require it to be there and remove it. + bool did_add = mapping.add_mapping(exp.get_subquery().get_table(), e.subquery_var, ""); + realm_precondition(did_add, util::format("Unable to create a subquery expression with variable '%1' since an identical variable already exists in this context", e.subquery_var)); + query_builder::apply_predicate(exp.get_subquery(), *e.subquery, args, mapping); + mapping.remove_mapping(exp.get_subquery().get_table(), e.subquery_var); + storage = std::move(exp); + } else { type = ExpressionInternal::exp_Value; storage = ValueExpression(query, &args, &e); @@ -125,6 +139,12 @@ CollectionOperatorExpression& Express return util::any_cast&>(storage); } +SubqueryExpression& ExpressionContainer::get_subexpression() +{ + REALM_ASSERT_DEBUG(type == ExpressionInternal::exp_SubQuery); + return util::any_cast(storage); +} + DataType ExpressionContainer::check_type_compatibility(DataType other_type) { util::Optional self_type; @@ -133,7 +153,7 @@ DataType ExpressionContainer::check_type_compatibility(DataType other_type) self_type = other_type; // we'll try to parse the value as other_type and fail there if not possible break; case ExpressionInternal::exp_Property: - self_type = get_property().col_type; // must match + self_type = get_property().get_dest_type(); // must match break; case ExpressionInternal::exp_OpMin: self_type = get_min().post_link_col_type; @@ -147,6 +167,8 @@ DataType ExpressionContainer::check_type_compatibility(DataType other_type) case ExpressionInternal::exp_OpAvg: self_type = get_avg().post_link_col_type; break; + case ExpressionInternal::exp_SubQuery: + REALM_FALLTHROUGH; case ExpressionInternal::exp_OpCount: // linklist count can handle any numeric type if (other_type == type_Int || other_type == type_Double || other_type == type_Float) { @@ -176,15 +198,16 @@ bool is_count_type(ExpressionContainer::ExpressionInternal exp_type) { return exp_type == ExpressionContainer::ExpressionInternal::exp_OpCount || exp_type == ExpressionContainer::ExpressionInternal::exp_OpSizeString - || exp_type == ExpressionContainer::ExpressionInternal::exp_OpSizeBinary; + || exp_type == ExpressionContainer::ExpressionInternal::exp_OpSizeBinary + || exp_type == ExpressionContainer::ExpressionInternal::exp_SubQuery; } DataType ExpressionContainer::get_comparison_type(ExpressionContainer& rhs) { // check for strongly typed expressions first if (type == ExpressionInternal::exp_Property) { - return rhs.check_type_compatibility(get_property().col_type); + return rhs.check_type_compatibility(get_property().get_dest_type()); } else if (rhs.type == ExpressionInternal::exp_Property) { - return check_type_compatibility(rhs.get_property().col_type); + return check_type_compatibility(rhs.get_property().get_dest_type()); } else if (type == ExpressionInternal::exp_OpMin) { return rhs.check_type_compatibility(get_min().post_link_col_type); } else if (type == ExpressionInternal::exp_OpMax) { diff --git a/src/realm/parser/expression_container.hpp b/src/realm/parser/expression_container.hpp index bd2d31e7f1c..a9ee9d6e864 100644 --- a/src/realm/parser/expression_container.hpp +++ b/src/realm/parser/expression_container.hpp @@ -25,6 +25,7 @@ #include "parser.hpp" #include "property_expression.hpp" #include "query_builder.hpp" +#include "subquery_expression.hpp" #include "value_expression.hpp" namespace realm { @@ -33,7 +34,7 @@ namespace parser { class ExpressionContainer { public: - ExpressionContainer(Query& query, const parser::Expression& e, query_builder::Arguments& args); + ExpressionContainer(Query& query, const parser::Expression& e, query_builder::Arguments& args, parser::KeyPathMapping& mapping); bool is_null(); @@ -46,6 +47,7 @@ class ExpressionContainer CollectionOperatorExpression& get_count(); CollectionOperatorExpression& get_size_string(); CollectionOperatorExpression& get_size_binary(); + SubqueryExpression& get_subexpression(); DataType check_type_compatibility(DataType type); DataType get_comparison_type(ExpressionContainer& rhs); @@ -60,7 +62,8 @@ class ExpressionContainer exp_OpAvg, exp_OpCount, exp_OpSizeString, - exp_OpSizeBinary + exp_OpSizeBinary, + exp_SubQuery }; ExpressionInternal type; diff --git a/src/realm/parser/keypath_mapping.cpp b/src/realm/parser/keypath_mapping.cpp new file mode 100644 index 00000000000..f848b67c100 --- /dev/null +++ b/src/realm/parser/keypath_mapping.cpp @@ -0,0 +1,148 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "keypath_mapping.hpp" + +#include + +namespace realm { +namespace parser { + +std::size_t TableAndColHash::operator () (const std::pair &p) const { + // in practice, table names are unique between tables and column names are unique within a table + std::string combined = std::string(p.first->get_name()) + p.second; + return std::hash{}(combined); +} + + +KeyPathMapping::KeyPathMapping() +: m_allow_backlinks(true) +, m_mapping() +{ +} + +bool KeyPathMapping::add_mapping(ConstTableRef table, std::string name, std::string alias) +{ + if (m_mapping.find({table, name}) == m_mapping.end()) { + m_mapping[{table, name}] = alias; + return true; + } + return false; +} + +void KeyPathMapping::remove_mapping(ConstTableRef table, std::string name) +{ + auto it = m_mapping.find({table, name}); + REALM_ASSERT_DEBUG(it != m_mapping.end()); + m_mapping.erase(it); +} + +bool KeyPathMapping::has_mapping(ConstTableRef table, std::string name) +{ + return m_mapping.find({table, name}) != m_mapping.end(); +} + +// This may be premature optimisation, but it'll be super fast and it doesn't +// bother dragging in anything locale specific for case insensitive comparisons. +bool is_backlinks_prefix(std::string& s) { + return s.size() == 6 && s[0] == '@' + && (s[1] == 'l' || s[1] == 'L') + && (s[2] == 'i' || s[2] == 'I') + && (s[3] == 'n' || s[3] == 'N') + && (s[4] == 'k' || s[4] == 'K') + && (s[5] == 's' || s[5] == 'S'); +} + +KeyPathElement KeyPathMapping::process_next_path(ConstTableRef table, KeyPath& keypath, size_t& index) +{ + REALM_ASSERT_DEBUG(index < keypath.size()); + + // Perform substitution if the alias is found in the mapping + auto it = m_mapping.find({table, keypath[index]}); + if (it != m_mapping.end()) { + // the alias needs to be mapped because it might be more than a single path + // for example named backlinks must alias to @links.class_Name.property + KeyPath mapped_path = key_path_from_string(it->second); + keypath.erase(keypath.begin() + index); + keypath.insert(keypath.begin() + index, mapped_path.begin(), mapped_path.end()); + } + + // Process backlinks which consumes 3 parts of the keypath + if (is_backlinks_prefix(keypath[index])) { + realm_precondition(index + 2 < keypath.size(), "'@links' must be proceeded by type name and a property name"); + + Table::BacklinkOrigin info = table->find_backlink_origin(keypath[index + 1], keypath[index + 2]); + realm_precondition(bool(info), util::format("No property '%1' found in type '%2' which links to type '%3'", + keypath[index + 2], get_printable_table_name(keypath[index + 1]), get_printable_table_name(*table))); + + if (!m_allow_backlinks) { + throw BacklinksRestrictedError(util::format( + "Querying over backlinks is disabled but backlinks were found in the inverse relationship of property '%1' on type '%2'", + keypath[index + 2], get_printable_table_name(keypath[index + 1]))); + } + + index = index + 3; + KeyPathElement element; + element.table = info->first; + element.col_ndx = info->second; + element.col_type = type_LinkList; // backlinks should be operated on as a list + element.is_backlink = true; + return element; + } + + // Process a single property + size_t col_ndx = table->get_column_index(keypath[index]); + realm_precondition(col_ndx != realm::not_found, + util::format("No property '%1' on object of type '%2'", keypath[index], get_printable_table_name(*table))); + + DataType cur_col_type = table->get_column_type(col_ndx); + + index++; + KeyPathElement element; + element.table = table; + element.col_ndx = col_ndx; + element.col_type = cur_col_type; + element.is_backlink = false; + return element; +} + +void KeyPathMapping::set_allow_backlinks(bool allow) +{ + m_allow_backlinks = allow; +} + +Table* KeyPathMapping::table_getter(TableRef table, const std::vector& links) +{ + if (links.empty()) { + return table.get(); + } + // mutates m_link_chain on table + for (size_t i = 0; i < links.size() - 1; i++) { + if (links[i].is_backlink) { + table->backlink(*links[i].table, links[i].col_ndx); + } + else { + table->link(links[i].col_ndx); + } + } + return table.get(); +} + + +} // namespace parser +} // namespace realm diff --git a/src/realm/parser/keypath_mapping.hpp b/src/realm/parser/keypath_mapping.hpp new file mode 100644 index 00000000000..20c0f9bf2a6 --- /dev/null +++ b/src/realm/parser/keypath_mapping.hpp @@ -0,0 +1,73 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_KEYPATH_MAPPING_HPP +#define REALM_KEYPATH_MAPPING_HPP + +#include + +#include "parser_utils.hpp" + +#include +#include + +namespace realm { +namespace parser { + +struct KeyPathElement +{ + ConstTableRef table; + size_t col_ndx; + DataType col_type; + bool is_backlink; +}; + +class BacklinksRestrictedError : public std::runtime_error { +public: + BacklinksRestrictedError(const std::string& msg) : std::runtime_error(msg) {} + /// runtime_error::what() returns the msg provided in the constructor. +}; + +struct TableAndColHash { + std::size_t operator () (const std::pair &p) const; +}; + + +// This class holds state which allows aliasing variable names in key paths used in queries. +// It is used to allow variable naming in subqueries such as 'SUBQUERY(list, $obj, $obj.intCol = 5).@count' +// It can also be used to allow querying named backlinks if bindings provide the mappings themselves. +class KeyPathMapping +{ +public: + KeyPathMapping(); + // returns true if added, false if duplicate key already exists + bool add_mapping(ConstTableRef table, std::string name, std::string alias); + void remove_mapping(ConstTableRef table, std::string name); + bool has_mapping(ConstTableRef table, std::string name); + KeyPathElement process_next_path(ConstTableRef table, KeyPath& path, size_t& index); + void set_allow_backlinks(bool allow); + static Table* table_getter(TableRef table, const std::vector& links); +protected: + bool m_allow_backlinks; + std::unordered_map, std::string, TableAndColHash> m_mapping; +}; + +} // namespace parser +} // namespace realm + +#endif // REALM_KEYPATH_MAPPING_HPP diff --git a/src/realm/parser/parser.cpp b/src/realm/parser/parser.cpp index 682b2a25fcf..0fc5dfd6758 100644 --- a/src/realm/parser/parser.cpp +++ b/src/realm/parser/parser.cpp @@ -33,6 +33,12 @@ using namespace tao::pegtl; namespace realm { namespace parser { +// forward declarations +struct expr; +struct string_oper; +struct symbolic_oper; +struct pred; + // strings struct unicode : list< seq< one< 'u' >, rep< 4, must< xdigit > > >, one< '\\' > > {}; struct escaped_char : one< '"', '\'', '\\', '/', 'b', 'f', 'n', 'r', 't', '0' > {}; @@ -86,12 +92,13 @@ struct avg : TAOCPP_PEGTL_ISTRING(".@avg.") {}; // these operators are normal strings (no proceeding string characters) struct count : string_token_t(".@count") {}; struct size : string_token_t(".@size") {}; +struct backlinks : string_token_t("@links") {}; struct single_collection_operators : sor< count, size > {}; struct key_collection_operators : sor< min, max, sum, avg > {}; // key paths -struct key_path : list< seq< sor< alpha, one< '_' > >, star< sor< alnum, one< '_', '-' > > > >, one< '.' > > {}; +struct key_path : list< sor< backlinks, seq< sor< alpha, one< '_', '$' > >, star< sor< alnum, one< '_', '-', '$' > > > > >, one< '.' > > {}; struct key_path_prefix : disable< key_path > {}; struct key_path_suffix : disable< key_path > {}; @@ -100,14 +107,31 @@ struct collection_operator_match : sor< seq< key_path_prefix, key_collection_ope // argument struct argument_index : plus< digit > {}; -struct argument : seq< one< '$' >, must< argument_index > > {}; +struct argument : seq< one< '$' >, argument_index > {}; + +// subquery eg: SUBQUERY(items, $x, $x.name CONTAINS 'a' && $x.price > 5).@count +struct subq_prefix : seq< string_token_t("subquery"), star< blank >, one< '(' > > {}; +struct subq_suffix : one< ')' > {}; +struct sub_path : disable < key_path > {}; +struct sub_var_name : seq < one< '$' >, seq< sor< alpha, one< '_', '$' > >, star< sor< alnum, one< '_', '-', '$' > > > > > {}; +struct sub_result_op : sor< count, size > {}; +struct sub_preamble : seq< subq_prefix, pad< sub_path, blank >, one< ',' >, pad< sub_var_name, blank >, one< ',' > > {}; +struct subquery : seq< sub_preamble, pad< pred, blank >, pad< subq_suffix, blank >, sub_result_op > {}; + +// list aggregate operations +struct agg_target : seq< key_path > {}; +struct agg_any : seq< sor< string_token_t("any"), string_token_t("some") >, plus, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {}; +struct agg_all : seq< string_token_t("all"), plus, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {}; +struct agg_none : seq< string_token_t("none"), plus, agg_target, pad< sor< string_oper, symbolic_oper >, blank >, expr > {}; +struct agg_shortcut_pred : sor< agg_any, agg_all, agg_none > {}; // expressions and operators -struct expr : sor< dq_string, sq_string, timestamp, number, argument, true_value, false_value, null_value, base64, collection_operator_match, key_path > {}; +struct expr : sor< dq_string, sq_string, timestamp, number, argument, true_value, false_value, null_value, base64, collection_operator_match, subquery, key_path > {}; struct case_insensitive : TAOCPP_PEGTL_ISTRING("[c]") {}; struct eq : seq< sor< two< '=' >, one< '=' > >, star< blank >, opt< case_insensitive > >{}; struct noteq : seq< sor< tao::pegtl::string< '!', '=' >, tao::pegtl::string< '<', '>' > >, star< blank >, opt< case_insensitive > > {}; +struct in: seq< string_token_t("in"), star< blank >, opt< case_insensitive > >{}; struct lteq : sor< tao::pegtl::string< '<', '=' >, tao::pegtl::string< '=', '<' > > {}; struct lt : one< '<' > {}; struct gteq : sor< tao::pegtl::string< '>', '=' >, tao::pegtl::string< '=', '>' > > {}; @@ -130,18 +154,22 @@ struct descriptor_ordering : sor< sort, distinct > {}; struct string_oper : seq< sor< contains, begins, ends, like>, star< blank >, opt< case_insensitive > > {}; // "=" is equality and since other operators can start with "=" we must check equal last -struct symbolic_oper : sor< noteq, lteq, lt, gteq, gt, eq > {}; +struct symbolic_oper : sor< noteq, lteq, lt, gteq, gt, eq, in > {}; // predicates struct comparison_pred : seq< expr, pad< sor< string_oper, symbolic_oper >, blank >, expr > {}; -struct pred; -struct group_pred : if_must< one< '(' >, pad< pred, blank >, one< ')' > > {}; +// we need to alias the group tokens because these are also used in other expressions above and we have to match +// the predicate group tokens without also matching () in other expressions. +struct begin_pred_group : one< '(' > {}; +struct end_pred_group : one< ')' > {}; + +struct group_pred : if_must< begin_pred_group, pad< pred, blank >, end_pred_group > {}; struct true_pred : string_token_t("truepredicate") {}; struct false_pred : string_token_t("falsepredicate") {}; struct not_pre : seq< sor< one< '!' >, string_token_t("not") > > {}; -struct atom_pred : seq< opt< not_pre >, pad< sor< group_pred, true_pred, false_pred, comparison_pred >, blank >, star< pad< descriptor_ordering, blank > > > {}; +struct atom_pred : seq< opt< not_pre >, pad< sor, blank >, star< pad< descriptor_ordering, blank > > > {}; struct and_op : pad< sor< two< '&' >, string_token_t("and") >, blank > {}; struct or_op : pad< sor< two< '|' >, string_token_t("or") >, blank > {}; @@ -161,6 +189,9 @@ struct ParserState Expression::KeyPathOp pending_op; DescriptorOrderingState ordering_state; DescriptorOrderingState::SingleOrderingState temp_ordering; + std::string subquery_path, subquery_var; + std::vector subqueries; + Predicate::ComparisonType pending_comparison_type; Predicate *current_group() { @@ -203,6 +234,12 @@ struct ParserState pending_op = Expression::KeyPathOp::None; } + void apply_list_aggregate_operation() + { + last_predicate()->cmpr.compare_type = pending_comparison_type; + pending_comparison_type = Predicate::ComparisonType::Unspecified; + } + void add_expression(Expression && exp) { Predicate *current = last_predicate(); @@ -429,6 +466,56 @@ template<> struct action< distinct > } }; +template<> struct action< sub_path > +{ + template< typename Input > + static void apply(const Input& in, ParserState & state) + { + DEBUG_PRINT_TOKEN(in.string() + " SUB PATH"); + state.subquery_path = in.string(); + } +}; + +template<> struct action< sub_var_name > +{ + template< typename Input > + static void apply(const Input& in, ParserState & state) + { + DEBUG_PRINT_TOKEN(in.string() + " SUB VAR NAME"); + state.subquery_var = in.string(); + } +}; + +// the preamble acts as the opening for a sub predicate group which is the subquery conditions +template<> struct action< sub_preamble > +{ + template< typename Input > + static void apply(const Input& in, ParserState & state) + { + DEBUG_PRINT_TOKEN(in.string() + ""); + + Expression exp(Expression::Type::SubQuery); + exp.subquery_path = state.subquery_path; + exp.subquery_var = state.subquery_var; + exp.subquery = std::make_shared(Predicate::Type::And); + REALM_ASSERT_DEBUG(!state.subquery_var.empty() && !state.subquery_path.empty()); + Predicate* sub_pred = exp.subquery.get(); + state.add_expression(std::move(exp)); + state.group_stack.push_back(sub_pred); + } +}; + + +// once the whole subquery syntax is matched, we close the subquery group and add the expression +template<> struct action< subquery > +{ + template< typename Input > + static void apply(const Input& in, ParserState & state) + { + DEBUG_PRINT_TOKEN(in.string() + ""); + state.group_stack.pop_back(); + } +}; #define COLLECTION_OPERATION_ACTION(rule, type) \ template<> struct action< rule > { \ @@ -469,6 +556,25 @@ template<> struct action< collection_operator_match > { } }; +#define LIST_AGG_OP_TYPE_ACTION(rule, type) \ +template<> struct action< rule > { \ +template< typename Input > \ + static void apply(const Input& in, ParserState& state) { \ + DEBUG_PRINT_TOKEN(in.string() + #rule); \ + state.pending_comparison_type = type; }}; + +LIST_AGG_OP_TYPE_ACTION(agg_any, Predicate::ComparisonType::Any) +LIST_AGG_OP_TYPE_ACTION(agg_all, Predicate::ComparisonType::All) +LIST_AGG_OP_TYPE_ACTION(agg_none, Predicate::ComparisonType::None) + +template<> struct action< agg_shortcut_pred > { + template< typename Input > + static void apply(const Input& in, ParserState& state) { + DEBUG_PRINT_TOKEN(in.string() + " Aggregate shortcut matched"); + state.apply_list_aggregate_operation(); + } +}; + template<> struct action< true_pred > { template< typename Input > @@ -493,11 +599,12 @@ template<> struct action< false_pred > template<> struct action< rule > { \ template< typename Input > \ static void apply(const Input& in, ParserState& state) { \ - DEBUG_PRINT_TOKEN(in.string()); \ + DEBUG_PRINT_TOKEN(in.string() + #oper); \ state.last_predicate()->cmpr.op = oper; }}; OPERATOR_ACTION(eq, Predicate::Operator::Equal) OPERATOR_ACTION(noteq, Predicate::Operator::NotEqual) +OPERATOR_ACTION(in, Predicate::Operator::In) OPERATOR_ACTION(gteq, Predicate::Operator::GreaterThanOrEqual) OPERATOR_ACTION(gt, Predicate::Operator::GreaterThan) OPERATOR_ACTION(lteq, Predicate::Operator::LessThanOrEqual) @@ -517,7 +624,7 @@ template<> struct action< case_insensitive > } }; -template<> struct action< one< '(' > > +template<> struct action< begin_pred_group > { template< typename Input > static void apply(const Input&, ParserState & state) diff --git a/src/realm/parser/parser.hpp b/src/realm/parser/parser.hpp index da8f7192ca7..abaf38fd3cd 100644 --- a/src/realm/parser/parser.hpp +++ b/src/realm/parser/parser.hpp @@ -19,19 +19,25 @@ #ifndef REALM_PARSER_HPP #define REALM_PARSER_HPP -#include +#include #include +#include namespace realm { namespace parser { + +struct Predicate; + struct Expression { - enum class Type { None, Number, String, KeyPath, Argument, True, False, Null, Timestamp, Base64 } type; + enum class Type { None, Number, String, KeyPath, Argument, True, False, Null, Timestamp, Base64, SubQuery } type; enum class KeyPathOp { None, Min, Max, Avg, Sum, Count, SizeString, SizeBinary } collection_op; std::string s; std::vector time_inputs; std::string op_suffix; + std::string subquery_path, subquery_var; + std::shared_ptr subquery; Expression(Type t = Type::None, std::string input = "") : type(t), collection_op(KeyPathOp::None), s(input) {} Expression(std::vector&& timestamp) : type(Type::Timestamp), collection_op(KeyPathOp::None), time_inputs(timestamp) {} Expression(std::string prefix, KeyPathOp op, std::string suffix) : type(Type::KeyPath), collection_op(op), s(prefix), op_suffix(suffix) {} @@ -60,7 +66,8 @@ struct Predicate BeginsWith, EndsWith, Contains, - Like + Like, + In }; enum class OperatorOption @@ -69,11 +76,20 @@ struct Predicate CaseInsensitive, }; + enum class ComparisonType + { + Unspecified, + Any, + All, + None, + }; + struct Comparison { Operator op = Operator::None; OperatorOption option = OperatorOption::None; Expression expr[2] = {{Expression::Type::None, ""}, {Expression::Type::None, ""}}; + ComparisonType compare_type = ComparisonType::Unspecified; }; struct Compound diff --git a/src/realm/parser/parser_utils.cpp b/src/realm/parser/parser_utils.cpp index 7f78f136faa..2363ebd3525 100644 --- a/src/realm/parser/parser_utils.cpp +++ b/src/realm/parser/parser_utils.cpp @@ -126,6 +126,21 @@ const char* collection_operator_to_str(parser::Expression::KeyPathOp op) return ""; } +const char* comparison_type_to_str(parser::Predicate::ComparisonType type) +{ + switch (type) { + case parser::Predicate::ComparisonType::Unspecified: + return ""; + case parser::Predicate::ComparisonType::All: + return "ALL"; + case parser::Predicate::ComparisonType::None: + return "NONE"; + case parser::Predicate::ComparisonType::Any: + return "ANY"; + } + return ""; +} + using KeyPath = std::vector; KeyPath key_path_from_string(const std::string &s) { @@ -138,9 +153,22 @@ KeyPath key_path_from_string(const std::string &s) { return key_path; } -StringData get_printable_table_name(const Table& table) +std::string key_path_to_string(const KeyPath& keypath) +{ + std::string path = ""; + for (size_t i = 0; i < keypath.size(); ++i) { + if (!keypath[i].empty()) { + path += keypath[i]; + if (i < keypath.size() - 1) { + path += serializer::value_separator; + } + } + } + return path; +} + +StringData get_printable_table_name(StringData name) { - StringData name = table.get_name(); // the "class_" prefix is an implementation detail of the object store that shouldn't be exposed to users static const std::string prefix = "class_"; if (name.size() > prefix.size() && strncmp(name.data(), prefix.data(), prefix.size()) == 0) { @@ -149,5 +177,11 @@ StringData get_printable_table_name(const Table& table) return name; } +StringData get_printable_table_name(const Table& table) +{ + StringData name = table.get_name(); + return get_printable_table_name(name); +} + } // namespace utils } // namespace realm diff --git a/src/realm/parser/parser_utils.hpp b/src/realm/parser/parser_utils.hpp index 7518eb19b99..12b79c99153 100644 --- a/src/realm/parser/parser_utils.hpp +++ b/src/realm/parser/parser_utils.hpp @@ -37,7 +37,7 @@ namespace util { // check a precondition and throw an exception if it is not met // this should be used iff the condition being false indicates a bug in the caller // of the function checking its preconditions -#define precondition(condition, message) if (!REALM_LIKELY(condition)) { throw std::logic_error(message); } +#define realm_precondition(condition, message) if (!REALM_LIKELY(condition)) { throw std::logic_error(message); } template @@ -61,11 +61,13 @@ template <> const char* type_to_str(); const char* data_type_to_str(DataType type); - const char* collection_operator_to_str(parser::Expression::KeyPathOp op); +const char* comparison_type_to_str(parser::Predicate::ComparisonType type); using KeyPath = std::vector; KeyPath key_path_from_string(const std::string &s); +std::string key_path_to_string(const KeyPath& keypath); +StringData get_printable_table_name(StringData name); StringData get_printable_table_name(const Table& table); template diff --git a/src/realm/parser/property_expression.cpp b/src/realm/parser/property_expression.cpp index 2d9befa0bb6..9576d98078f 100644 --- a/src/realm/parser/property_expression.cpp +++ b/src/realm/parser/property_expression.cpp @@ -26,40 +26,34 @@ namespace realm { namespace parser { -PropertyExpression::PropertyExpression(Query &q, const std::string &key_path_string) +PropertyExpression::PropertyExpression(Query &q, const std::string &key_path_string, parser::KeyPathMapping& mapping) : query(q) { + ConstTableRef cur_table = query.get_table(); KeyPath key_path = key_path_from_string(key_path_string); - TableRef cur_table = query.get_table(); - for (size_t index = 0; index < key_path.size(); index++) { - size_t cur_col_ndx = cur_table->get_column_index(key_path[index]); - - StringData object_name = get_printable_table_name(*cur_table); - - precondition(cur_col_ndx != realm::not_found, - util::format("No property '%1' on object of type '%2'", key_path[index], object_name)); - - DataType cur_col_type = cur_table->get_column_type(cur_col_ndx); - if (index != key_path.size() - 1) { - precondition(cur_col_type == type_Link || cur_col_type == type_LinkList, - util::format("Property '%1' is not a link in object of type '%2'", key_path[index], object_name)); - indexes.push_back(cur_col_ndx); - cur_table = cur_table->get_link_target(cur_col_ndx); - } - else { - col_ndx = cur_col_ndx; - col_type = cur_col_type; + size_t index = 0; + + while (index < key_path.size()) { + KeyPathElement element = mapping.process_next_path(cur_table, key_path, index); + if (index != key_path.size()) { + realm_precondition(element.col_type == type_Link || element.col_type == type_LinkList, + util::format("Property '%1' is not a link in object of type '%2'", + element.table->get_column_name(element.col_ndx), + get_printable_table_name(*element.table))); + if (element.table == cur_table) { + cur_table = element.table->get_link_target(element.col_ndx); // advance through forward link + } else { + cur_table = element.table; // advance through backlink + } } + link_chain.push_back(element); } } Table* PropertyExpression::table_getter() const { auto& tbl = query.get_table(); - for (size_t col : indexes) { - tbl->link(col); // mutates m_link_chain on table - } - return tbl.get(); + return KeyPathMapping::table_getter(tbl, link_chain); } } // namespace parser diff --git a/src/realm/parser/property_expression.hpp b/src/realm/parser/property_expression.hpp index e087a96e485..8f4a7ec0623 100644 --- a/src/realm/parser/property_expression.hpp +++ b/src/realm/parser/property_expression.hpp @@ -19,6 +19,7 @@ #ifndef REALM_PROPERTY_EXPRESSION_HPP #define REALM_PROPERTY_EXPRESSION_HPP +#include #include #include @@ -27,22 +28,49 @@ namespace parser { struct PropertyExpression { - std::vector indexes; - size_t col_ndx; - DataType col_type; Query &query; + std::vector link_chain; + DataType get_dest_type() const; + size_t get_dest_ndx() const; + ConstTableRef get_dest_table() const; + bool dest_type_is_backlink() const; - PropertyExpression(Query &q, const std::string &key_path_string); + PropertyExpression(Query &q, const std::string &key_path_string, parser::KeyPathMapping& mapping); Table* table_getter() const; template auto value_of_type_for_query() const { - return this->table_getter()->template column(this->col_ndx); + return this->table_getter()->template column(get_dest_ndx()); } }; +inline DataType PropertyExpression::get_dest_type() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().col_type; +} + +inline bool PropertyExpression::dest_type_is_backlink() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().is_backlink; +} + +inline size_t PropertyExpression::get_dest_ndx() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().col_ndx; +} + +inline ConstTableRef PropertyExpression::get_dest_table() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().table; +} + + } // namespace parser } // namespace realm diff --git a/src/realm/parser/query_builder.cpp b/src/realm/parser/query_builder.cpp index 292c42fdabd..913975ff80e 100644 --- a/src/realm/parser/query_builder.cpp +++ b/src/realm/parser/query_builder.cpp @@ -51,11 +51,13 @@ void do_add_null_comparison_to_query(Query &, Predicate::Operator, const ValueEx template void do_add_null_comparison_to_query(Query &query, Predicate::Operator op, const PropertyExpression &expr) { - Columns column = expr.table_getter()->template column(expr.col_ndx); + Columns column = expr.table_getter()->template column(expr.get_dest_ndx()); switch (op) { case Predicate::Operator::NotEqual: query.and_query(column != realm::null()); break; + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: query.and_query(column == realm::null()); break; @@ -67,13 +69,15 @@ void do_add_null_comparison_to_query(Query &query, Predicate::Operator op, const template<> void do_add_null_comparison_to_query(Query &query, Predicate::Operator op, const PropertyExpression &expr) { - precondition(expr.indexes.empty(), "KeyPath queries not supported for object comparisons."); + realm_precondition(expr.link_chain.size() == 1, "KeyPath queries not supported for object comparisons."); switch (op) { case Predicate::Operator::NotEqual: query.Not(); REALM_FALLTHROUGH; + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: - query.and_query(query.get_table()->column(expr.col_ndx).is_null()); + query.and_query(query.get_table()->column(expr.get_dest_ndx()).is_null()); break; default: throw std::logic_error("Only 'equal' and 'not equal' operators supported for object comparison."); @@ -100,6 +104,8 @@ void add_numeric_constraint_to_query(Query& query, case Predicate::Operator::GreaterThanOrEqual: query.and_query(lhs >= rhs); break; + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: query.and_query(lhs == rhs); break; @@ -114,6 +120,8 @@ void add_numeric_constraint_to_query(Query& query, template void add_bool_constraint_to_query(Query &query, Predicate::Operator operatorType, A lhs, B rhs) { switch (operatorType) { + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: query.and_query(lhs == rhs); break; @@ -160,6 +168,8 @@ void add_string_constraint_to_query(realm::Query &query, Columns &&column) { bool case_sensitive = (cmp.option != Predicate::OperatorOption::CaseInsensitive); switch (cmp.op) { + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: query.and_query(column.equal(value, case_sensitive)); break; @@ -236,6 +246,8 @@ void add_binary_constraint_to_query(realm::Query &query, BinaryData &&value, Columns &&column) { switch (cmp.op) { + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: query.and_query(column == value); break; @@ -288,13 +300,15 @@ void add_link_constraint_to_query(realm::Query &query, const PropertyExpression &prop_expr, const ValueExpression &value_expr) { size_t row_index = value_expr.arguments->object_index_for_argument(stot(value_expr.value->s)); - precondition(prop_expr.indexes.empty(), "KeyPath queries not supported for object comparisons."); + realm_precondition(prop_expr.link_chain.size() == 1, "KeyPath queries not supported for object comparisons."); switch (op) { case Predicate::Operator::NotEqual: query.Not(); REALM_FALLTHROUGH; + case Predicate::Operator::In: + REALM_FALLTHROUGH; case Predicate::Operator::Equal: { - size_t col = prop_expr.col_ndx; + size_t col = prop_expr.get_dest_ndx(); query.links_to(col, query.get_table()->get_link_target(col)->get(row_index)); break; } @@ -375,8 +389,8 @@ enum class NullLocation { template void do_add_null_comparison_to_query(Query &query, Predicate::Comparison cmp, const T &expr, DataType type, NullLocation location) { - if (type == type_LinkList) { // when backlinks are supported, this should check those as well - throw std::logic_error("Comparing Lists to 'null' is not supported"); + if (type == type_LinkList) { // this handles backlinks as well since they are set to type LinkList + throw std::logic_error("Comparing a list property to 'null' is not supported"); } switch (type) { case realm::type_Bool: @@ -426,7 +440,7 @@ void add_null_comparison_to_query(Query &query, Predicate::Comparison cmp, Expre case ExpressionContainer::ExpressionInternal::exp_Value: throw std::runtime_error("Unsupported query comparing 'null' and a literal. A comparison must include at least one keypath."); case ExpressionContainer::ExpressionInternal::exp_Property: - do_add_null_comparison_to_query(query, cmp, exp.get_property(), exp.get_property().col_type, location); + do_add_null_comparison_to_query(query, cmp, exp.get_property(), exp.get_property().get_dest_type(), location); break; case ExpressionContainer::ExpressionInternal::exp_OpMin: do_add_null_comparison_to_query(query, cmp, exp.get_min(), exp.get_min().post_link_col_type, location); @@ -440,6 +454,8 @@ void add_null_comparison_to_query(Query &query, Predicate::Comparison cmp, Expre case ExpressionContainer::ExpressionInternal::exp_OpAvg: do_add_null_comparison_to_query(query, cmp, exp.get_avg(), exp.get_avg().post_link_col_type, location); break; + case ExpressionContainer::ExpressionInternal::exp_SubQuery: + REALM_FALLTHROUGH; case ExpressionContainer::ExpressionInternal::exp_OpCount: REALM_FALLTHROUGH; case ExpressionContainer::ExpressionInternal::exp_OpSizeString: @@ -480,6 +496,9 @@ void internal_add_comparison_to_query(Query& query, LHS_T& lhs, Predicate::Compa case ExpressionContainer::ExpressionInternal::exp_OpSizeBinary: do_add_comparison_to_query(query, cmp, lhs, rhs.get_size_binary(), comparison_type); return; + case realm::parser::ExpressionContainer::ExpressionInternal::exp_SubQuery: + do_add_comparison_to_query(query, cmp, lhs, rhs.get_subexpression(), comparison_type); + return; } } @@ -514,21 +533,147 @@ void add_comparison_to_query(Query &query, ExpressionContainer& lhs, Predicate:: case ExpressionContainer::ExpressionInternal::exp_OpSizeBinary: internal_add_comparison_to_query(query, lhs.get_size_binary(), cmp, rhs, comparison_type); return; + case realm::parser::ExpressionContainer::ExpressionInternal::exp_SubQuery: + internal_add_comparison_to_query(query, lhs.get_subexpression(), cmp, rhs, comparison_type); + return; } } -void add_comparison_to_query(Query &query, const Predicate &pred, Arguments &args) + +std::pair separate_list_parts(PropertyExpression& pe) { + std::string pre_and_list = ""; + std::string post_list = ""; + bool past_list = false; + for (KeyPathElement& e : pe.link_chain) { + std::string cur_name; + if (e.is_backlink) { + cur_name = std::string("@links") + util::serializer::value_separator + std::string(e.table->get_name()) + util::serializer::value_separator + std::string(e.table->get_column_name(e.col_ndx)); + } else { + cur_name = e.table->get_column_name(e.col_ndx); + } + if (!past_list) { + if (!pre_and_list.empty()) { + pre_and_list += util::serializer::value_separator; + } + pre_and_list += cur_name; + } else { + realm_precondition(!e.is_backlink && e.col_type != type_LinkList, util::format("The keypath after '%1' must not contain any additional list properties, but '%2' is a list.", pre_and_list, cur_name)); + if (!post_list.empty()) { + post_list += util::serializer::value_separator; + } + post_list += cur_name; + } + if (e.is_backlink || e.col_type == type_LinkList) { + past_list = true; + } + } + return {pre_and_list, post_list}; +} + + +// some query types are not supported in core but can be produced by a transformation: +// "ALL path.to.list.property >= 20" --> "SUBQUERY(path.to.list, $x, $x.property >= 20).@count == path.to.list.@count" +// "NONE path.to.list.property >= 20" --> "SUBQUERY(path.to.list, $x, $x.property >= 20).@count == 0" +void preprocess_for_comparison_types(Query &query, Predicate::Comparison &cmpr, ExpressionContainer &lhs, ExpressionContainer &rhs, Arguments &args, parser::KeyPathMapping& mapping) { - const Predicate::Comparison &cmpr = pred.cmpr; + auto get_cmp_type_name = [&]() { + if (cmpr.compare_type == Predicate::ComparisonType::Any) { + return util::format("'%1' or 'SOME'", comparison_type_to_str(Predicate::ComparisonType::Any)); + } + return util::format("'%1'", comparison_type_to_str(cmpr.compare_type)); + }; + + if (cmpr.compare_type != Predicate::ComparisonType::Unspecified) + { + realm_precondition(lhs.type == ExpressionContainer::ExpressionInternal::exp_Property, + util::format("The expression after %1 must be a keypath containing a list", + get_cmp_type_name())); + size_t list_count = 0; + for (KeyPathElement e : lhs.get_property().link_chain) { + if (e.col_type == type_LinkList || e.is_backlink) { + list_count++; + } + } + realm_precondition(list_count > 0, util::format("The keypath following %1 must contain a list", + get_cmp_type_name())); + realm_precondition(list_count == 1, util::format("The keypath following %1 must contain only one list", + get_cmp_type_name())); + } + + if (cmpr.compare_type == Predicate::ComparisonType::All || cmpr.compare_type == Predicate::ComparisonType::None) { + realm_precondition(rhs.type == ExpressionContainer::ExpressionInternal::exp_Value, + util::format("The comparison in an %1 clause must be between a keypath and a value", + get_cmp_type_name())); + + parser::Expression exp(parser::Expression::Type::SubQuery); + std::pair path_parts = separate_list_parts(lhs.get_property()); + exp.subquery_path = path_parts.first; + + util::serializer::SerialisationState temp_state; + std::string var_name; + for (var_name = temp_state.get_variable_name(query.get_table()); + mapping.has_mapping(query.get_table(), var_name); + var_name = temp_state.get_variable_name(query.get_table())) { + temp_state.subquery_prefix_list.push_back(var_name); + } + + exp.subquery_var = var_name; + exp.subquery = std::make_shared(Predicate::Type::Comparison); + exp.subquery->cmpr.expr[0] = parser::Expression(parser::Expression::Type::KeyPath, var_name + util::serializer::value_separator + path_parts.second); + exp.subquery->cmpr.op = cmpr.op; + exp.subquery->cmpr.option = cmpr.option; + exp.subquery->cmpr.expr[1] = cmpr.expr[1]; + cmpr.expr[0] = exp; + + lhs = ExpressionContainer(query, cmpr.expr[0], args, mapping); + cmpr.op = parser::Predicate::Operator::Equal; + cmpr.option = parser::Predicate::OperatorOption::None; + + if (cmpr.compare_type == Predicate::ComparisonType::All) { + cmpr.expr[1] = parser::Expression(path_parts.first, parser::Expression::KeyPathOp::Count, ""); + rhs = ExpressionContainer(query, cmpr.expr[1], args, mapping); + } else if (cmpr.compare_type == Predicate::ComparisonType::None) { + cmpr.expr[1] = parser::Expression(parser::Expression::Type::Number, "0"); + rhs = ExpressionContainer(query, cmpr.expr[1], args, mapping); + } else { + REALM_UNREACHABLE(); + } + } + + // Check that operator "IN" has a RHS keypath which is a list + if (cmpr.op == Predicate::Operator::In) { + realm_precondition(rhs.type == ExpressionContainer::ExpressionInternal::exp_Property, + "The expression following 'IN' must be a keypath to a list"); + size_t list_count = 0; + for (KeyPathElement e : rhs.get_property().link_chain) { + if (e.col_type == type_LinkList || e.is_backlink) { + list_count++; + } + } + realm_precondition(list_count > 0, "The keypath following 'IN' must contain a list"); + realm_precondition(list_count == 1, "The keypath following 'IN' must contain only one list"); + } +} + + +bool is_property_operation(parser::Expression::Type type) +{ + return type == parser::Expression::Type::KeyPath || type == parser::Expression::Type::SubQuery; +} + +void add_comparison_to_query(Query &query, const Predicate &pred, Arguments &args, parser::KeyPathMapping& mapping) +{ + Predicate::Comparison cmpr = pred.cmpr; auto lhs_type = cmpr.expr[0].type, rhs_type = cmpr.expr[1].type; - if (lhs_type != parser::Expression::Type::KeyPath && rhs_type != parser::Expression::Type::KeyPath) { + if (!is_property_operation(lhs_type) && !is_property_operation(rhs_type)) { // value vs value expressions are not supported (ex: 2 < 3 or null != null) throw std::logic_error("Predicate expressions must compare a keypath and another keypath or a constant value"); } + ExpressionContainer lhs(query, cmpr.expr[0], args, mapping); + ExpressionContainer rhs(query, cmpr.expr[1], args, mapping); - ExpressionContainer lhs(query, cmpr.expr[0], args); - ExpressionContainer rhs(query, cmpr.expr[1], args); + preprocess_for_comparison_types(query, cmpr, lhs, rhs, args, mapping); if (lhs.is_null()) { add_null_comparison_to_query(query, cmpr, rhs, NullLocation::NullOnLHS); @@ -541,7 +686,7 @@ void add_comparison_to_query(Query &query, const Predicate &pred, Arguments &arg } } -void update_query_with_predicate(Query &query, const Predicate &pred, Arguments &arguments) +void update_query_with_predicate(Query &query, const Predicate &pred, Arguments &arguments, parser::KeyPathMapping& mapping) { if (pred.negate) { query.Not(); @@ -551,7 +696,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments case Predicate::Type::And: query.group(); for (auto &sub : pred.cpnd.sub_predicates) { - update_query_with_predicate(query, sub, arguments); + update_query_with_predicate(query, sub, arguments, mapping); } if (!pred.cpnd.sub_predicates.size()) { query.and_query(std::unique_ptr(new TrueExpression)); @@ -563,7 +708,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments query.group(); for (auto &sub : pred.cpnd.sub_predicates) { query.Or(); - update_query_with_predicate(query, sub, arguments); + update_query_with_predicate(query, sub, arguments, mapping); } if (!pred.cpnd.sub_predicates.size()) { query.and_query(std::unique_ptr(new FalseExpression)); @@ -572,7 +717,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments break; case Predicate::Type::Comparison: { - add_comparison_to_query(query, pred, arguments); + add_comparison_to_query(query, pred, arguments, mapping); break; } case Predicate::Type::True: @@ -592,7 +737,7 @@ void update_query_with_predicate(Query &query, const Predicate &pred, Arguments namespace realm { namespace query_builder { -void apply_predicate(Query &query, const Predicate &predicate, Arguments &arguments) +void apply_predicate(Query &query, const Predicate &predicate, Arguments &arguments, parser::KeyPathMapping mapping) { if (predicate.type == Predicate::Type::True && !predicate.negate) { @@ -600,34 +745,14 @@ void apply_predicate(Query &query, const Predicate &predicate, Arguments &argume return; } - update_query_with_predicate(query, predicate, arguments); + update_query_with_predicate(query, predicate, arguments, mapping); // Test the constructed query in core std::string validateMessage = query.validate(); - precondition(validateMessage.empty(), validateMessage.c_str()); + realm_precondition(validateMessage.empty(), validateMessage.c_str()); } -struct EmptyArgContext -{ - template - T unbox(std::string) { - return T{}; //dummy - } - bool is_null(std::string) { - return false; - } -}; - -void apply_predicate(Query &query, const Predicate &predicate) -{ - EmptyArgContext ctx; - std::string empty_string; - realm::query_builder::ArgumentConverter args(ctx, &empty_string, 0); - - apply_predicate(query, predicate, args); -} - -void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser::DescriptorOrderingState& state, Arguments&) +void apply_ordering(DescriptorOrdering& ordering, ConstTableRef target, const parser::DescriptorOrderingState& state, Arguments&) { for (const DescriptorOrderingState::SingleOrderingState& cur_ordering : state.orderings) { std::vector> property_indices; @@ -635,7 +760,7 @@ void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser: for (const DescriptorOrderingState::PropertyState& cur_property : cur_ordering.properties) { KeyPath path = key_path_from_string(cur_property.key_path); std::vector indices; - TableRef cur_table = target; + ConstTableRef cur_table = target; for (size_t ndx_in_path = 0; ndx_in_path < path.size(); ++ndx_in_path) { size_t col_ndx = cur_table->get_column_index(path[ndx_in_path]); if (col_ndx == realm::not_found) { @@ -660,12 +785,9 @@ void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser: } } -void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser::DescriptorOrderingState& state) +void apply_ordering(DescriptorOrdering& ordering, ConstTableRef target, const parser::DescriptorOrderingState& state) { - EmptyArgContext ctx; - std::string empty_string; - realm::query_builder::ArgumentConverter args(ctx, &empty_string, 0); - + NoArguments args; apply_ordering(ordering, target, state, args); } diff --git a/src/realm/parser/query_builder.hpp b/src/realm/parser/query_builder.hpp index 1fb628b446f..db05f9261b1 100644 --- a/src/realm/parser/query_builder.hpp +++ b/src/realm/parser/query_builder.hpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -46,11 +47,10 @@ namespace parser { namespace query_builder { class Arguments; -void apply_predicate(Query& query, const parser::Predicate& predicate, Arguments& arguments); -void apply_predicate(Query& query, const parser::Predicate& predicate); // zero out of string args version +void apply_predicate(Query& query, const parser::Predicate& predicate, Arguments& arguments, parser::KeyPathMapping = parser::KeyPathMapping()); -void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser::DescriptorOrderingState& state, Arguments& arguments); -void apply_ordering(DescriptorOrdering& ordering, TableRef target, const parser::DescriptorOrderingState& state); +void apply_ordering(DescriptorOrdering& ordering, ConstTableRef target, const parser::DescriptorOrderingState& state, Arguments& arguments); +void apply_ordering(DescriptorOrdering& ordering, ConstTableRef target, const parser::DescriptorOrderingState& state); struct AnyContext @@ -81,7 +81,9 @@ class Arguments { virtual Timestamp timestamp_for_argument(size_t argument_index) = 0; virtual size_t object_index_for_argument(size_t argument_index) = 0; virtual bool is_argument_null(size_t argument_index) = 0; - util::StringBuffer buffer_space; // dynamic conversion space with lifetime tied to this + // dynamic conversion space with lifetime tied to this + // it is used for storing literal binary/string data + std::vector buffer_space; }; template @@ -121,6 +123,25 @@ class ArgumentConverter : public Arguments { return m_ctx.template unbox(at(index)); } }; + +class NoArgsError : public std::runtime_error { +public: + NoArgsError() : std::runtime_error("Attempt to retreive an argument when no arguments were given") {} +}; + +class NoArguments : public Arguments { +public: + bool bool_for_argument(size_t) { throw NoArgsError(); } + long long long_for_argument(size_t) { throw NoArgsError(); } + float float_for_argument(size_t) { throw NoArgsError(); } + double double_for_argument(size_t) { throw NoArgsError(); } + StringData string_for_argument(size_t) { throw NoArgsError(); } + BinaryData binary_for_argument(size_t) { throw NoArgsError(); } + Timestamp timestamp_for_argument(size_t) { throw NoArgsError(); } + size_t object_index_for_argument(size_t) { throw NoArgsError(); } + bool is_argument_null(size_t) { throw NoArgsError(); } +}; + } // namespace query_builder } // namespace realm diff --git a/src/realm/parser/subquery_expression.cpp b/src/realm/parser/subquery_expression.cpp new file mode 100644 index 00000000000..8675d61c065 --- /dev/null +++ b/src/realm/parser/subquery_expression.cpp @@ -0,0 +1,85 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#include "subquery_expression.hpp" +#include "parser_utils.hpp" + +#include + +#include + +namespace realm { +namespace parser { + +SubqueryExpression::SubqueryExpression(Query &q, const std::string &key_path_string, const std::string &variable_name, parser::KeyPathMapping &mapping) +: var_name(variable_name), query(q) +{ + ConstTableRef cur_table = query.get_table(); + KeyPath key_path = key_path_from_string(key_path_string); + size_t index = 0; + + while (index < key_path.size()) { + KeyPathElement element = mapping.process_next_path(cur_table, key_path, index); + if (index != key_path.size()) { + realm_precondition(element.col_type == type_Link || element.col_type == type_LinkList, + util::format("Property '%1' is not a link in object of type '%2'", + element.table->get_column_name(element.col_ndx), + get_printable_table_name(*element.table))); + if (element.is_backlink) { + cur_table = element.table; // advance through backlink + } else { + cur_table = cur_table->get_link_target(element.col_ndx); // advance through forward link + } + } + else { + StringData dest_type; + if (element.is_backlink) { + dest_type = "linking object"; + } else { + dest_type = data_type_to_str(element.col_type); + } + realm_precondition(element.col_type == type_LinkList, + util::format("A subquery must operate on a list property, but '%1' is type '%2'", + element.table->get_column_name(element.col_ndx), + dest_type)); + ConstTableRef subquery_table; + if (element.is_backlink) { + subquery_table = element.table; // advance through backlink + } else { + subquery_table = cur_table->get_link_target(element.col_ndx); // advance through forward link + } + + subquery = subquery_table->where(); + } + link_chain.push_back(element); + } +} + +Query& SubqueryExpression::get_subquery() +{ + return subquery; +} + +Table* SubqueryExpression::table_getter() const +{ + auto& tbl = query.get_table(); + return KeyPathMapping::table_getter(tbl, link_chain); +} + +} // namespace parser +} // namespace realm diff --git a/src/realm/parser/subquery_expression.hpp b/src/realm/parser/subquery_expression.hpp new file mode 100644 index 00000000000..20c04d161db --- /dev/null +++ b/src/realm/parser/subquery_expression.hpp @@ -0,0 +1,112 @@ +//////////////////////////////////////////////////////////////////////////// +// +// Copyright 2015 Realm Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////// + +#ifndef REALM_SUBQUERY_EXPRESSION_HPP +#define REALM_SUBQUERY_EXPRESSION_HPP + +#include +#include +#include +#include + +#include "parser_utils.hpp" + +namespace realm { +namespace parser { + +template +struct SubqueryGetter; + +struct SubqueryExpression +{ + std::string var_name; + Query &query; + Query subquery; + std::vector link_chain; + DataType get_dest_type() const; + size_t get_dest_ndx() const; + ConstTableRef get_dest_table() const; + bool dest_type_is_backlink() const; + + + SubqueryExpression(Query &q, const std::string &key_path_string, const std::string &variable_name, parser::KeyPathMapping &mapping); + Query& get_subquery(); + + Table* table_getter() const; + + template + auto value_of_type_for_query() const + { + return SubqueryGetter::convert(*this); + } +}; + +inline DataType SubqueryExpression::get_dest_type() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().col_type; +} + +inline bool SubqueryExpression::dest_type_is_backlink() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().is_backlink; +} + +inline size_t SubqueryExpression::get_dest_ndx() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().col_ndx; +} + +inline ConstTableRef SubqueryExpression::get_dest_table() const +{ + REALM_ASSERT_DEBUG(link_chain.size() > 0); + return link_chain.back().table; +} + +// Certain operations are disabled for some types (eg. a sum of timestamps is invalid). +// The operations that are supported have a specialisation with std::enable_if for that type below +// any type/operation combination that is not specialised will get the runtime error from the following +// default implementation. The return type is just a dummy to make things compile. +template +struct SubqueryGetter { + static Columns convert(const SubqueryExpression&) { + throw std::runtime_error(util::format("Predicate error: comparison of type '%1' with result of a subquery count is not supported.", + type_to_str())); + } +}; + +template +struct SubqueryGetter::value> >{ + static SubQueryCount convert(const SubqueryExpression& expr) + { + if (expr.dest_type_is_backlink()) { + return expr.table_getter()->template column(*expr.get_dest_table(), expr.get_dest_ndx(), expr.subquery).count(); + } else { + return expr.table_getter()->template column(expr.get_dest_ndx(), expr.subquery).count(); + } + } +}; + +} // namespace parser +} // namespace realm + +#endif // REALM_SUBQUERY_EXPRESSION_HPP + diff --git a/src/realm/parser/value_expression.cpp b/src/realm/parser/value_expression.cpp index 00c8727e8e4..4615929bb78 100644 --- a/src/realm/parser/value_expression.cpp +++ b/src/realm/parser/value_expression.cpp @@ -189,11 +189,14 @@ StringData ValueExpression::value_of_type_for_query() return arguments->string_for_argument(stot(value->s)); } else if (value->type == parser::Expression::Type::String) { - return value->s; + arguments->buffer_space.push_back({}); + arguments->buffer_space.back().append(value->s); + return StringData(arguments->buffer_space.back().data(), arguments->buffer_space.back().size()); } else if (value->type == parser::Expression::Type::Base64) { // the return value points to data in the lifetime of args - return from_base64(value->s, arguments->buffer_space); + arguments->buffer_space.push_back({}); + return from_base64(value->s, arguments->buffer_space.back()); } throw std::logic_error("Attempting to compare String property to a non-String value"); } @@ -205,10 +208,14 @@ BinaryData ValueExpression::value_of_type_for_query() return arguments->binary_for_argument(stot(value->s)); } else if (value->type == parser::Expression::Type::String) { - return BinaryData(value->s); + arguments->buffer_space.push_back({}); + arguments->buffer_space.back().append(value->s); + return BinaryData(arguments->buffer_space.back().data(), arguments->buffer_space.back().size()); } else if (value->type == parser::Expression::Type::Base64) { - StringData converted = from_base64(value->s, arguments->buffer_space); + // the return value points to data in the lifetime of args + arguments->buffer_space.push_back({}); + StringData converted = from_base64(value->s, arguments->buffer_space[arguments->buffer_space.size() - 1]); // returning a pointer to data in the lifetime of args return BinaryData(converted.data(), converted.size()); } diff --git a/src/realm/query.cpp b/src/realm/query.cpp index f232b0090fe..85241d7041b 100644 --- a/src/realm/query.cpp +++ b/src/realm/query.cpp @@ -1599,19 +1599,25 @@ std::string Query::validate() return root_node()->validate(); // errors detected by QueryEngine } -std::string Query::get_description() const +std::string Query::get_description(util::serializer::SerialisationState& state) const { if (root_node()) { if (m_view) { throw SerialisationError("Serialisation of a query constrianed by a view is not currently supported"); } - return root_node()->describe_expression(); + return root_node()->describe_expression(state); } // An empty query returns all results and one way to indicate this // is to serialise TRUEPREDICATE which is functionally equivilent return "TRUEPREDICATE"; } +std::string Query::get_description() const +{ + util::serializer::SerialisationState state; + return get_description(state); +} + void Query::init() const { REALM_ASSERT(m_table); diff --git a/src/realm/query.hpp b/src/realm/query.hpp index ce8dac753fc..9b1ad07123c 100644 --- a/src/realm/query.hpp +++ b/src/realm/query.hpp @@ -42,6 +42,7 @@ #include #include #include +#include namespace realm { @@ -343,6 +344,7 @@ class Query final { std::string validate(); std::string get_description() const; + std::string get_description(util::serializer::SerialisationState& state) const; private: Query(Table& table, TableViewBase* tv = nullptr); diff --git a/src/realm/query_engine.cpp b/src/realm/query_engine.cpp index 6fdc9b064ce..d1771fa1479 100644 --- a/src/realm/query_engine.cpp +++ b/src/realm/query_engine.cpp @@ -544,10 +544,10 @@ void ExpressionNode::table_changed() m_expression->set_base_table(m_table.get()); } -std::string ExpressionNode::describe() const +std::string ExpressionNode::describe(util::serializer::SerialisationState& state) const { if (m_expression) { - return m_expression->description(); + return m_expression->description(state); } else { return "empty expression"; diff --git a/src/realm/query_engine.hpp b/src/realm/query_engine.hpp index 1d6583f0c24..3bf2949f521 100644 --- a/src/realm/query_engine.hpp +++ b/src/realm/query_engine.hpp @@ -268,20 +268,7 @@ class ParentNode { virtual void verify_column() const = 0; - virtual std::string describe_column() const - { - return describe_column(m_condition_column_idx); - } - - virtual std::string describe_column(size_t col_ndx) const - { - if (m_table && col_ndx != npos) { - return std::string(m_table->get_column_name(col_ndx)); - } - return ""; - } - - virtual std::string describe() const + virtual std::string describe(util::serializer::SerialisationState&) const { return ""; } @@ -291,12 +278,12 @@ class ParentNode { return "matches"; } - virtual std::string describe_expression() const + virtual std::string describe_expression(util::serializer::SerialisationState& state) const { std::string s; - s = describe(); + s = describe(state); if (m_child) { - s = s + " and " + m_child->describe_expression(); + s = s + " and " + m_child->describe_expression(state); } return s; } @@ -413,13 +400,7 @@ class SubtableNode : public ParentNode { return m_condition->validate(); } - virtual std::string describe_column() const override - { - REALM_ASSERT(m_column != nullptr); - return ParentNode::describe_column(m_column->get_column_index()); - } - - std::string describe() const override + std::string describe(util::serializer::SerialisationState&) const override { throw SerialisationError("Serialising a query which contains a subtable expression is currently unsupported."); } @@ -684,12 +665,6 @@ class IntegerNodeBase : public ColumnNodeBase { do_verify_column(m_condition_column); } - virtual std::string describe_column() const override - { - REALM_ASSERT(m_condition_column != nullptr); - return ParentNode::describe_column(m_condition_column->get_column_index()); - } - void init() override { ColumnNodeBase::init(); @@ -819,9 +794,10 @@ class IntegerNode : public IntegerNodeBase { return not_found; } - virtual std::string describe() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { - return this->describe_column() + " " + describe_condition() + " " + util::serializer::print_value(IntegerNodeBase::m_value); + return state.describe_column(ParentNode::m_table, IntegerNodeBase::m_condition_column->get_column_index()) + + " " + describe_condition() + " " + util::serializer::print_value(IntegerNodeBase::m_value); } virtual std::string describe_condition() const override @@ -957,15 +933,11 @@ class FloatDoubleNode : public ParentNode { return find(false); } - virtual std::string describe_column() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { REALM_ASSERT(m_condition_column.m_column != nullptr); - return ParentNode::describe_column(m_condition_column.m_column->get_column_index()); - } - - virtual std::string describe() const override - { - return describe_column() + " " + describe_condition() + " " + util::serializer::print_value(FloatDoubleNode::m_value); + return state.describe_column(ParentNode::m_table, m_condition_column.m_column->get_column_index()) + + " " + describe_condition() + " " + util::serializer::print_value(FloatDoubleNode::m_value); } virtual std::string describe_condition() const override { @@ -1096,15 +1068,11 @@ class BinaryNode : public ParentNode { return not_found; } - virtual std::string describe_column() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { REALM_ASSERT(m_condition_column != nullptr); - return ParentNode::describe_column(m_condition_column->get_column_index()); - } - - virtual std::string describe() const override - { - return describe_column() + " " + TConditionFunction::description() + " " + return state.describe_column(ParentNode::m_table, m_condition_column->get_column_index()) + + " " + TConditionFunction::description() + " " + util::serializer::print_value(BinaryNode::m_value.get()); } @@ -1168,15 +1136,11 @@ class TimestampNode : public ParentNode { return ret; } - virtual std::string describe_column() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { REALM_ASSERT(m_condition_column != nullptr); - return ParentNode::describe_column(m_condition_column->get_column_index()); - } - - virtual std::string describe() const override - { - return describe_column() + " " + TConditionFunction::description() + " " + util::serializer::print_value(TimestampNode::m_value); + return state.describe_column(ParentNode::m_table, m_condition_column->get_column_index()) + + " " + TConditionFunction::description() + " " + util::serializer::print_value(TimestampNode::m_value); } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override @@ -1248,19 +1212,15 @@ class StringNodeBase : public ParentNode { m_condition_column_idx = m_condition_column->get_column_index(); } - virtual std::string describe_column() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { REALM_ASSERT(m_condition_column != nullptr); - return ParentNode::describe_column(m_condition_column->get_column_index()); - } - - virtual std::string describe() const override - { StringData sd; if (bool(StringNodeBase::m_value)) { sd = StringData(StringNodeBase::m_value.value()); } - return this->describe_column() + " " + describe_condition() + " " + util::serializer::print_value(sd); + return state.describe_column(ParentNode::m_table, m_condition_column->get_column_index()) + + " " + describe_condition() + " " + util::serializer::print_value(sd); } protected: @@ -1681,7 +1641,7 @@ class OrNode : public ParentNode { condition->verify_column(); } } - std::string describe() const override + std::string describe(util::serializer::SerialisationState& state) const override { if (m_conditions.size() >= 2) { @@ -1689,7 +1649,7 @@ class OrNode : public ParentNode { std::string s; for (size_t i = 0; i < m_conditions.size(); ++i) { if (m_conditions[i]) { - s += m_conditions[i]->describe_expression(); + s += m_conditions[i]->describe_expression(state); if (i != m_conditions.size() - 1) { s += " or "; } @@ -1861,10 +1821,10 @@ class NotNode : public ParentNode { return ""; } - virtual std::string describe() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { if (m_condition) { - return "!(" + m_condition->describe_expression() + ")"; + return "!(" + m_condition->describe_expression(state) + ")"; } return "!()"; } @@ -1939,17 +1899,12 @@ class TwoColumnsNode : public ParentNode { do_verify_column(m_getter2.m_column, m_condition_column_idx2); } - virtual std::string describe_column() const override - { - REALM_ASSERT(m_condition_column != nullptr); - return ParentNode::describe_column(m_condition_column->get_column_index()); - } - - virtual std::string describe() const override + virtual std::string describe(util::serializer::SerialisationState& state) const override { REALM_ASSERT(m_getter1.m_column != nullptr && m_getter2.m_column != nullptr); - return ParentNode::describe_column(m_getter1.m_column->get_column_index()) + " " + describe_condition() + " " - + ParentNode::describe_column(m_getter2.m_column->get_column_index()); + return state.describe_column(ParentNode::m_table, m_getter1.m_column->get_column_index()) + + " " + describe_condition() + " " + + state.describe_column(ParentNode::m_table,m_getter2.m_column->get_column_index()); } virtual std::string describe_condition() const override @@ -2054,7 +2009,7 @@ class ExpressionNode : public ParentNode { void table_changed() override; void verify_column() const override; - virtual std::string describe() const override; + virtual std::string describe(util::serializer::SerialisationState& state) const override; std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override; void apply_handover_patch(QueryNodeHandoverPatches& patches, Group& group) override; @@ -2093,13 +2048,7 @@ class LinksToNode : public ParentNode { do_verify_column(m_column, m_origin_column); } - virtual std::string describe_column() const override - { - REALM_ASSERT(m_column != nullptr); - return ParentNode::describe_column(m_column->get_column_index()); - } - - virtual std::string describe() const override + virtual std::string describe(util::serializer::SerialisationState&) const override { throw SerialisationError("Serialising a query which links to an object is currently unsupported."); // We can do something like the following when core gets stable keys diff --git a/src/realm/query_expression.hpp b/src/realm/query_expression.hpp index 3c3b4d642c0..29a63dee0dc 100644 --- a/src/realm/query_expression.hpp +++ b/src/realm/query_expression.hpp @@ -398,7 +398,7 @@ class Expression { virtual void set_base_table(const Table* table) = 0; virtual void verify_column() const = 0; virtual const Table* get_base_table() const = 0; - virtual std::string description() const = 0; + virtual std::string description(util::serializer::SerialisationState& state) const = 0; virtual std::unique_ptr clone(QueryNodeHandoverPatches*) const = 0; virtual void apply_handover_patch(QueryNodeHandoverPatches&, Group&) @@ -435,7 +435,7 @@ class Subexpr { } virtual void verify_column() const = 0; - virtual std::string description() const = 0; + virtual std::string description(util::serializer::SerialisationState& state) const = 0; // Recursively fetch tables of columns in expression tree. Used when user first builds a stand-alone expression // and @@ -1145,7 +1145,7 @@ struct TrueExpression : Expression { void verify_column() const override { } - std::string description() const override + std::string description(util::serializer::SerialisationState&) const override { return "TRUEPREDICATE"; } @@ -1167,7 +1167,7 @@ struct FalseExpression : Expression { void verify_column() const override { } - std::string description() const override + std::string description(util::serializer::SerialisationState&) const override { return "FALSEPREDICATE"; } @@ -1226,7 +1226,7 @@ class Value : public ValueBase, public Subexpr2 { { } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState&) const override { if (ValueBase::m_from_link_list) { return util::serializer::print_value(util::to_string(ValueBase::m_values) @@ -1855,17 +1855,12 @@ class LinkMap { } } - virtual std::string description() const + virtual std::string description(util::serializer::SerialisationState& state) const { std::string s; for (size_t i = 0; i < m_link_column_indexes.size(); ++i) { if (i < m_tables.size() && m_tables[i]) { - if (m_link_types[i] == col_type_BackLink) { - throw SerialisationError("Serialising a query which contains backlinks is currently unsupported."); - //s += "backlink"; - } else if (m_link_column_indexes[i] < m_tables[i]->get_column_count()) { - s += std::string(m_tables[i]->get_column_name(m_link_column_indexes[i])); - } + s += state.get_column_name(m_tables[i]->get_table_ref(), m_link_column_indexes[i]); if (i != m_link_column_indexes.size() - 1) { s += util::serializer::value_separator; } @@ -1909,6 +1904,11 @@ class LinkMap { return m_tables.back(); } + bool links_exist() const + { + return !m_link_columns.empty(); + } + std::vector m_link_columns; private: @@ -2072,17 +2072,9 @@ class SimpleQuerySupport : public Subexpr2 { return m_link_map.m_link_columns.size() > 0; } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - std::string desc; - if (links_exist()) { - desc = m_link_map.description() + util::serializer::value_separator; - } - const Table* target_table = m_link_map.target_table(); - if (target_table && target_table->is_attached()) { - desc += std::string(target_table->get_column_name(m_column_ndx)); - } - return desc; + return state.describe_columns(m_link_map, m_column_ndx); } std::unique_ptr clone(QueryNodeHandoverPatches* patches = nullptr) const override @@ -2294,9 +2286,9 @@ class UnaryLinkCompare : public Expression { return not_found; } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - return m_link_map.description() + (has_links ? " != NULL" : " == NULL"); + return state.describe_columns(m_link_map, realm::npos) + (has_links ? " != NULL" : " == NULL"); } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override @@ -2352,9 +2344,9 @@ class LinkCount : public Subexpr2 { destination.import(Value(false, 1, count)); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - return m_link_map.description() + util::serializer::value_separator + "@count"; + return state.describe_columns(m_link_map, realm::npos) + util::serializer::value_separator + "@count"; } private: @@ -2411,10 +2403,10 @@ class SizeOperator : public Subexpr2 { } } - std::string description() const override + std::string description(util::serializer::SerialisationState& state) const override { if (m_expr) { - return m_expr->description() + util::serializer::value_separator + "@size"; + return m_expr->description(state) + util::serializer::value_separator + "@size"; } return "@size"; } @@ -2475,7 +2467,7 @@ class ConstantRowValue : public Subexpr2 { } } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState&) const override { throw SerialisationError("Serialising a query which links to an object is currently unsupported."); // TODO: we can do something like the following when core gets stable keys: @@ -2570,9 +2562,9 @@ class Columns : public Subexpr2 { m_link_map.verify_columns(); } - std::string description() const override + std::string description(util::serializer::SerialisationState& state) const override { - return m_link_map.description(); + return state.describe_columns(m_link_map, realm::npos); } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override @@ -2634,9 +2626,9 @@ class Columns : public Subexpr2 { m_link_map.target_table()->verify_column(m_column_ndx, m_column); } - std::string description() const override + std::string description(util::serializer::SerialisationState&) const override { - return m_link_map.description(); + throw SerialisationError("Serialisation of query expressions involving subtables is not yet supported."); } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override @@ -2757,19 +2749,9 @@ class ListColumnsBase : public Subexpr2 { destination.import(v); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState&) const override { - const Table* table = get_base_table(); - if (table && table->is_attached()) { - if (m_subtable_column.m_column) { - return std::string(table->get_column_name(m_subtable_column.m_column_ndx)); - - } - else { - return std::string(table->get_column_name(m_column_ndx)); - } - } - return ""; + throw SerialisationError("Serialisation of subtable expressions is not yet supported."); } ListColumnAggregate> min() const @@ -2889,13 +2871,9 @@ class ListColumnAggregate : public Subexpr2 { destination.import(v); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState&) const override { - const Table* table = get_base_table(); - if (table && table->is_attached()) { - return std::string(table->get_column_name(m_column_ndx)) + util::serializer::value_separator + Operation::description() + "()"; - } - return ""; + throw SerialisationError("Serialisation of queries involving subtable expressions is not yet supported."); } private: @@ -3135,18 +3113,9 @@ class Columns : public Subexpr2 { } } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - std::string desc = ""; - if (links_exist()) { - desc = m_link_map.description() + util::serializer::value_separator; - } - const Table* target_table = m_link_map.target_table(); - if (target_table && target_table->is_attached() && m_column_ndx != npos) { - desc += std::string(target_table->get_column_name(m_column_ndx)); - return desc; - } - return ""; + return state.describe_columns(m_link_map, m_column_ndx); } // Load values from Column into destination @@ -3237,7 +3206,7 @@ class SubColumns : public Subexpr { REALM_ASSERT(false); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState&) const override { return ""; // by itself there are no conditions, see SubColumnAggregate } @@ -3338,9 +3307,10 @@ class SubColumnAggregate : public Subexpr2 { } } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - return m_link_map.description() + util::serializer::value_separator + Operation::description() + util::serializer::value_separator + m_column.description(); + util::serializer::SerialisationState empty_state; + return state.describe_columns(m_link_map, realm::npos) + util::serializer::value_separator + Operation::description() + util::serializer::value_separator + m_column.description(empty_state); } private: @@ -3387,11 +3357,16 @@ class SubQueryCount : public Subexpr2 { destination.import(Value(false, 1, size_t(count))); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { - throw SerialisationError("Serialising a subquery expression is currently unsupported."); - //return m_link_map.description() + util::serializer::value_separator + "SUBQUERY(" + m_query.get_description() + ")" - // + util::serializer::value_separator + "@count"; + REALM_ASSERT(m_link_map.base_table() != nullptr); + std::string target = state.describe_columns(m_link_map, realm::npos); + std::string var_name = state.get_variable_name(m_link_map.base_table()->get_table_ref()); + state.subquery_prefix_list.push_back(var_name); + std::string desc = "SUBQUERY(" + target + ", " + var_name + ", " + m_query.get_description(state) + ")" + + util::serializer::value_separator + "@count"; + state.subquery_prefix_list.pop_back(); + return desc; } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override @@ -3609,10 +3584,10 @@ class UnaryOperator : public Subexpr2 { destination.import(result); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { if (m_left) { - return m_left->description(); + return m_left->description(state); } return ""; } @@ -3700,15 +3675,15 @@ class Operator : public Subexpr2 { destination.import(result); } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { std::string s; if (m_left) { - s += m_left->description(); + s += m_left->description(state); } s += (" " + oper::description() + " "); if (m_right) { - s += m_right->description(); + s += m_right->description(state); } return s; } @@ -3790,7 +3765,7 @@ class Compare : public Expression { return not_found; // no match } - virtual std::string description() const override + virtual std::string description(util::serializer::SerialisationState& state) const override { if (std::is_same::value || std::is_same::value @@ -3802,11 +3777,11 @@ class Compare : public Expression { || std::is_same::value) { // these string conditions have the arguments reversed but the order is important // operations ==, and != can be reversed because the produce the same results both ways - return util::serializer::print_value(m_right->description() + " " + TCond::description() - + " " + m_left->description()); + return util::serializer::print_value(m_right->description(state) + " " + TCond::description() + + " " + m_left->description(state)); } - return util::serializer::print_value(m_left->description() + " " + TCond::description() - + " " + m_right->description()); + return util::serializer::print_value(m_left->description(state) + " " + TCond::description() + + " " + m_right->description(state)); } std::unique_ptr clone(QueryNodeHandoverPatches* patches) const override diff --git a/src/realm/table.cpp b/src/realm/table.cpp index bd008c8c20c..c0790498c74 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -2032,6 +2032,24 @@ size_t Table::get_size_from_ref(ref_type spec_ref, ref_type columns_ref, Allocat return size; } +Table::BacklinkOrigin Table::find_backlink_origin(StringData origin_table_name, StringData origin_col_name) const noexcept +{ + size_t backlink_columns_begin = m_spec->first_backlink_column_index(); + size_t backlink_columns_end = backlink_columns_begin + m_spec->backlink_column_count(); + + for (size_t i = backlink_columns_begin; i != backlink_columns_end; ++i) { + const BacklinkColumn& backlink_col = get_column_backlink(i); + ConstTableRef origin_table = backlink_col.get_origin_table().get_table_ref(); + const LinkColumnBase& link_col = backlink_col.get_origin_column(); + size_t link_col_ndx = link_col.get_column_index(); + if (origin_table->get_name() == origin_table_name + && origin_table->get_column_name(link_col_ndx) == origin_col_name) { + return BacklinkOrigin{{origin_table, link_col_ndx}}; + } + } + return BacklinkOrigin{}; +} + ref_type Table::create_empty_table(Allocator& alloc) { diff --git a/src/realm/table.hpp b/src/realm/table.hpp index a7a0c272df8..d5b9be7900e 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -173,6 +173,8 @@ class Table { DataType get_column_type(size_t column_ndx) const noexcept; StringData get_column_name(size_t column_ndx) const noexcept; size_t get_column_index(StringData name) const noexcept; + typedef util::Optional> BacklinkOrigin; + BacklinkOrigin find_backlink_origin(StringData origin_table_name, StringData origin_col_name) const noexcept; //@} //@{ @@ -1568,6 +1570,7 @@ class Table { friend class ParentNode; template friend class SequentialGetter; + friend struct util::serializer::SerialisationState; friend class RowBase; friend class LinksToNode; friend class LinkMap; diff --git a/src/realm/util/serializer.cpp b/src/realm/util/serializer.cpp index 4b1cb652ef7..e477ea806f1 100644 --- a/src/realm/util/serializer.cpp +++ b/src/realm/util/serializer.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -99,6 +100,95 @@ std::string print_value<>(realm::Timestamp t) return ss.str(); } + +// The variable name must be unique with respect to the already chosen variables at +// this level of subquery nesting and with respect to the names of the columns in the table. +// This assumes that columns can start with '$' and that we might one day want to support +// referencing the parent table columns in the subquery. This is currently disabled by an assertion in the +// core SubQuery constructor. +std::string SerialisationState::get_variable_name(ConstTableRef table) { + std::string guess_prefix = "$"; + const char start_char = 'x'; + char add_char = start_char; + + auto next_guess = [&]() { + add_char = (((add_char + 1) - 'a') % ('z' - 'a' + 1)) + 'a'; + if (add_char == start_char) { + guess_prefix += add_char; + } + }; + + while (true) { + std::string guess = guess_prefix + add_char; + bool found_duplicate = false; + for (size_t i = 0; i < subquery_prefix_list.size(); ++i) { + if (guess == subquery_prefix_list[i]) { + found_duplicate = true; + break; + } + } + if (found_duplicate) { + next_guess(); + continue; + } + if (table->get_column_index(guess) != realm::npos) { + next_guess(); + continue; + } + return guess; + } +} + +std::string SerialisationState::get_column_name(ConstTableRef table, size_t col_ndx) +{ + ColumnType col_type = table->get_real_column_type(col_ndx); + if (col_type == col_type_BackLink) { + const BacklinkColumn& col = table->get_column_backlink(col_ndx); + std::string source_table_name = col.get_origin_table().get_name(); + std::string source_col_name = col.get_origin_table().get_column_name(col.get_origin_column().get_column_index()); + return "@links" + util::serializer::value_separator + source_table_name + util::serializer::value_separator + source_col_name; + } + else if (col_ndx < table->get_column_count()) { + return std::string(table->get_column_name(col_ndx)); + } + return ""; +} + +std::string SerialisationState::describe_column(ConstTableRef table, size_t col_ndx) +{ + if (table && col_ndx != npos) { + std::string desc; + if (!subquery_prefix_list.empty()) { + desc += subquery_prefix_list.back() + value_separator; + } + desc += get_column_name(table, col_ndx); + return desc; + } + return ""; +} + +std::string SerialisationState::describe_columns(const LinkMap& link_map, size_t target_col_ndx) +{ + std::string desc; + if (!subquery_prefix_list.empty()) { + desc += subquery_prefix_list.back(); + } + if (link_map.links_exist()) { + if (!desc.empty()) { + desc += util::serializer::value_separator; + } + desc += link_map.description(*this); + } + const Table* target = link_map.target_table(); + if (target && target_col_ndx != npos) { + if (!desc.empty()) { + desc += util::serializer::value_separator; + } + desc += get_column_name(target->get_table_ref(), target_col_ndx); + } + return desc; +} + } // namespace serializer } // namespace util } // namespace realm diff --git a/src/realm/util/serializer.hpp b/src/realm/util/serializer.hpp index a9441dff780..ff56f1356ac 100644 --- a/src/realm/util/serializer.hpp +++ b/src/realm/util/serializer.hpp @@ -19,10 +19,12 @@ #ifndef REALM_UTIL_SERIALIZER_HPP #define REALM_UTIL_SERIALIZER_HPP +#include #include #include #include +#include namespace realm { @@ -30,6 +32,7 @@ class BinaryData; struct null; class StringData; class Timestamp; +class LinkMap; namespace util { namespace serializer { @@ -70,6 +73,15 @@ std::string print_value(Optional value) } } +struct SerialisationState +{ + std::string describe_column(ConstTableRef table, size_t col_ndx); + std::string describe_columns(const LinkMap& link_map, size_t target_col_ndx); + std::string get_column_name(ConstTableRef table, size_t col_ndx); + std::string get_variable_name(ConstTableRef table); + std::vector subquery_prefix_list; +}; + } // namespace serializer } // namespace util } // namespace realm diff --git a/test/test_metrics.cpp b/test/test_metrics.cpp index bd390a9a68a..50d87e435c3 100644 --- a/test/test_metrics.cpp +++ b/test/test_metrics.cpp @@ -492,7 +492,7 @@ TEST(Metrics_LinkQueries) q0.find_all(); q1.find_all(); q2.find_all(); - CHECK_THROW(q3.find_all(), SerialisationError); + q3.find_all();; std::shared_ptr metrics = sg.get_metrics(); CHECK(metrics); @@ -502,8 +502,7 @@ TEST(Metrics_LinkQueries) // FIXME: q3 adds 6 queries: the find_all() + 1 sub query per row in person // that's how subqueries across links are executed currently so it is accurate // but not sure if this is acceptable for how we track queries - // CHECK_EQUAL(queries->size(), 10); - CHECK_EQUAL(queries->size(), 3); + CHECK_EQUAL(queries->size(), 10); std::string null_links_description = queries->at(0).get_description(); CHECK_EQUAL(find_count(null_links_description, "NULL"), 1); @@ -519,13 +518,12 @@ TEST(Metrics_LinkQueries) CHECK_EQUAL(find_count(count_link_description, pet_link_col_name), 1); CHECK_EQUAL(find_count(count_link_description, "=="), 1); -// CHECK_THROW(queries->at(3), SerialisationError); -// std::string link_subquery_description = queries->at(3).get_description(); -// CHECK_EQUAL(find_count(link_subquery_description, "@count"), 1); -// CHECK_EQUAL(find_count(link_subquery_description, pet_link_col_name), 1); -// CHECK_EQUAL(find_count(link_subquery_description, "=="), 1); -// CHECK_EQUAL(find_count(link_subquery_description, column_names[0]), 1); -// CHECK_EQUAL(find_count(link_subquery_description, ">"), 1); + std::string link_subquery_description = queries->at(3).get_description(); + CHECK_EQUAL(find_count(link_subquery_description, "@count"), 1); + CHECK_EQUAL(find_count(link_subquery_description, pet_link_col_name), 1); + CHECK_EQUAL(find_count(link_subquery_description, "=="), 1); + CHECK_EQUAL(find_count(link_subquery_description, column_names[0]), 1); + CHECK_EQUAL(find_count(link_subquery_description, ">"), 1); } @@ -569,14 +567,14 @@ TEST(Metrics_LinkListQueries) q2.find_all(); CHECK_THROW(q3.find_all(), SerialisationError); q4.find_all(); - CHECK_THROW(q5.find_all(), SerialisationError); + q5.find_all(); std::shared_ptr metrics = sg.get_metrics(); CHECK(metrics); std::unique_ptr queries = metrics->take_queries(); CHECK(queries); - CHECK_EQUAL(queries->size(), 4); + CHECK_EQUAL(queries->size(), 16); std::string null_links_description = queries->at(0).get_description(); CHECK_EQUAL(find_count(null_links_description, "NULL"), 1); @@ -601,11 +599,11 @@ TEST(Metrics_LinkListQueries) // the query system can choose to flip the argument order and operators so that >= might be <= CHECK_EQUAL(find_count(sum_link_description, "<=") + find_count(sum_link_description, ">="), 1); -// std::string link_subquery_description = queries->at(4).get_description(); -// CHECK_EQUAL(find_count(link_subquery_description, "@count"), 1); -// CHECK_EQUAL(find_count(link_subquery_description, column_names[ll_col_ndx]), 1); -// CHECK_EQUAL(find_count(link_subquery_description, "=="), 2); -// CHECK_EQUAL(find_count(link_subquery_description, column_names[str_col_ndx]), 1); + std::string link_subquery_description = queries->at(4).get_description(); + CHECK_EQUAL(find_count(link_subquery_description, "@count"), 1); + CHECK_EQUAL(find_count(link_subquery_description, column_names[ll_col_ndx]), 1); + CHECK_EQUAL(find_count(link_subquery_description, "=="), 2); + CHECK_EQUAL(find_count(link_subquery_description, column_names[str_col_ndx]), 1); } @@ -673,13 +671,13 @@ TEST(Metrics_SubQueries) Query q2 = table->column(1).list().begins_with("Str"); Query q3 = table->column(1).list() == "Str_0"; - q0.find_all(); - q1.find_all(); - q2.find_all(); - q3.find_all(); + CHECK_THROW(q0.find_all(), SerialisationError); + CHECK_THROW(q1.find_all(), SerialisationError); + CHECK_THROW(q2.find_all(), SerialisationError); + CHECK_THROW(q3.find_all(), SerialisationError); sg.commit(); - +/* std::shared_ptr metrics = sg.get_metrics(); CHECK(metrics); std::unique_ptr queries = metrics->take_queries(); @@ -702,6 +700,7 @@ TEST(Metrics_SubQueries) std::string str_equal_description = queries->at(3).get_description(); CHECK_EQUAL(find_count(str_equal_description, "=="), 1); CHECK_EQUAL(find_count(str_equal_description, str_col_name), 1); +*/ } diff --git a/test/test_parser.cpp b/test/test_parser.cpp index bb5707d1ef0..5ef1c56309a 100644 --- a/test/test_parser.cpp +++ b/test/test_parser.cpp @@ -104,6 +104,11 @@ static std::vector valid_queries = { "a09._br.z = __-__.Z-9", "$0 = $19", "$0=$0", + // properties can contain '$' + "a$a = a", + "$-1 = $0", + "$a = $0", + "$ = $", // operators "0=0", @@ -190,6 +195,16 @@ static std::vector valid_queries = { "a == b sort(a ASC, b DESC) DISTINCT(p) sort(c ASC, d DESC) DISTINCT(q.r)", "a == b and c==d sort(a ASC, b DESC) DISTINCT(p) sort(c ASC, d DESC) DISTINCT(q.r)", "a == b sort( a ASC , b DESC) and c==d DISTINCT( p ) sort( c ASC , d DESC ) DISTINCT( q.r , p) ", + + // subquery expression + "SUBQUERY(items, $x, $x.name == 'Tom').@size > 0", + "SUBQUERY(items, $x, $x.name == 'Tom').@count > 0", + "SUBQUERY(items, $x, $x.allergens.@min.population_affected < 0.10).@count > 0", + "SUBQUERY(items, $x, $x.name == 'Tom').@count == SUBQUERY(items, $x, $x.price < 10).@count", + + // backlinks + "p.@links.class.prop.@count > 2", + "p.@links.class.prop.@sum.prop2 > 2", }; static std::vector invalid_queries = { @@ -212,11 +227,7 @@ static std::vector invalid_queries = { "0x = 1", "- = a", "a..b = a", - "a$a = a", "{} = $0", - "$-1 = $0", - "$a = $0", - "$ = $", // operators "0===>0", @@ -267,6 +278,22 @@ static std::vector invalid_queries = { "a=b DISTINCT(p", // no braces "a=b sort(p.q DESC a ASC)", // missing comma "a=b DISTINCT(p q)", // missing comma + + // subquery + "SUBQUERY(items, $x, $x.name == 'Tom') > 0", // missing .@count + "SUBQUERY(items, $x, $x.name == 'Tom').@min > 0", // @min not yet supported + "SUBQUERY(items, $x, $x.name == 'Tom').@max > 0", // @max not yet supported + "SUBQUERY(items, $x, $x.name == 'Tom').@sum > 0", // @sum not yet supported + "SUBQUERY(items, $x, $x.name == 'Tom').@avg > 0", // @avg not yet supported + "SUBQUERY(items, var, var.name == 'Tom').@avg > 0", // variable must start with '$' + "SUBQUERY(, $x, $x.name == 'Tom').@avg > 0", // a target keypath is required + "SUBQUERY(items, , name == 'Tom').@avg > 0", // a variable name is required + "SUBQUERY(items, $x, ).@avg > 0", // the subquery is required + + // no @ allowed in keypaths except for keyword '@links' + "@prop > 2", + "@backlinks.@count > 2", + "prop@links > 2", }; TEST(Parser_valid_queries) { @@ -290,17 +317,18 @@ TEST(Parser_grammar_analysis) Query verify_query(test_util::unit_test::TestContext& test_context, TableRef t, std::string query_string, size_t num_results) { Query q = t->where(); + realm::query_builder::NoArguments args; - realm::parser::Predicate p = realm::parser::parse(query_string).predicate; - realm::query_builder::apply_predicate(q, p); + parser::ParserResult res = realm::parser::parse(query_string); + realm::query_builder::apply_predicate(q, res.predicate, args); CHECK_EQUAL(q.count(), num_results); std::string description = q.get_description(); //std::cerr << "original: " << query_string << "\tdescribed: " << description << "\n"; Query q2 = t->where(); - realm::parser::Predicate p2 = realm::parser::parse(description).predicate; - realm::query_builder::apply_predicate(q2, p2); + parser::ParserResult res2 = realm::parser::parse(description); + realm::query_builder::apply_predicate(q2, res2.predicate, args); CHECK_EQUAL(q2.count(), num_results); return q2; @@ -323,7 +351,8 @@ TEST(Parser_empty_input) CHECK(!empty_description.empty()); CHECK_EQUAL(0, empty_description.compare("TRUEPREDICATE")); realm::parser::Predicate p = realm::parser::parse(empty_description).predicate; - realm::query_builder::apply_predicate(q, p); + query_builder::NoArguments args; + realm::query_builder::apply_predicate(q, p, args); CHECK_EQUAL(q.count(), 5); verify_query(test_context, t, "TRUEPREDICATE", 5); @@ -799,8 +828,11 @@ TEST(Parser_NullableBinaries) verify_query(test_context, people, "fav_item.data == fav_item.nullable_data", 3); verify_query(test_context, people, "fav_item.data == fav_item.data", 5); verify_query(test_context, people, "fav_item.nullable_data == fav_item.nullable_data", 5); + + verify_query(test_context, items, "data contains NULL && data contains 'fo' && !(data contains 'asdfasdfasdf') && data contains 'rk'", 1); } + TEST(Parser_OverColumnIndexChanges) { Group g; @@ -1003,7 +1035,7 @@ void verify_query_sub(test_util::unit_test::TestContext& test_context, TableRef Query q2 = t->where(); realm::parser::Predicate p2 = realm::parser::parse(description).predicate; - realm::query_builder::apply_predicate(q2, p2); + realm::query_builder::apply_predicate(q2, p2, args); CHECK_EQUAL(q2.count(), num_results); } @@ -1283,14 +1315,15 @@ TEST(Parser_string_binary_encoding) std::string binary_description = qbin.get_description(); //std::cerr << "original: " << buff << "\tdescribed: " << string_description << "\n"; + query_builder::NoArguments args; Query qstr2 = t->where(); realm::parser::Predicate pstr2 = realm::parser::parse(string_description).predicate; - realm::query_builder::apply_predicate(qstr2, pstr2); + realm::query_builder::apply_predicate(qstr2, pstr2, args); CHECK_EQUAL(qstr2.count(), num_results); Query qbin2 = t->where(); realm::parser::Predicate pbin2 = realm::parser::parse(binary_description).predicate; - realm::query_builder::apply_predicate(qbin2, pbin2); + realm::query_builder::apply_predicate(qbin2, pbin2, args); CHECK_EQUAL(qbin2.count(), num_results); } } @@ -1483,9 +1516,10 @@ TEST(Parser_SortAndDistinctSerialisation) TableView get_sorted_view(TableRef t, std::string query_string) { Query q = t->where(); + query_builder::NoArguments args; parser::ParserResult result = realm::parser::parse(query_string); - realm::query_builder::apply_predicate(q, result.predicate); + realm::query_builder::apply_predicate(q, result.predicate, args); DescriptorOrdering ordering; realm::query_builder::apply_ordering(ordering, t, result.ordering); @@ -1497,7 +1531,7 @@ TableView get_sorted_view(TableRef t, std::string query_string) Query q2 = t->where(); parser::ParserResult result2 = realm::parser::parse(combined); - realm::query_builder::apply_predicate(q2, result2.predicate); + realm::query_builder::apply_predicate(q2, result2.predicate, args); DescriptorOrdering ordering2; realm::query_builder::apply_ordering(ordering2, t, result2.ordering); @@ -1614,4 +1648,551 @@ TEST(Parser_SortAndDistinct) CHECK_EQUAL(message, "No property 'name' found on object type 'account' specified in 'sort' clause"); } + +TEST(Parser_Backlinks) +{ + Group g; + + TableRef items = g.add_table("class_Items"); + size_t item_name_col = items->add_column(type_String, "name"); + size_t item_price_col = items->add_column(type_Double, "price"); + using item_t = std::pair; + std::vector item_info = {{"milk", 5.5}, {"oranges", 4.0}, {"pizza", 9.5}, {"cereal", 6.5}, {"bread", 3.5}}; + for (item_t i : item_info) { + size_t row_ndx = items->add_empty_row(); + items->set_string(item_name_col, row_ndx, i.first); + items->set_double(item_price_col, row_ndx, i.second); + } + + TableRef t = g.add_table("class_Person"); + size_t id_col_ndx = t->add_column(type_Int, "customer_id"); + size_t name_col_ndx = t->add_column(type_String, "name"); + size_t account_col_ndx = t->add_column(type_Double, "account_balance"); + size_t items_col_ndx = t->add_column_link(type_LinkList, "items", *items); + size_t fav_col_ndx = t->add_column_link(type_Link, "fav_item", *items); + t->add_empty_row(3); + for (size_t i = 0; i < t->size(); ++i) { + t->set_int(id_col_ndx, i, i); + t->set_double(account_col_ndx, i, double((i + 1) * 10.0)); + t->set_link(fav_col_ndx, i, i); + } + + t->set_string(name_col_ndx, 0, "Adam"); + LinkViewRef list_0 = t->get_linklist(items_col_ndx, 0); + list_0->add(0); + list_0->add(1); + list_0->add(2); + list_0->add(3); + + t->set_string(name_col_ndx, 1, "James"); + LinkViewRef list_1 = t->get_linklist(items_col_ndx, 1); + for (size_t i = 0; i < 10; ++i) { + list_1->add(0); + } + + t->set_string(name_col_ndx, 2, "John"); + LinkViewRef list_2 = t->get_linklist(items_col_ndx, 2); + list_2->add(2); + list_2->add(2); + list_2->add(3); + + Query q = items->backlink(*t, fav_col_ndx).column(account_col_ndx) > 20; + CHECK_EQUAL(q.count(), 1); + std::string desc = q.get_description(); + CHECK(desc.find("@links.class_Person.fav_item.account_balance") != std::string::npos); + + q = items->backlink(*t, items_col_ndx).column(account_col_ndx) > 20; + CHECK_EQUAL(q.count(), 2); + desc = q.get_description(); + CHECK(desc.find("@links.class_Person.items.account_balance") != std::string::npos); + + // favourite items bought by people who have > 20 in their account + verify_query(test_context, items, "@links.class_Person.fav_item.account_balance > 20", 1); // backlinks via link + // items bought by people who have > 20 in their account + verify_query(test_context, items, "@links.class_Person.items.account_balance > 20", 2); // backlinks via list + // items bought by people who have 'J' as the first letter of their name + verify_query(test_context, items, "@links.class_Person.items.name LIKE[c] 'j*'", 3); + verify_query(test_context, items, "@links.class_Person.items.name BEGINSWITH 'J'", 3); + + // items purchased more than twice + verify_query(test_context, items, "@links.class_Person.items.@count > 2", 2); + verify_query(test_context, items, "@LINKS.class_Person.items.@size > 2", 2); + // items bought by people with only $10 in their account + verify_query(test_context, items, "@links.class_Person.items.@min.account_balance <= 10", 4); + // items bought by people with more than $10 in their account + verify_query(test_context, items, "@links.class_Person.items.@max.account_balance > 10", 3); + // items bought where the sum of the account balance of purchasers is more than $20 + verify_query(test_context, items, "@links.class_Person.items.@sum.account_balance > 20", 3); + verify_query(test_context, items, "@links.class_Person.items.@avg.account_balance > 20", 1); + + // subquery over backlinks + verify_query(test_context, items, "SUBQUERY(@links.class_Person.items, $x, $x.account_balance >= 20).@count > 2", 1); + + // backlinks over link + // people having a favourite item which is also the favourite item of another person + verify_query(test_context, t, "fav_item.@links.class_Person.fav_item.@count > 1", 0); + // people having a favourite item which is purchased more than once (by anyone) + verify_query(test_context, t, "fav_item.@links.class_Person.items.@count > 1 ", 2); + + std::string message; + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, items, "@links.class_Person.items == NULL", 1), message); + CHECK_EQUAL(message, "Comparing a list property to 'null' is not supported"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, items, "@links.class_Person.fav_item == NULL", 1), message); + CHECK_EQUAL(message, "Comparing a list property to 'null' is not supported"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, items, "@links.@count > 0", 1), message); + CHECK_EQUAL(message, "'@links' must be proceeded by type name and a property name"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, items, "@links.class_Factory.items > 0", 1), message); + CHECK_EQUAL(message, "No property 'items' found in type 'Factory' which links to type 'Items'"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, items, "@links.class_Person.artifacts > 0", 1), message); + CHECK_EQUAL(message, "No property 'artifacts' found in type 'Person' which links to type 'Items'"); + + // check that arbitrary aliasing for named backlinks works + parser::KeyPathMapping mapping; + mapping.add_mapping(items, "purchasers", "@links.class_Person.items"); + mapping.add_mapping(t, "money", "account_balance"); + query_builder::NoArguments args; + + q = items->where(); + realm::parser::Predicate p = realm::parser::parse("purchasers.@count > 2").predicate; + realm::query_builder::apply_predicate(q, p, args, mapping); + CHECK_EQUAL(q.count(), 2); + + q = items->where(); + p = realm::parser::parse("purchasers.@max.money >= 20").predicate; + realm::query_builder::apply_predicate(q, p, args, mapping); + CHECK_EQUAL(q.count(), 3); + + // disable parsing backlink queries + mapping.set_allow_backlinks(false); + q = items->where(); + p = realm::parser::parse("purchasers.@max.money >= 20").predicate; + CHECK_THROW_ANY_GET_MESSAGE(realm::query_builder::apply_predicate(q, p, args, mapping), message); + CHECK_EQUAL(message, "Querying over backlinks is disabled but backlinks were found in the inverse relationship of property 'items' on type 'Person'"); +} + + +TEST(Parser_SubqueryVariableNames) +{ + Group g; + util::serializer::SerialisationState test_state; + + TableRef test_table = g.add_table("test"); + + CHECK_EQUAL(test_state.get_variable_name(test_table), "$x"); + + for (char c = 'a'; c <= 'z'; ++c) { + std::string col_name = std::string("$") + c; + test_table->add_column(type_Int, col_name); + } + test_state.subquery_prefix_list.push_back("$xx"); + test_state.subquery_prefix_list.push_back("$xy"); + test_state.subquery_prefix_list.push_back("$xz"); + test_state.subquery_prefix_list.push_back("$xa"); + + std::string unique_variable = test_state.get_variable_name(test_table); + + CHECK_EQUAL(unique_variable, "$xb"); +} + + +TEST(Parser_Subquery) +{ + Group g; + + TableRef discounts = g.add_table("class_Discounts"); + size_t discount_name_col = discounts->add_column(type_String, "promotion", true); + size_t discount_off_col = discounts->add_column(type_Double, "reduced_by"); + size_t discount_active_col = discounts->add_column(type_Bool, "active"); + + using discount_t = std::pair; + std::vector discount_info = {{3.0, false}, {2.5, true}, {0.50, true}, {1.50, true}}; + for (discount_t i : discount_info) { + size_t row_ndx = discounts->add_empty_row(); + discounts->set_double(discount_off_col, row_ndx, i.first); + discounts->set_bool(discount_active_col, row_ndx, i.second); + } + discounts->set_string(discount_name_col, 0, "back to school"); + discounts->set_string(discount_name_col, 1, "pizza lunch special"); + discounts->set_string(discount_name_col, 2, "manager's special"); + + TableRef ingredients = g.add_table("class_Allergens"); + size_t ingredient_name_col = ingredients->add_column(type_String, "name"); + size_t population_col = ingredients->add_column(type_Double, "population_affected"); + std::vector> ingredients_list = { {"dairy", 0.75}, {"nuts", 0.01}, {"wheat", 0.01}, {"soy", 0.005} }; + for (size_t i = 0; i < ingredients_list.size(); ++i) { + size_t row_ndx = ingredients->add_empty_row(); + ingredients->set_string(ingredient_name_col, row_ndx, ingredients_list[i].first); + ingredients->set_double(population_col, row_ndx, ingredients_list[i].second); + } + + TableRef items = g.add_table("class_Items"); + size_t item_name_col = items->add_column(type_String, "name"); + size_t item_price_col = items->add_column(type_Double, "price"); + size_t item_discount_col = items->add_column_link(type_Link, "discount", *discounts); + size_t item_contains_col = items->add_column_link(type_LinkList, "allergens", *ingredients); + using item_t = std::pair; + std::vector item_info = {{"milk", 5.5}, {"oranges", 4.0}, {"pizza", 9.5}, {"cereal", 6.5}}; + for (item_t i : item_info) { + size_t row_ndx = items->add_empty_row(); + items->set_string(item_name_col, row_ndx, i.first); + items->set_double(item_price_col, row_ndx, i.second); + } + items->set_link(item_discount_col, 0, 2); // milk -0.50 + items->set_link(item_discount_col, 2, 1); // pizza -2.5 + items->set_link(item_discount_col, 3, 0); // cereal -3.0 inactive + LinkViewRef milk_contains = items->get_linklist(item_contains_col, 0); + milk_contains->add(0); + LinkViewRef pizza_contains = items->get_linklist(item_contains_col, 2); + pizza_contains->add(0); + pizza_contains->add(2); + pizza_contains->add(3); + LinkViewRef cereal_contains = items->get_linklist(item_contains_col, 3); + cereal_contains->add(0); + cereal_contains->add(1); + cereal_contains->add(2); + + TableRef t = g.add_table("class_Person"); + size_t id_col_ndx = t->add_column(type_Int, "customer_id"); + size_t account_col_ndx = t->add_column(type_Double, "account_balance"); + size_t items_col_ndx = t->add_column_link(type_LinkList, "items", *items); + size_t fav_col_ndx = t->add_column_link(type_Link, "fav_item", *items); + t->add_empty_row(3); + for (size_t i = 0; i < t->size(); ++i) { + t->set_int(id_col_ndx, i, i); + t->set_double(account_col_ndx, i, double((i + 1) * 10.0)); + t->set_link(fav_col_ndx, i, i); + } + + LinkViewRef list_0 = t->get_linklist(items_col_ndx, 0); + list_0->add(0); + list_0->add(1); + list_0->add(2); + list_0->add(3); + + LinkViewRef list_1 = t->get_linklist(items_col_ndx, 1); + for (size_t i = 0; i < 10; ++i) { + list_1->add(0); + } + + LinkViewRef list_2 = t->get_linklist(items_col_ndx, 2); + list_2->add(2); + list_2->add(2); + list_2->add(3); + + Query q = t->column(items_col_ndx, items->column(item_name_col).contains("a") + && items->column(item_price_col) > 5.0 + && items->link(item_discount_col).column(discount_off_col) > 0.5 + && items->column(item_contains_col).count() > 1).count() > 1; + + std::string subquery_description = q.get_description(); + CHECK(subquery_description.find("SUBQUERY(items, $x,") != std::string::npos); + CHECK(subquery_description.find(" $x.name ") != std::string::npos); + CHECK(subquery_description.find(" $x.price ") != std::string::npos); + CHECK(subquery_description.find(" $x.discount.reduced_by ") != std::string::npos); + CHECK(subquery_description.find(" $x.allergens.@count") != std::string::npos); + TableView tv = q.find_all(); + CHECK_EQUAL(tv.size(), 2); + + // not variations inside/outside subquery, no variable substitution + verify_query(test_context, t, "SUBQUERY(items, $x, TRUEPREDICATE).@count > 0", 3); + verify_query(test_context, t, "!SUBQUERY(items, $x, TRUEPREDICATE).@count > 0", 0); + verify_query(test_context, t, "SUBQUERY(items, $x, !TRUEPREDICATE).@count > 0", 0); + verify_query(test_context, t, "SUBQUERY(items, $x, FALSEPREDICATE).@count == 0", 3); + verify_query(test_context, t, "!SUBQUERY(items, $x, FALSEPREDICATE).@count == 0", 0); + verify_query(test_context, t, "SUBQUERY(items, $x, !FALSEPREDICATE).@count == 0", 0); + + // simple variable substitution + verify_query(test_context, t, "SUBQUERY(items, $x, 5.5 == $x.price ).@count > 0", 2); + // string constraint subquery + verify_query(test_context, t, "SUBQUERY(items, $x, $x.name CONTAINS[c] 'MILK').@count >= 1", 2); + // compound subquery && + verify_query(test_context, t, "SUBQUERY(items, $x, $x.name CONTAINS[c] 'MILK' && $x.price == 5.5).@count >= 1", 2); + // compound subquery || + verify_query(test_context, t, "SUBQUERY(items, $x, $x.name CONTAINS[c] 'MILK' || $x.price >= 5.5).@count >= 1", 3); + // variable name change + verify_query(test_context, t, "SUBQUERY(items, $anyNAME_-0123456789, 5.5 == $anyNAME_-0123456789.price ).@count > 0", 2); + // variable names cannot contain '.' + CHECK_THROW_ANY(verify_query(test_context, t, "SUBQUERY(items, $x.y, 5.5 == $x.y.price ).@count > 0", 2)); + // variable name must begin with '$' + CHECK_THROW_ANY(verify_query(test_context, t, "SUBQUERY(items, x, 5.5 == x.y.price ).@count > 0", 2)); + // subquery with string size + verify_query(test_context, t, "SUBQUERY(items, $x, $x.name.@size == 4).@count > 0", 2); + // subquery with list count + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.@count > 1).@count > 0", 2); + // subquery with list aggregate operation + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.@min.population_affected < 0.10).@count > 0", 2); + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.@max.population_affected > 0.50).@count > 0", 3); + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.@sum.population_affected > 0.75).@count > 0", 2); + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.@avg.population_affected > 0.50).@count > 0", 2); + // two column subquery + verify_query(test_context, t, "SUBQUERY(items, $x, $x.discount.promotion CONTAINS[c] $x.name).@count > 0", 2); + // subquery count (int) vs double + verify_query(test_context, t, "SUBQUERY(items, $x, $x.discount.promotion CONTAINS[c] $x.name).@count < account_balance", 3); + // subquery over link + verify_query(test_context, t, "SUBQUERY(fav_item.allergens, $x, $x.name CONTAINS[c] 'dairy').@count > 0", 2); + // nested subquery + verify_query(test_context, t, "SUBQUERY(items, $x, SUBQUERY($x.allergens, $allergy, $allergy.name CONTAINS[c] 'dairy').@count > 0).@count > 0", 3); + // nested subquery operating on the same table with same variable is not allowed + std::string message; + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "SUBQUERY(items, $x, SUBQUERY($x.discount.@links.class_Items.discount, $x, $x.price > 5).@count > 0).@count > 0", 2), message); + CHECK_EQUAL(message, "Unable to create a subquery expression with variable '$x' since an identical variable already exists in this context"); + + // target property must be a list + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "SUBQUERY(account_balance, $x, TRUEPREDICATE).@count > 0", 3), message); + CHECK_EQUAL(message, "A subquery must operate on a list property, but 'account_balance' is type 'Double'"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "SUBQUERY(fav_item, $x, TRUEPREDICATE).@count > 0", 3), message); + CHECK_EQUAL(message, "A subquery must operate on a list property, but 'fav_item' is type 'Link'"); +} + + +TEST(Parser_AggregateShortcuts) +{ + Group g; + + TableRef ingredients = g.add_table("class_Allergens"); + size_t ingredient_name_col = ingredients->add_column(type_String, "name"); + size_t population_col = ingredients->add_column(type_Double, "population_affected"); + std::vector> ingredients_list = { {"dairy", 0.75}, {"nuts", 0.01}, {"wheat", 0.01}, {"soy", 0.005} }; + for (size_t i = 0; i < ingredients_list.size(); ++i) { + size_t row_ndx = ingredients->add_empty_row(); + ingredients->set_string(ingredient_name_col, row_ndx, ingredients_list[i].first); + ingredients->set_double(population_col, row_ndx, ingredients_list[i].second); + } + + TableRef items = g.add_table("class_Items"); + size_t item_name_col = items->add_column(type_String, "name"); + size_t item_price_col = items->add_column(type_Double, "price"); + size_t item_contains_col = items->add_column_link(type_LinkList, "allergens", *ingredients); + using item_t = std::pair; + std::vector item_info = {{"milk", 5.5}, {"oranges", 4.0}, {"pizza", 9.5}, {"cereal", 6.5}}; + for (item_t i : item_info) { + size_t row_ndx = items->add_empty_row(); + items->set_string(item_name_col, row_ndx, i.first); + items->set_double(item_price_col, row_ndx, i.second); + } + LinkViewRef milk_contains = items->get_linklist(item_contains_col, 0); + milk_contains->add(0); + LinkViewRef pizza_contains = items->get_linklist(item_contains_col, 2); + pizza_contains->add(0); + pizza_contains->add(2); + pizza_contains->add(3); + LinkViewRef cereal_contains = items->get_linklist(item_contains_col, 3); + cereal_contains->add(0); + cereal_contains->add(1); + cereal_contains->add(2); + + TableRef t = g.add_table("class_Person"); + size_t id_col_ndx = t->add_column(type_Int, "customer_id"); + size_t account_col_ndx = t->add_column(type_Double, "account_balance"); + size_t items_col_ndx = t->add_column_link(type_LinkList, "items", *items); + size_t fav_col_ndx = t->add_column_link(type_Link, "fav_item", *items); + t->add_empty_row(3); + for (size_t i = 0; i < t->size(); ++i) { + t->set_int(id_col_ndx, i, i); + t->set_double(account_col_ndx, i, double((i + 1) * 10.0)); + t->set_link(fav_col_ndx, i, i); + } + + LinkViewRef list_0 = t->get_linklist(items_col_ndx, 0); + list_0->add(0); + list_0->add(1); + list_0->add(2); + list_0->add(3); + + LinkViewRef list_1 = t->get_linklist(items_col_ndx, 1); + for (size_t i = 0; i < 10; ++i) { + list_1->add(0); + } + + LinkViewRef list_2 = t->get_linklist(items_col_ndx, 2); + list_2->add(2); + list_2->add(2); + list_2->add(3); + + // any is implied over list properties + verify_query(test_context, t, "items.price == 5.5", 2); + + // check basic equality + verify_query(test_context, t, "ANY items.price == 5.5", 2); // 0, 1 + verify_query(test_context, t, "SOME items.price == 5.5", 2); // 0, 1 + verify_query(test_context, t, "ALL items.price == 5.5", 1); // 1 + verify_query(test_context, t, "NONE items.price == 5.5", 1); // 2 + + // and + verify_query(test_context, t, "customer_id > 0 and ANY items.price == 5.5", 1); + verify_query(test_context, t, "customer_id > 0 and SOME items.price == 5.5", 1); + verify_query(test_context, t, "customer_id > 0 and ALL items.price == 5.5", 1); + verify_query(test_context, t, "customer_id > 0 and NONE items.price == 5.5", 1); + // or + verify_query(test_context, t, "customer_id > 1 or ANY items.price == 5.5", 3); + verify_query(test_context, t, "customer_id > 1 or SOME items.price == 5.5", 3); + verify_query(test_context, t, "customer_id > 1 or ALL items.price == 5.5", 2); + verify_query(test_context, t, "customer_id > 1 or NONE items.price == 5.5", 1); + // not + verify_query(test_context, t, "!(ANY items.price == 5.5)", 1); + verify_query(test_context, t, "!(SOME items.price == 5.5)", 1); + verify_query(test_context, t, "!(ALL items.price == 5.5)", 2); + verify_query(test_context, t, "!(NONE items.price == 5.5)", 2); + + // inside subquery people with any items containing WHEAT + verify_query(test_context, t, "SUBQUERY(items, $x, $x.allergens.name CONTAINS[c] 'WHEAT').@count > 0", 2); + verify_query(test_context, t, "SUBQUERY(items, $x, ANY $x.allergens.name CONTAINS[c] 'WHEAT').@count > 0", 2); + verify_query(test_context, t, "SUBQUERY(items, $x, SOME $x.allergens.name CONTAINS[c] 'WHEAT').@count > 0", 2); + verify_query(test_context, t, "SUBQUERY(items, $x, ALL $x.allergens.name CONTAINS[c] 'WHEAT').@count > 0", 1); + verify_query(test_context, t, "SUBQUERY(items, $x, NONE $x.allergens.name CONTAINS[c] 'WHEAT').@count > 0", 2); + + // backlinks + verify_query(test_context, items, "ANY @links.class_Person.items.account_balance > 15", 3); + verify_query(test_context, items, "SOME @links.class_Person.items.account_balance > 15", 3); + verify_query(test_context, items, "ALL @links.class_Person.items.account_balance > 15", 0); + verify_query(test_context, items, "NONE @links.class_Person.items.account_balance > 15", 1); + + // links in prefix + verify_query(test_context, t, "ANY fav_item.allergens.name CONTAINS 'dairy'", 2); + verify_query(test_context, t, "SOME fav_item.allergens.name CONTAINS 'dairy'", 2); + verify_query(test_context, t, "ALL fav_item.allergens.name CONTAINS 'dairy'", 2); + verify_query(test_context, t, "NONE fav_item.allergens.name CONTAINS 'dairy'", 1); + + // links in suffix + verify_query(test_context, items, "ANY @links.class_Person.items.fav_item.name CONTAINS 'milk'", 4); + verify_query(test_context, items, "SOME @links.class_Person.items.fav_item.name CONTAINS 'milk'", 4); + verify_query(test_context, items, "ALL @links.class_Person.items.fav_item.name CONTAINS 'milk'", 1); + verify_query(test_context, items, "NONE @links.class_Person.items.fav_item.name CONTAINS 'milk'", 0); + + // compare with property + verify_query(test_context, t, "ANY items.name == fav_item.name", 2); + verify_query(test_context, t, "SOME items.name == fav_item.name", 2); + verify_query(test_context, t, "ANY items.price == items.@max.price", 3); + verify_query(test_context, t, "SOME items.price == items.@max.price", 3); + verify_query(test_context, t, "ANY items.price == items.@min.price", 3); + verify_query(test_context, t, "SOME items.price == items.@min.price", 3); + verify_query(test_context, t, "ANY items.price > items.@avg.price", 2); + verify_query(test_context, t, "SOME items.price > items.@avg.price", 2); + + // ALL/NONE do not support testing against other columns currently because of how they are implemented in a subquery + // The restriction is because subqueries must operate on properties on the target table and cannot reference + // properties in the parent scope. This restriction may be lifted if we actually implement ALL/NONE in core. + std::string message; + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "ALL items.name == fav_item.name", 1), message); + CHECK_EQUAL(message, "The comparison in an 'ALL' clause must be between a keypath and a value"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "NONE items.name == fav_item.name", 1), message); + CHECK_EQUAL(message, "The comparison in an 'NONE' clause must be between a keypath and a value"); + + // no list in path should throw + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "ANY fav_item.name == 'milk'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ANY' or 'SOME' must contain a list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "SOME fav_item.name == 'milk'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ANY' or 'SOME' must contain a list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "ALL fav_item.name == 'milk'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ALL' must contain a list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "NONE fav_item.name == 'milk'", 1), message); + CHECK_EQUAL(message, "The keypath following 'NONE' must contain a list"); + + // multiple lists in path should throw + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "ANY items.allergens.name == 'dairy'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ANY' or 'SOME' must contain only one list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "SOME items.allergens.name == 'dairy'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ANY' or 'SOME' must contain only one list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "ALL items.allergens.name == 'dairy'", 1), message); + CHECK_EQUAL(message, "The keypath following 'ALL' must contain only one list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "NONE items.allergens.name == 'dairy'", 1), message); + CHECK_EQUAL(message, "The keypath following 'NONE' must contain only one list"); + + // the expression following ANY/SOME/ALL/NONE must be a keypath list + // currently this is restricted by the parser syntax so it is a predicate error + CHECK_THROW_ANY(verify_query(test_context, t, "ANY 'milk' == fav_item.name", 1)); + CHECK_THROW_ANY(verify_query(test_context, t, "SOME 'milk' == fav_item.name", 1)); + CHECK_THROW_ANY(verify_query(test_context, t, "ALL 'milk' == fav_item.name", 1)); + CHECK_THROW_ANY(verify_query(test_context, t, "NONE 'milk' == fav_item.name", 1)); +} + + +TEST(Parser_OperatorIN) +{ + Group g; + + TableRef ingredients = g.add_table("class_Allergens"); + size_t ingredient_name_col = ingredients->add_column(type_String, "name"); + size_t population_col = ingredients->add_column(type_Double, "population_affected"); + std::vector> ingredients_list = { {"dairy", 0.75}, {"nuts", 0.01}, {"wheat", 0.01}, {"soy", 0.005} }; + for (size_t i = 0; i < ingredients_list.size(); ++i) { + size_t row_ndx = ingredients->add_empty_row(); + ingredients->set_string(ingredient_name_col, row_ndx, ingredients_list[i].first); + ingredients->set_double(population_col, row_ndx, ingredients_list[i].second); + } + + TableRef items = g.add_table("class_Items"); + size_t item_name_col = items->add_column(type_String, "name"); + size_t item_price_col = items->add_column(type_Double, "price", true); + size_t item_contains_col = items->add_column_link(type_LinkList, "allergens", *ingredients); + using item_t = std::pair; + std::vector item_info = {{"milk", 5.5}, {"oranges", 4.0}, {"pizza", 9.5}, {"cereal", 6.5}}; + for (item_t i : item_info) { + size_t row_ndx = items->add_empty_row(); + items->set_string(item_name_col, row_ndx, i.first); + items->set_double(item_price_col, row_ndx, i.second); + } + LinkViewRef milk_contains = items->get_linklist(item_contains_col, 0); + milk_contains->add(0); + LinkViewRef pizza_contains = items->get_linklist(item_contains_col, 2); + pizza_contains->add(0); + pizza_contains->add(2); + pizza_contains->add(3); + LinkViewRef cereal_contains = items->get_linklist(item_contains_col, 3); + cereal_contains->add(0); + cereal_contains->add(1); + cereal_contains->add(2); + + TableRef t = g.add_table("class_Person"); + size_t id_col_ndx = t->add_column(type_Int, "customer_id"); + size_t account_col_ndx = t->add_column(type_Double, "account_balance"); + size_t items_col_ndx = t->add_column_link(type_LinkList, "items", *items); + size_t fav_col_ndx = t->add_column_link(type_Link, "fav_item", *items); + t->add_empty_row(3); + for (size_t i = 0; i < t->size(); ++i) { + t->set_int(id_col_ndx, i, i); + t->set_double(account_col_ndx, i, double((i + 1) * 10.0)); + t->set_link(fav_col_ndx, i, i); + } + + LinkViewRef list_0 = t->get_linklist(items_col_ndx, 0); + list_0->add(0); + list_0->add(1); + list_0->add(2); + list_0->add(3); + + LinkViewRef list_1 = t->get_linklist(items_col_ndx, 1); + for (size_t i = 0; i < 10; ++i) { + list_1->add(0); + } + + LinkViewRef list_2 = t->get_linklist(items_col_ndx, 2); + list_2->add(2); + list_2->add(2); + list_2->add(3); + + verify_query(test_context, t, "5.5 IN items.price", 2); + verify_query(test_context, t, "!(5.5 IN items.price)", 1); // group not + verify_query(test_context, t, "'milk' IN items.name", 2); // string compare + verify_query(test_context, t, "'MiLk' IN[c] items.name", 2); // string compare with insensitivity + verify_query(test_context, t, "NULL IN items.price", 0); // null + verify_query(test_context, t, "'dairy' IN fav_item.allergens.name", 2); // through link prefix + verify_query(test_context, items, "20 IN @links.class_Person.items.account_balance", 1); // backlinks + verify_query(test_context, t, "fav_item.price IN items.price", 2); // single property in list + + // aggregate modifiers must operate on a list + CHECK_THROW_ANY(verify_query(test_context, t, "ANY 5.5 IN items.price", 2)); + CHECK_THROW_ANY(verify_query(test_context, t, "SOME 5.5 IN items.price", 2)); + CHECK_THROW_ANY(verify_query(test_context, t, "ALL 5.5 IN items.price", 1)); + CHECK_THROW_ANY(verify_query(test_context, t, "NONE 5.5 IN items.price", 1)); + + std::string message; + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "items.price IN 5.5", 1), message); + CHECK_EQUAL(message, "The expression following 'IN' must be a keypath to a list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "5.5 in fav_item.price", 1), message); + CHECK_EQUAL(message, "The keypath following 'IN' must contain a list"); + CHECK_THROW_ANY_GET_MESSAGE(verify_query(test_context, t, "'dairy' in items.allergens.name", 1), message); + CHECK_EQUAL(message, "The keypath following 'IN' must contain only one list"); + +} #endif // TEST_PARSER