Repository navigation
feat(kv-ir): Integrate QueryHandler into the IR stream Deserializer.
#890
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 15 commits
7b203a6
e74e31f
e3636fe
10008b5
00e2ecc
ca87b29
8f51308
e7a578b
6791912
7ea6075
dd35aa2
45320b1
bfe5873
3ad1f84
5d4e9e3
f32d575
3985819
85a5798
f4a627f
9f8dc3a
9ee374c
939f0ab
4af0b4a
e6e32e2
27b69c5
2cc5050
c4add7b
c37c224
e4aec2e
f66a2ca
d6b3774
97516f4
eec952b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,7 @@ | |
| #include <memory> | ||
| #include <string> | ||
| #include <system_error> | ||
| #include <tuple> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| #include <nlohmann/json.hpp> | ||
|
|
@@ -20,37 +20,63 @@ | |
| #include "IrUnitHandlerInterface.hpp" | ||
| #include "IrUnitType.hpp" | ||
| #include "protocol_constants.hpp" | ||
| #include "search/QueryHandler.hpp" | ||
| #include "search/QueryHandlerReq.hpp" | ||
| #include "utils.hpp" | ||
|
|
||
| namespace clp::ffi::ir_stream { | ||
| /** | ||
| * A deserializer for reading IR units from a CLP kv-pair IR stream. An IR unit handler should be | ||
| * provided to perform user-defined operations on each deserialized IR unit. | ||
| * provided to perform user-defined operations on each deserialized IR unit. Additionally, a query | ||
| * handler can be provided to handle KQL queries and column projections. | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * | ||
| * NOTE: This class is designed only to provide deserialization functionalities. Callers are | ||
| * responsible for maintaining a `ReaderInterface` to input IR bytes from an I/O stream. | ||
| * | ||
| * @tparam IrUnitHandler | ||
| * @tparam QueryHandlerType | ||
| */ | ||
| template <IrUnitHandlerInterface IrUnitHandler> | ||
| template < | ||
| IrUnitHandlerInterface IrUnitHandler, | ||
| search::QueryHandlerReq QueryHandlerType = search::EmptyQueryHandler> | ||
| requires(std::move_constructible<IrUnitHandler>) | ||
| class Deserializer { | ||
| public: | ||
| // Factory function | ||
| /** | ||
| * Creates a deserializer by reading the stream's preamble from the given reader. | ||
| * Creates a deserializer with an empty query handler. | ||
| * This factory function is used when the deserializer doesn't requires any query evaluations. | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * @param reader | ||
| * @param ir_unit_handler | ||
| * @return A result containing the deserializer or an error code indicating the failure: | ||
| * - std::errc::result_out_of_range if the IR stream is truncated | ||
| * - std::errc::protocol_error if the IR stream is corrupted | ||
| * - std::errc::protocol_not_supported if either: | ||
| * - the IR stream contains an unsupported metadata format; | ||
| * - the IR stream's version is unsupported; | ||
| * - or the IR stream's user-defined metadata is not a JSON object. | ||
| * @return A result containing the deserializer on success, or an error code indicating the | ||
| * failure: | ||
| * - Forwards `create_generic`'s return values. | ||
| */ | ||
| [[nodiscard]] static auto create(ReaderInterface& reader, IrUnitHandler ir_unit_handler) | ||
| -> OUTCOME_V2_NAMESPACE::std_result<Deserializer>; | ||
| -> OUTCOME_V2_NAMESPACE::std_result<Deserializer> | ||
| requires std::is_same_v<QueryHandlerType, search::EmptyQueryHandler> | ||
| { | ||
| return create_generic(reader, std::move(ir_unit_handler), {}); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a deserializer with a query handler. | ||
| * This factory function is used when the deserializer needs to perform query evaluations or | ||
| * column projections. | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * @param reader | ||
| * @param ir_unit_handler | ||
| * @param query_handler | ||
| * @return A result containing the deserializer on success, or an error code indicating the | ||
| * failure: | ||
| * - Forwards `create_generic`'s return values. | ||
| */ | ||
| [[nodiscard]] static auto | ||
| create(ReaderInterface& reader, IrUnitHandler ir_unit_handler, QueryHandlerType query_handler) | ||
| -> OUTCOME_V2_NAMESPACE::std_result<Deserializer> | ||
| requires search::IsQueryHandler<QueryHandlerType>::value | ||
| { | ||
| return create_generic(reader, std::move(ir_unit_handler), std::move(query_handler)); | ||
| } | ||
|
|
||
| // Delete copy constructor and assignment | ||
| Deserializer(Deserializer const&) = delete; | ||
|
|
@@ -65,34 +91,44 @@ class Deserializer { | |
|
|
||
| // Methods | ||
| /** | ||
| * Deserializes the stream from the given reader up to and including the next log event IR unit. | ||
| * Deserializes the stream from the given reader up to and including the next log event IR unit, | ||
| * and invokes the user-defined IR unit handler according to the deserialized IR unit type. | ||
| * | ||
| * NOTE: If the deserialized IR unit is `IrUnitType::LogEvent` and the query handler is not | ||
| * `search::EmptyQueryHandler`, `handle_log_event` will only be invoked if the query evaluation | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * returns `search::AstEvaluationResult::True`. | ||
| * | ||
| * @param reader | ||
| * @return Forwards `deserialize_tag`s return values if no tag bytes can be read to determine | ||
| * the next IR unit type. | ||
| * @return std::errc::protocol_not_supported if the IR unit type is not supported. | ||
| * @return std::errc::operation_not_permitted if the deserializer already reached the end of | ||
| * stream by deserializing an end-of-stream IR unit in the previous calls. | ||
| * @return IRUnitType::LogEvent if a log event IR unit is deserialized, or an error code | ||
| * @return IrUnitType::LogEvent if a log event IR unit is deserialized, or an error code | ||
| * indicating the failure: | ||
| * - Forwards `deserialize_ir_unit_kv_pair_log_event`'s return values if it failed to | ||
| * deserialize and construct the log event. | ||
| * - Forwards `handle_log_event`'s return values from the user-defined IR unit handler on | ||
| * unit handling failure. | ||
| * @return IRUnitType::SchemaTreeNodeInsertion if a schema tree node insertion IR unit is | ||
| * - Forwards `search::QueryHandler::evaluate_kv_pair_log_event`'s return values on failure, if | ||
| * `m_query_handler` is not `search::EmptyQueryHandler`. | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * @return IrUnitType::SchemaTreeNodeInsertion if a schema tree node insertion IR unit is | ||
| * deserialized, or an error code indicating the failure: | ||
| * - Forwards `deserialize_ir_unit_schema_tree_node_insertion`'s return values if it failed to | ||
| * deserialize and construct the schema tree node locator. | ||
| * - Forwards `handle_schema_tree_node_insertion`'s return values from the user-defined IR unit | ||
| * handler on unit handling failure. | ||
| * - Forwards `search::QueryHandler::update_partially_resolved_columns`'s return values on | ||
| * failure, if `m_query_handler` is not `search::EmptyQueryHandler`. | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| * - std::errc::protocol_error if the deserialized schema tree node already exists in the schema | ||
| * tree. | ||
| * @return IRUnitType::UtcOffsetChange if a UTC offset change IR unit is deserialized, or an | ||
| * @return IrUnitType::UtcOffsetChange if a UTC offset change IR unit is deserialized, or an | ||
| * error code indicating the failure: | ||
| * - Forwards `deserialize_ir_unit_utc_offset_change`'s return values if it failed to | ||
| * deserialize the UTC offset. | ||
| * - Forwards `handle_utc_offset_change`'s return values from the user-defined IR unit handler | ||
| * on unit handling failure. | ||
| * @return IRUnitType::EndOfStream if an end-of-stream IR unit is deserialized, or an error code | ||
| * @return IrUnitType::EndOfStream if an end-of-stream IR unit is deserialized, or an error code | ||
| * indicating the failure: | ||
| * - Forwards `handle_end_of_stream`'s return values from the user-defined IR unit handler on | ||
| * unit handling failure. | ||
|
|
@@ -118,10 +154,34 @@ class Deserializer { | |
| [[nodiscard]] auto get_metadata() const -> nlohmann::json const& { return m_metadata; } | ||
|
|
||
| private: | ||
| // Factory function | ||
| /** | ||
| * Creates a deserializer by reading the stream's preamble from the given reader. | ||
| * @param reader | ||
| * @param ir_unit_handler | ||
| * @return A result containing the deserializer or an error code indicating the failure: | ||
| * - std::errc::result_out_of_range if the IR stream is truncated | ||
| * - std::errc::protocol_error if the IR stream is corrupted | ||
| * - std::errc::protocol_not_supported if either: | ||
| * - the IR stream contains an unsupported metadata format; | ||
| * - the IR stream's version is unsupported; | ||
| * - or the IR stream's user-defined metadata is not a JSON object. | ||
| */ | ||
| [[nodiscard]] static auto create_generic( | ||
| ReaderInterface& reader, | ||
| IrUnitHandler ir_unit_handler, | ||
| QueryHandlerType query_handler | ||
|
LinZhihao-723 marked this conversation as resolved.
Resolved
|
||
| ) -> OUTCOME_V2_NAMESPACE::std_result<Deserializer>; | ||
|
|
||
| // Constructor | ||
| Deserializer(IrUnitHandler ir_unit_handler, nlohmann::json metadata) | ||
| Deserializer( | ||
| IrUnitHandler ir_unit_handler, | ||
| nlohmann::json metadata, | ||
| QueryHandlerType query_handler | ||
| ) | ||
| : m_ir_unit_handler{std::move(ir_unit_handler)}, | ||
| m_metadata(std::move(metadata)) {} | ||
| m_metadata(std::move(metadata)), | ||
| m_query_handler{std::move(query_handler)} {} | ||
|
|
||
| // Variables | ||
| std::shared_ptr<SchemaTree> m_auto_gen_keys_schema_tree{std::make_shared<SchemaTree>()}; | ||
|
|
@@ -130,12 +190,16 @@ class Deserializer { | |
| UtcOffset m_utc_offset{0}; | ||
| IrUnitHandler m_ir_unit_handler; | ||
| bool m_is_complete{false}; | ||
| [[no_unique_address]] QueryHandlerType m_query_handler; | ||
| }; | ||
|
|
||
| template <IrUnitHandlerInterface IrUnitHandler> | ||
| template <IrUnitHandlerInterface IrUnitHandler, search::QueryHandlerReq QueryHandlerType> | ||
| requires(std::move_constructible<IrUnitHandler>) | ||
| auto Deserializer<IrUnitHandler>::create(ReaderInterface& reader, IrUnitHandler ir_unit_handler) | ||
| -> OUTCOME_V2_NAMESPACE::std_result<Deserializer> { | ||
| auto Deserializer<IrUnitHandler, QueryHandlerType>::create_generic( | ||
| ReaderInterface& reader, | ||
| IrUnitHandler ir_unit_handler, | ||
| QueryHandlerType query_handler | ||
| ) -> OUTCOME_V2_NAMESPACE::std_result<Deserializer> { | ||
| bool is_four_byte_encoded{}; | ||
| if (auto const err{get_encoding_type(reader, is_four_byte_encoded)}; | ||
|
Comment on lines
+196
to
204
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick (assertive) SFINAE overlap risk between the two Both factory overloads participate in overload resolution when Although currently well-ordered by arity, adding a defaulted parameter to the requires(!std::is_same_v<QueryHandlerType, search::EmptyQueryHandler>)(This mirrors the |
||
| IRErrorCode::IRErrorCode_Success != err) | ||
|
|
@@ -176,13 +240,18 @@ auto Deserializer<IrUnitHandler>::create(ReaderInterface& reader, IrUnitHandler | |
| return std::errc::protocol_not_supported; | ||
| } | ||
|
|
||
| return Deserializer{std::move(ir_unit_handler), std::move(metadata_json)}; | ||
| return Deserializer{ | ||
| std::move(ir_unit_handler), | ||
| std::move(metadata_json), | ||
| std::move(query_handler) | ||
| }; | ||
| } | ||
|
|
||
| template <IrUnitHandlerInterface IrUnitHandler> | ||
| template <IrUnitHandlerInterface IrUnitHandler, search::QueryHandlerReq QueryHandlerType> | ||
| requires(std::move_constructible<IrUnitHandler>) | ||
| auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& reader) | ||
| -> OUTCOME_V2_NAMESPACE::std_result<IrUnitType> { | ||
| auto Deserializer<IrUnitHandler, QueryHandlerType>::deserialize_next_ir_unit( | ||
| ReaderInterface& reader | ||
| ) -> OUTCOME_V2_NAMESPACE::std_result<IrUnitType> { | ||
| if (is_stream_completed()) { | ||
| return std::errc::operation_not_permitted; | ||
| } | ||
|
|
@@ -211,6 +280,14 @@ auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& read | |
| return result.error(); | ||
| } | ||
|
|
||
| if constexpr (search::IsQueryHandler<QueryHandlerType>::value) { | ||
| if (search::AstEvaluationResult::True | ||
| != OUTCOME_TRYX(m_query_handler.evaluate(result.value()))) | ||
| { | ||
| break; | ||
| } | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Equality comparison on a bitmask may drop true-positive matches
if (search::AstEvaluationResult::True
!= OUTCOME_TRYX(m_query_handler.evaluate(result.value())))
{
break;
}If the evaluator later ORs additional flags (e.g., Safer logic: -auto res = OUTCOME_TRYX(m_query_handler.evaluate(result.value()));
-if (0 == (res & search::AstEvaluationResult::True)) { // no TRUE bit ⇒ skip
+auto res = OUTCOME_TRYX(m_query_handler.evaluate(result.value()));
+if (0 == (res & search::AstEvaluationResult::True)) { // TRUE bit absent ⇒ skip
break;
}This change future-proofs the deserializer against extended flag semantics. |
||
|
|
||
| if (auto const err{m_ir_unit_handler.handle_log_event(std::move(result.value()))}; | ||
| IRErrorCode::IRErrorCode_Success != err) | ||
| { | ||
|
|
@@ -237,7 +314,15 @@ auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& read | |
| return std::errc::protocol_error; | ||
| } | ||
|
|
||
| std::ignore = schema_tree_to_insert->insert_node(node_locator); | ||
| auto const node_id{schema_tree_to_insert->insert_node(node_locator)}; | ||
|
|
||
| if constexpr (search::IsQueryHandler<QueryHandlerType>::value) { | ||
| OUTCOME_TRYV(m_query_handler.update_partially_resolved_columns( | ||
| is_auto_generated, | ||
| node_locator, | ||
| node_id | ||
| )); | ||
| } | ||
|
|
||
| if (auto const err{m_ir_unit_handler.handle_schema_tree_node_insertion( | ||
| is_auto_generated, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,11 @@ auto ErrorCategory::message(ErrorCodeEnum error_enum) const -> std::string { | |
| switch (error_enum) { | ||
| case ErrorCodeEnum::AstDynamicCastFailure: | ||
| return "Failed to dynamically cast an AST node to the expected type."; | ||
| case ErrorCodeEnum::AstEvaluationInvariantViolation: | ||
| return "Internal invariant violated during AST evaluation. This indicates a serious " | ||
| "bug in the evaluation logic."; | ||
| case ErrorCodeEnum::AttemptToIterateAstLeafExpr: | ||
| return "Attempted to iterate an leaf expression of an AST."; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧹 Nitpick (assertive) Fixed grammar in error message There's a minor grammatical error in the message for case ErrorCodeEnum::AttemptToIterateAstLeafExpr:
- return "Attempted to iterate an leaf expression of an AST.";
+ return "Attempted to iterate a leaf expression of an AST."; |
||
| case ErrorCodeEnum::ColumnDescriptorTokenIteratorOutOfBounds: | ||
| return "Attempted to access a token beyond the end of the column descriptor."; | ||
| case ErrorCodeEnum::ColumnTokenizationFailure: | ||
|
|
@@ -27,6 +32,8 @@ auto ErrorCategory::message(ErrorCodeEnum error_enum) const -> std::string { | |
| return "The projected column is not unique."; | ||
| case ErrorCodeEnum::EncodedTextAstDecodingFailure: | ||
| return "Failed to decode the given encoded text AST."; | ||
| case ErrorCodeEnum::ExpressionTypeUnexpected: | ||
| return "Unexpected expression type."; | ||
| case ErrorCodeEnum::LiteralTypeUnexpected: | ||
| return "Unexpected literal type."; | ||
| case ErrorCodeEnum::LiteralTypeUnsupported: | ||
|
|
@@ -35,8 +42,6 @@ auto ErrorCategory::message(ErrorCodeEnum error_enum) const -> std::string { | |
| return "The requested method is not implemented."; | ||
| case ErrorCodeEnum::ProjectionColumnDescriptorCreationFailure: | ||
| return "Failed to create a column descriptor for the given projection."; | ||
| case ErrorCodeEnum::QueryExpressionIsNull: | ||
| return "The query expression is NULL."; | ||
| case ErrorCodeEnum::QueryTransformationPassFailed: | ||
| return "Failed to execute transformation passes on the query expression."; | ||
| default: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@coderabbitai review
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No description provided.