From 909b3c4d0a795a5d7f3604764b2c8c8d15127913 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Wed, 10 May 2023 14:20:28 +0800 Subject: [PATCH] [GIE/Runtime] Remove the dependency of GIE Runtime on GlobalQueryStore to accelerate the compilation (#2681) ## What do these changes do? As titled. Notice that by default compiling (`cargo build` in ir-core), it won't depend on `GlobalQuery` anymore. If you want to compile GIE based on `GlobalQuery` (including Groot and V6d), do `cargo build --features with_global_query`. ## Related issue number Fixes #2679 --- interactive_engine/compiler/Makefile | 1 + interactive_engine/executor/assembly/groot/Cargo.toml | 2 +- interactive_engine/executor/assembly/v6d/Cargo.toml | 3 +-- interactive_engine/executor/ir/core/src/plan/physical.rs | 2 +- interactive_engine/executor/ir/graph_proxy/Cargo.toml | 5 +++-- .../ir/graph_proxy/src/adapters/gs_store/read_graph.rs | 4 ++-- .../executor/ir/graph_proxy/src/adapters/mod.rs | 4 ++++ interactive_engine/executor/ir/graph_proxy/src/lib.rs | 7 +++---- interactive_engine/executor/ir/integrated/Cargo.toml | 5 +++-- .../executor/ir/integrated/src/assemble/mod.rs | 4 ++++ interactive_engine/executor/ir/integrated/src/lib.rs | 4 +++- 11 files changed, 26 insertions(+), 15 deletions(-) diff --git a/interactive_engine/compiler/Makefile b/interactive_engine/compiler/Makefile index dce331ead3d8..d0f417dfc6f3 100644 --- a/interactive_engine/compiler/Makefile +++ b/interactive_engine/compiler/Makefile @@ -31,6 +31,7 @@ clean: test: cd $(CUR_DIR)/../executor/ir && cargo test && \ cd $(CUR_DIR)/../executor/ir && cargo test --features with_v6d && \ + cd $(CUR_DIR)/../executor/ir && cargo test --features with_global_query && \ cd $(CUR_DIR) && mvn test # start rpc server diff --git a/interactive_engine/executor/assembly/groot/Cargo.toml b/interactive_engine/executor/assembly/groot/Cargo.toml index f41df17ea5fa..a64706f17963 100644 --- a/interactive_engine/executor/assembly/groot/Cargo.toml +++ b/interactive_engine/executor/assembly/groot/Cargo.toml @@ -13,7 +13,7 @@ groot-store = { path = "../../store/groot" } gaia_pegasus = { path = "../../engine/pegasus/pegasus", package = "pegasus" } pegasus_network = { path = "../../engine/pegasus/network" } pegasus_server = { path = "../../engine/pegasus/server" } -runtime_integration = { path = "../../ir/integrated" } +runtime_integration = { path = "../../ir/integrated" , features = ["with_global_query"]} log = "0.4" log4rs = "1.2" tokio = { version = "1.24", features = ["macros", "sync"] } diff --git a/interactive_engine/executor/assembly/v6d/Cargo.toml b/interactive_engine/executor/assembly/v6d/Cargo.toml index d440dbee9319..59af4b10e60d 100644 --- a/interactive_engine/executor/assembly/v6d/Cargo.toml +++ b/interactive_engine/executor/assembly/v6d/Cargo.toml @@ -20,5 +20,4 @@ global_query = { path = "../../store/global_query" , features = ["with_v6d"] } pegasus = { path = "../../engine/pegasus/pegasus", package = "pegasus" } pegasus_network = { path = "../../engine/pegasus/network" } pegasus_server = { path = "../../engine/pegasus/server" } -runtime_integration = { path = "../../ir/integrated", features = ["with_v6d"] } - +runtime_integration = { path = "../../ir/integrated", features = ["with_global_query", "with_v6d"] } diff --git a/interactive_engine/executor/ir/core/src/plan/physical.rs b/interactive_engine/executor/ir/core/src/plan/physical.rs index ab43d224faa2..3e0a33e2564d 100644 --- a/interactive_engine/executor/ir/core/src/plan/physical.rs +++ b/interactive_engine/executor/ir/core/src/plan/physical.rs @@ -19,9 +19,9 @@ //! protobuf structure. //! -use ir_common::error::ParsePbError; use std::convert::TryInto; +use ir_common::error::ParsePbError; use ir_common::expr_parse::str_to_expr_pb; use ir_common::generated::algebra as pb; use ir_common::generated::common as common_pb; diff --git a/interactive_engine/executor/ir/graph_proxy/Cargo.toml b/interactive_engine/executor/ir/graph_proxy/Cargo.toml index 5a948042f7ae..ecf0a02a2fdf 100644 --- a/interactive_engine/executor/ir/graph_proxy/Cargo.toml +++ b/interactive_engine/executor/ir/graph_proxy/Cargo.toml @@ -10,7 +10,7 @@ log = "0.4" lazy_static = "1.3.0" mcsr = {path = "../../store/mcsr"} graph_store = {path = "../../store/exp_store"} -global_query = {path = "../../store/global_query"} +global_query = {path = "../../store/global_query", optional = true} dyn_type = { path = "../../common/dyn_type" } ir_common = {path = "../common"} pegasus = { path = "../../engine/pegasus/pegasus" } @@ -21,4 +21,5 @@ rand = "0.8.5" [features] default = [] proto_inplace = ["ir_common/proto_inplace"] -with_v6d = ["global_query/with_v6d"] \ No newline at end of file +with_global_query = ["global_query"] +with_v6d = ["global_query/with_v6d", "with_global_query"] diff --git a/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs b/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs index ee1430ffdc42..c76573b4b05b 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs @@ -29,7 +29,7 @@ use global_query::{ use graph_store::utils::IterList; use ir_common::{KeyId, LabelId, NameOrId, OneOrMany}; -use super::details::{LazyEdgeDetails, LazyVertexDetails}; +use crate::adapters::gs_store::details::{LazyEdgeDetails, LazyVertexDetails}; use crate::apis::graph::PKV; use crate::apis::{ from_fn, register_graph, Direction, DynDetails, Edge, QueryParams, ReadGraph, Statement, Vertex, ID, @@ -609,7 +609,7 @@ fn extract_needed_columns( ) -> GraphProxyResult>> { use ahash::HashSet; - use super::translation::zip_option_vecs; + use crate::adapters::gs_store::translation::zip_option_vecs; // Some(vec[]) means need all props, so can't merge it with props needed in filter if let Some(out_columns) = out_columns { diff --git a/interactive_engine/executor/ir/graph_proxy/src/adapters/mod.rs b/interactive_engine/executor/ir/graph_proxy/src/adapters/mod.rs index d724ecade6a8..95f351f970c6 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/adapters/mod.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/adapters/mod.rs @@ -15,10 +15,14 @@ mod csr_store; mod exp_store; +#[cfg(feature = "with_global_query")] mod gs_store; +#[cfg(feature = "with_global_query")] mod vineyard_store; pub use csr_store::{create_csr_store, CsrPartition}; pub use exp_store::{create_exp_store, SimplePartition}; +#[cfg(feature = "with_global_query")] pub use gs_store::{create_gs_store, GrootMultiPartition, VineyardMultiPartition}; +#[cfg(feature = "with_global_query")] pub use vineyard_store::VineyardGraphWriter; diff --git a/interactive_engine/executor/ir/graph_proxy/src/lib.rs b/interactive_engine/executor/ir/graph_proxy/src/lib.rs index 8a3a4423ef81..b2aa1dd5355b 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/lib.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/lib.rs @@ -17,10 +17,9 @@ extern crate log; #[macro_use] extern crate lazy_static; -pub use adapters::{ - create_csr_store, create_exp_store, create_gs_store, CsrPartition, GrootMultiPartition, - SimplePartition, VineyardGraphWriter, VineyardMultiPartition, -}; +pub use adapters::{create_csr_store, create_exp_store, CsrPartition, SimplePartition}; +#[cfg(feature = "with_global_query")] +pub use adapters::{create_gs_store, GrootMultiPartition, VineyardGraphWriter, VineyardMultiPartition}; pub use errors::{GraphProxyError, GraphProxyResult}; mod adapters; diff --git a/interactive_engine/executor/ir/integrated/Cargo.toml b/interactive_engine/executor/ir/integrated/Cargo.toml index 11962e9ec95c..01d5bec26b9e 100644 --- a/interactive_engine/executor/ir/integrated/Cargo.toml +++ b/interactive_engine/executor/ir/integrated/Cargo.toml @@ -23,10 +23,11 @@ runtime = {path="../runtime"} graph_proxy = {path = "../graph_proxy"} graph_store = {path = "../../store/exp_store"} dyn_type = {path = "../../common/dyn_type"} -global_query = {path = "../../store/global_query"} +global_query = {path = "../../store/global_query", optional = true} [features] default = [] proto_inplace = ["ir_common/proto_inplace", "pegasus_server/gcip"] with_v6d = ["runtime/with_v6d"] -column_filter_push_down = [] \ No newline at end of file +with_global_query = ["global_query", "graph_proxy/with_global_query"] +column_filter_push_down = [] diff --git a/interactive_engine/executor/ir/integrated/src/assemble/mod.rs b/interactive_engine/executor/ir/integrated/src/assemble/mod.rs index fba8ddb6a49b..5bd0d61b9f0c 100644 --- a/interactive_engine/executor/ir/integrated/src/assemble/mod.rs +++ b/interactive_engine/executor/ir/integrated/src/assemble/mod.rs @@ -15,10 +15,14 @@ mod csr; mod exp; +#[cfg(feature = "with_global_query")] mod groot; +#[cfg(feature = "with_global_query")] mod vineyard; pub use csr::QueryCsrGraph; pub use exp::QueryExpGraph; +#[cfg(feature = "with_global_query")] pub use groot::QueryGrootGraph; +#[cfg(feature = "with_global_query")] pub use vineyard::QueryVineyard; diff --git a/interactive_engine/executor/ir/integrated/src/lib.rs b/interactive_engine/executor/ir/integrated/src/lib.rs index c1e3d5578d27..61e55fefdbe7 100644 --- a/interactive_engine/executor/ir/integrated/src/lib.rs +++ b/interactive_engine/executor/ir/integrated/src/lib.rs @@ -14,7 +14,9 @@ //! limitations under the License. mod assemble; -pub use assemble::{QueryCsrGraph, QueryExpGraph, QueryGrootGraph, QueryVineyard}; +pub use assemble::{QueryCsrGraph, QueryExpGraph}; +#[cfg(feature = "with_global_query")] +pub use assemble::{QueryGrootGraph, QueryVineyard}; use runtime::IRJobAssembly; pub trait InitializeJobAssembly {